2014-11-16 20:13:20 +00:00
// Copyright (C) 2014 The Syncthing Authors.
2014-09-29 19:43:32 +00:00
//
2015-03-07 20:36:35 +00:00
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
2017-02-09 06:52:18 +00:00
// You can obtain one at https://mozilla.org/MPL/2.0/.
2014-06-01 20:50:14 +00:00
2021-05-08 10:52:06 +00:00
//go:generate -command counterfeiter go run github.com/maxbrunsfeld/counterfeiter/v6
2021-03-03 07:53:50 +00:00
//go:generate counterfeiter -o mocks/model.go --fake-name Model . Model
2014-05-15 03:26:55 +00:00
package model
2013-12-15 10:43:31 +00:00
import (
2018-05-05 08:24:44 +00:00
"bytes"
2019-11-19 08:56:53 +00:00
"context"
2015-03-10 22:45:43 +00:00
"encoding/json"
2022-08-16 08:01:49 +00:00
"errors"
2013-12-23 17:12:44 +00:00
"fmt"
2020-11-09 14:33:32 +00:00
"io"
2014-01-05 22:54:57 +00:00
"net"
2021-04-29 17:21:07 +00:00
"os"
2014-03-28 13:36:57 +00:00
"path/filepath"
2015-06-03 07:47:39 +00:00
"reflect"
2015-04-29 18:46:32 +00:00
"runtime"
2014-08-11 18:20:01 +00:00
"strings"
2018-10-05 08:26:25 +00:00
stdsync "sync"
2023-02-07 11:07:34 +00:00
"sync/atomic"
2013-12-15 10:43:31 +00:00
"time"
2014-06-21 07:43:12 +00:00
2020-11-17 12:19:04 +00:00
"github.com/thejerf/suture/v4"
2019-11-23 15:20:54 +00:00
2022-07-28 17:36:39 +00:00
"github.com/syncthing/syncthing/lib/build"
2015-08-06 09:29:25 +00:00
"github.com/syncthing/syncthing/lib/config"
2016-05-04 19:38:12 +00:00
"github.com/syncthing/syncthing/lib/connections"
2015-08-06 09:29:25 +00:00
"github.com/syncthing/syncthing/lib/db"
"github.com/syncthing/syncthing/lib/events"
2016-08-05 17:45:45 +00:00
"github.com/syncthing/syncthing/lib/fs"
2015-08-06 09:29:25 +00:00
"github.com/syncthing/syncthing/lib/ignore"
"github.com/syncthing/syncthing/lib/osutil"
2015-09-22 17:38:46 +00:00
"github.com/syncthing/syncthing/lib/protocol"
2023-09-06 10:52:01 +00:00
"github.com/syncthing/syncthing/lib/rand"
2015-08-06 09:29:25 +00:00
"github.com/syncthing/syncthing/lib/scanner"
2023-08-21 17:44:33 +00:00
"github.com/syncthing/syncthing/lib/semaphore"
2015-08-06 09:29:25 +00:00
"github.com/syncthing/syncthing/lib/stats"
2020-12-22 19:17:14 +00:00
"github.com/syncthing/syncthing/lib/svcutil"
2015-08-06 09:29:25 +00:00
"github.com/syncthing/syncthing/lib/sync"
2020-06-23 08:47:15 +00:00
"github.com/syncthing/syncthing/lib/ur/contract"
2015-08-06 09:29:25 +00:00
"github.com/syncthing/syncthing/lib/versioner"
2013-12-15 10:43:31 +00:00
)
2014-09-30 15:52:05 +00:00
type service interface {
2020-11-17 12:19:04 +00:00
suture . Service
2014-12-30 08:35:21 +00:00
BringToFront ( string )
2019-04-07 11:29:17 +00:00
Override ( )
Revert ( )
2015-05-01 12:30:17 +00:00
DelayScan ( d time . Duration )
2021-08-17 07:23:33 +00:00
ScheduleScan ( )
2019-06-27 18:25:38 +00:00
SchedulePull ( ) // something relevant changed, we should try a pull
Jobs ( page , perpage int ) ( [ ] string , [ ] string , int ) // In progress, Queued, skipped
2015-06-20 17:26:25 +00:00
Scan ( subs [ ] string ) error
2018-11-07 10:04:41 +00:00
Errors ( ) [ ] FileError
2018-02-04 21:46:24 +00:00
WatchError ( ) error
2020-05-01 09:08:59 +00:00
ScheduleForceRescan ( path string )
2019-11-30 12:03:24 +00:00
GetStatistics ( ) ( stats . FolderStatistics , error )
2015-03-16 20:14:19 +00:00
2016-06-29 06:37:34 +00:00
getState ( ) ( folderState , time . Time , error )
2014-09-30 15:52:05 +00:00
}
2016-04-15 10:59:41 +00:00
type Availability struct {
ID protocol . DeviceID ` json:"id" `
FromTemporary bool ` json:"fromTemporary" `
}
2019-02-26 08:09:25 +00:00
type Model interface {
suture . Service
connections . Model
2021-09-11 15:14:47 +00:00
ResetFolder ( folder string ) error
2019-02-26 08:09:25 +00:00
DelayScan ( folder string , next time . Duration )
ScanFolder ( folder string ) error
ScanFolders ( ) map [ string ] error
ScanFolderSubdirs ( folder string , subs [ ] string ) error
State ( folder string ) ( string , time . Time , error )
FolderErrors ( folder string ) ( [ ] FileError , error )
WatchError ( folder string ) error
Override ( folder string )
Revert ( folder string )
BringToFront ( folder , file string )
2021-01-12 15:25:21 +00:00
LoadIgnores ( folder string ) ( [ ] string , [ ] string , error )
CurrentIgnores ( folder string ) ( [ ] string , [ ] string , error )
2019-02-26 08:09:25 +00:00
SetIgnores ( folder string , content [ ] string ) error
GetFolderVersions ( folder string ) ( map [ string ] [ ] versioner . FileVersion , error )
2021-02-12 19:30:51 +00:00
RestoreFolderVersions ( folder string , versions map [ string ] time . Time ) ( map [ string ] error , error )
2019-02-26 08:09:25 +00:00
2020-01-21 17:23:08 +00:00
DBSnapshot ( folder string ) ( * db . Snapshot , error )
2020-11-27 10:26:36 +00:00
NeedFolderFiles ( folder string , page , perpage int ) ( [ ] db . FileInfoTruncated , [ ] db . FileInfoTruncated , [ ] db . FileInfoTruncated , error )
RemoteNeedFolderFiles ( folder string , device protocol . DeviceID , page , perpage int ) ( [ ] db . FileInfoTruncated , error )
LocalChangedFolderFiles ( folder string , page , perpage int ) ( [ ] db . FileInfoTruncated , error )
2020-01-21 17:23:08 +00:00
FolderProgressBytesCompleted ( folder string ) int64
2021-03-07 12:43:22 +00:00
CurrentFolderFile ( folder string , file string ) ( protocol . FileInfo , bool , error )
CurrentGlobalFile ( folder string , file string ) ( protocol . FileInfo , bool , error )
2021-05-03 10:28:25 +00:00
GetMtimeMapping ( folder string , file string ) ( fs . MtimeMapping , error )
2021-03-07 12:43:22 +00:00
Availability ( folder string , file protocol . FileInfo , block protocol . BlockInfo ) ( [ ] Availability , error )
2019-02-26 08:09:25 +00:00
2021-03-07 12:43:22 +00:00
Completion ( device protocol . DeviceID , folder string ) ( FolderCompletion , error )
2019-02-26 08:09:25 +00:00
ConnectionStats ( ) map [ string ] interface { }
2021-01-11 14:14:44 +00:00
DeviceStatistics ( ) ( map [ protocol . DeviceID ] stats . DeviceStatistics , error )
2019-11-30 12:03:24 +00:00
FolderStatistics ( ) ( map [ string ] stats . FolderStatistics , error )
2020-06-23 08:47:15 +00:00
UsageReportingStats ( report * contract . Report , version int , preview bool )
2023-09-06 10:52:01 +00:00
ConnectedTo ( remoteID protocol . DeviceID ) bool
2019-02-26 08:09:25 +00:00
2020-12-17 18:54:31 +00:00
PendingDevices ( ) ( map [ protocol . DeviceID ] db . ObservedDevice , error )
PendingFolders ( device protocol . DeviceID ) ( map [ string ] db . PendingFolder , error )
2021-06-07 08:29:24 +00:00
DismissPendingDevice ( device protocol . DeviceID ) error
DismissPendingFolder ( device protocol . DeviceID , folder string ) error
2020-12-17 18:54:31 +00:00
2019-02-26 08:09:25 +00:00
StartDeadlockDetector ( timeout time . Duration )
2021-02-01 08:27:34 +00:00
GlobalDirectoryTree ( folder , prefix string , levels int , dirsOnly bool ) ( [ ] * TreeEntry , error )
2019-02-26 08:09:25 +00:00
}
type model struct {
2015-06-12 11:04:00 +00:00
* suture . Supervisor
2020-02-01 07:12:25 +00:00
// constructor parameters
cfg config . Wrapper
id protocol . DeviceID
db * db . Lowlevel
protectedFiles [ ] string
evLogger events . Logger
// constant or concurrency safe fields
2020-08-18 07:26:33 +00:00
finder * db . BlockFinder
progressEmitter * ProgressEmitter
shortID protocol . ShortID
2020-02-01 07:12:25 +00:00
// globalRequestLimiter limits the amount of data in concurrent incoming
// requests
2023-08-21 17:44:33 +00:00
globalRequestLimiter * semaphore . Semaphore
2020-02-01 07:02:18 +00:00
// folderIOLimiter limits the number of concurrent I/O heavy operations,
2020-02-01 07:12:25 +00:00
// such as scans and pulls.
2023-08-21 17:44:33 +00:00
folderIOLimiter * semaphore . Semaphore
2020-12-21 11:59:22 +00:00
fatalChan chan error
2021-01-15 14:43:34 +00:00
started chan struct { }
2023-03-12 19:06:59 +00:00
keyGen * protocol . KeyGenerator
2023-09-06 10:52:01 +00:00
promotionTimer * time . Timer
2020-02-01 07:02:18 +00:00
2020-02-01 07:12:25 +00:00
// fields protected by fmut
2020-11-09 14:33:32 +00:00
fmut sync . RWMutex
folderCfgs map [ string ] config . FolderConfiguration // folder -> cfg
folderFiles map [ string ] * db . FileSet // folder -> files
deviceStatRefs map [ protocol . DeviceID ] * stats . DeviceStatisticsReference // deviceID -> statsRef
folderIgnores map [ string ] * ignore . Matcher // folder -> matcher object
2023-09-02 14:42:46 +00:00
folderRunners * serviceMap [ string , service ] // folder -> puller or scanner
2020-11-09 14:33:32 +00:00
folderRestartMuts syncMutexMap // folder -> restart mutex
folderVersioners map [ string ] versioner . Versioner // folder -> versioner (may be nil)
folderEncryptionPasswordTokens map [ string ] [ ] byte // folder -> encryption token (may be missing, and only for encryption type folders)
folderEncryptionFailures map [ string ] map [ protocol . DeviceID ] error // folder -> device -> error regarding encryption consistency (may be missing)
2014-03-29 17:53:48 +00:00
2020-02-01 07:12:25 +00:00
// fields protected by pmut
pmut sync . RWMutex
2023-09-06 10:52:01 +00:00
connections map [ string ] protocol . Connection // connection ID -> connection
deviceConnIDs map [ protocol . DeviceID ] [ ] string // device -> connection IDs (invariant: if the key exists, the value is len >= 1, with the primary connection at the start of the slice)
promotedConnID map [ protocol . DeviceID ] string // device -> latest promoted connection ID
2023-08-21 17:44:33 +00:00
connRequestLimiters map [ protocol . DeviceID ] * semaphore . Semaphore
2023-09-06 10:52:01 +00:00
closed map [ string ] chan struct { } // connection ID -> closed channel
2020-09-29 11:17:38 +00:00
helloMessages map [ protocol . DeviceID ] protocol . Hello
2016-12-21 18:41:25 +00:00
deviceDownloads map [ protocol . DeviceID ] * deviceDownloadState
2022-04-10 20:47:57 +00:00
remoteFolderStates map [ protocol . DeviceID ] map [ string ] remoteFolderState // deviceID -> folders
2023-08-21 16:39:13 +00:00
indexHandlers * serviceMap [ protocol . DeviceID , * indexHandlerRegistry ]
2018-10-05 08:26:25 +00:00
2020-11-17 12:19:04 +00:00
// for testing only
2023-02-07 11:07:34 +00:00
foldersRunning atomic . Int32
2013-12-15 10:43:31 +00:00
}
2021-11-22 07:45:29 +00:00
var _ config . Verifier = & model { }
2023-08-21 17:44:33 +00:00
type folderFactory func ( * model , * db . FileSet , * ignore . Matcher , config . FolderConfiguration , versioner . Versioner , events . Logger , * semaphore . Semaphore ) service
2016-05-04 10:47:33 +00:00
2023-03-12 19:06:59 +00:00
var folderFactories = make ( map [ config . FolderType ] folderFactory )
2014-01-06 20:31:36 +00:00
2016-06-26 10:07:27 +00:00
var (
2022-04-07 15:35:33 +00:00
errDeviceUnknown = errors . New ( "unknown device" )
errDevicePaused = errors . New ( "device is paused" )
ErrFolderPaused = errors . New ( "folder is paused" )
ErrFolderNotRunning = errors . New ( "folder is not running" )
ErrFolderMissing = errors . New ( "no such folder" )
errNoVersioner = errors . New ( "folder has no versioner" )
2019-01-09 16:31:09 +00:00
// errors about why a connection is closed
2021-05-11 05:55:44 +00:00
errStopped = errors . New ( "Syncthing is being stopped" )
errEncryptionInvConfigLocal = errors . New ( "can't encrypt outgoing data because local data is encrypted (folder-type receive-encrypted)" )
errEncryptionInvConfigRemote = errors . New ( "remote has encrypted data and encrypts that data for us - this is impossible" )
errEncryptionNotEncryptedLocal = errors . New ( "remote expects to exchange encrypted data, but is configured for plain data" )
errEncryptionPlainForReceiveEncrypted = errors . New ( "remote expects to exchange plain data, but is configured to be encrypted" )
errEncryptionPlainForRemoteEncrypted = errors . New ( "remote expects to exchange plain data, but local data is encrypted (folder-type receive-encrypted)" )
errEncryptionNotEncryptedUntrusted = errors . New ( "device is untrusted, but configured to receive plain data" )
errEncryptionPassword = errors . New ( "different encryption passwords used" )
errEncryptionTokenRead = errors . New ( "failed to read encryption token" )
errEncryptionTokenWrite = errors . New ( "failed to write encryption token" )
errMissingRemoteInClusterConfig = errors . New ( "remote device missing in cluster config" )
errMissingLocalInClusterConfig = errors . New ( "local device missing in cluster config" )
2016-06-26 10:07:27 +00:00
)
2014-01-06 10:11:18 +00:00
// NewModel creates and starts a new model. The model starts in read-only mode,
// where it sends index information to connected peers and responds to requests
2014-09-28 11:00:38 +00:00
// for file data without altering the local folder in any way.
2023-09-06 10:52:01 +00:00
func NewModel ( cfg config . Wrapper , id protocol . DeviceID , ldb * db . Lowlevel , protectedFiles [ ] string , evLogger events . Logger , keyGen * protocol . KeyGenerator ) Model {
2020-12-22 19:17:14 +00:00
spec := svcutil . SpecWithDebugLogger ( l )
2019-02-26 08:09:25 +00:00
m := & model {
2020-11-17 12:19:04 +00:00
Supervisor : suture . New ( "model" , spec ) ,
2020-02-01 07:12:25 +00:00
// constructor parameters
cfg : cfg ,
id : id ,
db : ldb ,
protectedFiles : protectedFiles ,
evLogger : evLogger ,
// constant or concurrency safe fields
2020-02-01 07:02:18 +00:00
finder : db . NewBlockFinder ( ldb ) ,
progressEmitter : NewProgressEmitter ( cfg , evLogger ) ,
shortID : id . Short ( ) ,
2023-08-21 17:44:33 +00:00
globalRequestLimiter : semaphore . New ( 1024 * cfg . Options ( ) . MaxConcurrentIncomingRequestKiB ( ) ) ,
folderIOLimiter : semaphore . New ( cfg . Options ( ) . MaxFolderConcurrency ( ) ) ,
2020-12-21 11:59:22 +00:00
fatalChan : make ( chan error ) ,
2021-01-15 14:43:34 +00:00
started : make ( chan struct { } ) ,
2023-03-12 19:06:59 +00:00
keyGen : keyGen ,
2023-09-06 10:52:01 +00:00
promotionTimer : time . NewTimer ( 0 ) ,
2020-02-01 07:12:25 +00:00
// fields protected by fmut
2020-11-09 14:33:32 +00:00
fmut : sync . NewRWMutex ( ) ,
folderCfgs : make ( map [ string ] config . FolderConfiguration ) ,
folderFiles : make ( map [ string ] * db . FileSet ) ,
deviceStatRefs : make ( map [ protocol . DeviceID ] * stats . DeviceStatisticsReference ) ,
folderIgnores : make ( map [ string ] * ignore . Matcher ) ,
2023-09-02 14:42:46 +00:00
folderRunners : newServiceMap [ string , service ] ( evLogger ) ,
2020-11-09 14:33:32 +00:00
folderVersioners : make ( map [ string ] versioner . Versioner ) ,
folderEncryptionPasswordTokens : make ( map [ string ] [ ] byte ) ,
folderEncryptionFailures : make ( map [ string ] map [ protocol . DeviceID ] error ) ,
2020-02-01 07:12:25 +00:00
// fields protected by pmut
pmut : sync . NewRWMutex ( ) ,
2023-09-06 10:52:01 +00:00
connections : make ( map [ string ] protocol . Connection ) ,
deviceConnIDs : make ( map [ protocol . DeviceID ] [ ] string ) ,
promotedConnID : make ( map [ protocol . DeviceID ] string ) ,
2023-08-21 17:44:33 +00:00
connRequestLimiters : make ( map [ protocol . DeviceID ] * semaphore . Semaphore ) ,
2023-09-06 10:52:01 +00:00
closed : make ( map [ string ] chan struct { } ) ,
2020-09-29 11:17:38 +00:00
helloMessages : make ( map [ protocol . DeviceID ] protocol . Hello ) ,
2020-02-01 07:12:25 +00:00
deviceDownloads : make ( map [ protocol . DeviceID ] * deviceDownloadState ) ,
2022-04-10 20:47:57 +00:00
remoteFolderStates : make ( map [ protocol . DeviceID ] map [ string ] remoteFolderState ) ,
2023-08-21 16:39:13 +00:00
indexHandlers : newServiceMap [ protocol . DeviceID , * indexHandlerRegistry ] ( evLogger ) ,
2013-12-15 10:43:31 +00:00
}
2023-09-06 10:52:01 +00:00
for devID , cfg := range cfg . Devices ( ) {
2021-01-05 16:45:07 +00:00
m . deviceStatRefs [ devID ] = stats . NewDeviceStatisticsReference ( m . db , devID )
2023-09-06 10:52:01 +00:00
m . setConnRequestLimitersPLocked ( cfg )
2019-11-08 11:32:51 +00:00
}
2023-09-02 14:42:46 +00:00
m . Add ( m . folderRunners )
2019-04-13 12:20:51 +00:00
m . Add ( m . progressEmitter )
2023-08-21 16:39:13 +00:00
m . Add ( m . indexHandlers )
2020-12-22 19:17:14 +00:00
m . Add ( svcutil . AsService ( m . serve , m . String ( ) ) )
2013-12-15 10:43:31 +00:00
return m
}
2020-11-17 12:19:04 +00:00
func ( m * model ) serve ( ctx context . Context ) error {
2020-12-21 11:59:22 +00:00
defer m . closeAllConnectionsAndWait ( )
2021-01-15 14:43:34 +00:00
cfg := m . cfg . Subscribe ( m )
2020-12-21 11:59:22 +00:00
defer m . cfg . Unsubscribe ( m )
2021-01-15 14:43:34 +00:00
if err := m . initFolders ( cfg ) ; err != nil {
2020-12-21 11:59:22 +00:00
close ( m . started )
2020-12-22 19:17:14 +00:00
return svcutil . AsFatalErr ( err , svcutil . ExitError )
2020-12-21 11:59:22 +00:00
}
close ( m . started )
2023-09-06 10:52:01 +00:00
for {
select {
case <- ctx . Done ( ) :
l . Debugln ( m , "context closed, stopping" , ctx . Err ( ) )
return ctx . Err ( )
case err := <- m . fatalChan :
l . Debugln ( m , "fatal error, stopping" , err )
return svcutil . AsFatalErr ( err , svcutil . ExitError )
case <- m . promotionTimer . C :
l . Debugln ( "promotion timer fired" )
m . promoteConnections ( )
}
2020-12-21 11:59:22 +00:00
}
}
2021-01-15 14:43:34 +00:00
func ( m * model ) initFolders ( cfg config . Configuration ) error {
clusterConfigDevices := make ( deviceIDSet , len ( cfg . Devices ) )
for _ , folderCfg := range cfg . Folders {
2019-11-08 09:56:16 +00:00
if folderCfg . Paused {
folderCfg . CreateRoot ( )
continue
}
2021-01-15 14:43:34 +00:00
err := m . newFolder ( folderCfg , cfg . Options . CacheIgnoredFiles )
2020-12-21 11:59:22 +00:00
if err != nil {
return err
}
2020-11-20 14:53:13 +00:00
clusterConfigDevices . add ( folderCfg . DeviceIDs ( ) )
2019-11-08 09:56:16 +00:00
}
2020-12-17 18:54:31 +00:00
ignoredDevices := observedDeviceSet ( m . cfg . IgnoredDevices ( ) )
2021-01-15 14:43:34 +00:00
m . cleanPending ( cfg . DeviceMap ( ) , cfg . FolderMap ( ) , ignoredDevices , nil )
2020-12-17 18:54:31 +00:00
2021-03-22 20:50:19 +00:00
m . sendClusterConfig ( clusterConfigDevices . AsSlice ( ) )
2020-12-21 11:59:22 +00:00
return nil
}
2020-11-17 12:19:04 +00:00
2020-12-21 11:59:22 +00:00
func ( m * model ) closeAllConnectionsAndWait ( ) {
2020-10-02 09:49:51 +00:00
m . pmut . RLock ( )
2023-09-06 10:52:01 +00:00
closed := make ( [ ] chan struct { } , 0 , len ( m . connections ) )
for connID , conn := range m . connections {
closed = append ( closed , m . closed [ connID ] )
2020-10-02 09:49:51 +00:00
go conn . Close ( errStopped )
}
m . pmut . RUnlock ( )
for _ , c := range closed {
<- c
2019-05-25 14:00:32 +00:00
}
2020-12-21 11:59:22 +00:00
}
func ( m * model ) fatal ( err error ) {
select {
case m . fatalChan <- err :
default :
}
2019-05-25 14:00:32 +00:00
}
2015-04-28 20:32:10 +00:00
// StartDeadlockDetector starts a deadlock detector on the models locks which
// causes panics in case the locks cannot be acquired in the given timeout
// period.
2019-02-26 08:09:25 +00:00
func ( m * model ) StartDeadlockDetector ( timeout time . Duration ) {
2015-04-08 12:35:03 +00:00
l . Infof ( "Starting deadlock detector with %v timeout" , timeout )
2021-07-27 19:27:52 +00:00
detector := newDeadlockDetector ( timeout , m . evLogger , m . fatal )
2016-10-29 23:14:38 +00:00
detector . Watch ( "fmut" , m . fmut )
detector . Watch ( "pmut" , m . pmut )
2015-04-08 12:35:03 +00:00
}
2020-05-06 06:34:54 +00:00
// Need to hold lock on m.fmut when calling this.
2020-08-18 07:26:33 +00:00
func ( m * model ) addAndStartFolderLocked ( cfg config . FolderConfiguration , fset * db . FileSet , cacheIgnoredFiles bool ) {
2022-04-10 18:55:05 +00:00
ignores := ignore . New ( cfg . Filesystem ( nil ) , ignore . WithCache ( cacheIgnoredFiles ) )
2021-03-16 14:04:11 +00:00
if cfg . Type != config . FolderTypeReceiveEncrypted {
if err := ignores . Load ( ".stignore" ) ; err != nil && ! fs . IsNotExist ( err ) {
l . Warnln ( "Loading ignores:" , err )
}
2020-05-06 06:34:54 +00:00
}
2019-11-18 20:15:26 +00:00
2020-05-06 06:34:54 +00:00
m . addAndStartFolderLockedWithIgnores ( cfg , fset , ignores )
2016-08-07 16:21:59 +00:00
}
2020-05-06 06:34:54 +00:00
// Only needed for testing, use addAndStartFolderLocked instead.
func ( m * model ) addAndStartFolderLockedWithIgnores ( cfg config . FolderConfiguration , fset * db . FileSet , ignores * ignore . Matcher ) {
m . folderCfgs [ cfg . ID ] = cfg
m . folderFiles [ cfg . ID ] = fset
m . folderIgnores [ cfg . ID ] = ignores
2023-09-02 14:42:46 +00:00
_ , ok := m . folderRunners . Get ( cfg . ID )
2019-07-23 08:51:16 +00:00
if ok {
2019-07-10 08:57:49 +00:00
l . Warnln ( "Cannot start already running folder" , cfg . Description ( ) )
panic ( "cannot start already running folder" )
2014-09-27 12:44:15 +00:00
}
2016-05-04 10:47:33 +00:00
folderFactory , ok := folderFactories [ cfg . Type ]
if ! ok {
2016-05-04 11:26:36 +00:00
panic ( fmt . Sprintf ( "unknown folder type 0x%x" , cfg . Type ) )
2016-05-04 10:47:33 +00:00
}
2019-04-21 12:21:36 +00:00
folder := cfg . ID
2016-08-07 16:21:59 +00:00
// Find any devices for which we hold the index in the db, but the folder
// is not shared, and drop it.
expected := mapDevices ( cfg . DeviceIDs ( ) )
2019-03-11 06:28:54 +00:00
for _ , available := range fset . ListDevices ( ) {
2016-08-07 16:21:59 +00:00
if _ , ok := expected [ available ] ; ! ok {
l . Debugln ( "dropping" , folder , "state for" , available )
2019-03-11 06:28:54 +00:00
fset . Drop ( available )
2016-08-07 16:21:59 +00:00
}
}
2019-03-11 06:28:54 +00:00
v , ok := fset . Sequence ( protocol . LocalDeviceID ) , true
2016-06-26 10:07:27 +00:00
indexHasFiles := ok && v > 0
if ! indexHasFiles {
// It's a blank folder, so this may the first time we're looking at
// it. Attempt to create and tag with our marker as appropriate. We
// don't really do anything with errors at this point except warn -
// if these things don't work, we still want to start the folder and
// it'll show up as errored later.
2018-06-10 13:41:20 +00:00
if err := cfg . CreateRoot ( ) ; err != nil {
l . Warnln ( "Failed to create folder root directory" , err )
} else if err = cfg . CreateMarker ( ) ; err != nil {
l . Warnln ( "Failed to create folder marker:" , err )
2016-06-26 10:07:27 +00:00
}
}
2020-11-09 14:33:32 +00:00
if cfg . Type == config . FolderTypeReceiveEncrypted {
if encryptionToken , err := readEncryptionToken ( cfg ) ; err == nil {
m . folderEncryptionPasswordTokens [ folder ] = encryptionToken
} else if ! fs . IsNotExist ( err ) {
l . Warnf ( "Failed to read encryption token: %v" , err )
}
}
2017-09-20 06:49:04 +00:00
// These are our metadata files, and they should always be hidden.
2022-04-10 18:55:05 +00:00
ffs := cfg . Filesystem ( nil )
2019-11-26 07:39:31 +00:00
_ = ffs . Hide ( config . DefaultMarkerName )
2023-08-09 07:10:06 +00:00
_ = ffs . Hide ( versioner . DefaultPath )
2019-11-26 07:39:31 +00:00
_ = ffs . Hide ( ".stignore" )
var ver versioner . Versioner
if cfg . Versioning . Type != "" {
var err error
2020-06-18 06:15:47 +00:00
ver , err = versioner . New ( cfg )
2019-11-26 07:39:31 +00:00
if err != nil {
2020-03-03 21:40:00 +00:00
panic ( fmt . Errorf ( "creating versioner: %w" , err ) )
2019-11-26 07:39:31 +00:00
}
}
m . folderVersioners [ folder ] = ver
2017-09-20 06:49:04 +00:00
2019-11-18 20:15:26 +00:00
m . warnAboutOverwritingProtectedFiles ( cfg , ignores )
2015-10-19 00:13:58 +00:00
2023-09-02 14:42:46 +00:00
p := folderFactory ( m , fset , ignores , cfg , ver , m . evLogger , m . folderIOLimiter )
m . folderRunners . Add ( folder , p )
2019-11-18 20:15:26 +00:00
l . Infof ( "Ready to synchronize %s (%s)" , cfg . Description ( ) , cfg . Type )
2014-03-28 13:36:57 +00:00
}
2014-01-06 10:11:18 +00:00
2019-11-18 20:15:26 +00:00
func ( m * model ) warnAboutOverwritingProtectedFiles ( cfg config . FolderConfiguration , ignores * ignore . Matcher ) {
if cfg . Type == config . FolderTypeSendOnly {
2015-10-19 00:13:58 +00:00
return
}
2017-08-19 14:36:56 +00:00
// This is a bit of a hack.
2022-04-10 18:55:05 +00:00
ffs := cfg . Filesystem ( nil )
2017-08-19 14:36:56 +00:00
if ffs . Type ( ) != fs . FilesystemTypeBasic {
return
}
folderLocation := ffs . URI ( )
2015-10-19 00:13:58 +00:00
var filesAtRisk [ ] string
for _ , protectedFilePath := range m . protectedFiles {
// check if file is synced in this folder
2018-11-22 10:16:45 +00:00
if protectedFilePath != folderLocation && ! fs . IsParent ( protectedFilePath , folderLocation ) {
2015-10-19 00:13:58 +00:00
continue
}
// check if file is ignored
2017-03-04 07:49:48 +00:00
relPath , _ := filepath . Rel ( folderLocation , protectedFilePath )
if ignores . Match ( relPath ) . IsIgnored ( ) {
2015-10-19 00:13:58 +00:00
continue
}
filesAtRisk = append ( filesAtRisk , protectedFilePath )
}
if len ( filesAtRisk ) > 0 {
2016-10-27 17:02:19 +00:00
l . Warnln ( "Some protected files may be overwritten and cause issues. See https://docs.syncthing.net/users/config.html#syncing-configuration-files for more information. The at risk files are:" , strings . Join ( filesAtRisk , ", " ) )
2015-10-19 00:13:58 +00:00
}
}
2019-11-08 09:56:16 +00:00
func ( m * model ) removeFolder ( cfg config . FolderConfiguration ) {
2020-10-02 09:49:51 +00:00
m . fmut . RLock ( )
2023-09-02 14:42:46 +00:00
wait := m . folderRunners . RemoveAndWaitChan ( cfg . ID , 0 )
2020-10-02 09:49:51 +00:00
m . fmut . RUnlock ( )
2023-09-02 14:42:46 +00:00
<- wait
2019-11-18 20:15:26 +00:00
2020-10-21 09:51:53 +00:00
// We need to hold both fmut and pmut and must acquire locks in the same
// order always. (The locks can be *released* in any order.)
2015-11-13 12:30:52 +00:00
m . fmut . Lock ( )
2020-10-21 09:51:53 +00:00
m . pmut . RLock ( )
2019-04-21 12:21:36 +00:00
2019-10-30 15:11:07 +00:00
isPathUnique := true
for folderID , folderCfg := range m . folderCfgs {
if folderID != cfg . ID && folderCfg . Path == cfg . Path {
isPathUnique = false
break
}
}
if isPathUnique {
2021-04-15 16:13:35 +00:00
// Remove (if empty and removable) or move away (if non-empty or
// otherwise not removable) Syncthing-specific marker files.
2022-04-10 18:55:05 +00:00
fs := cfg . Filesystem ( nil )
2021-04-15 16:13:35 +00:00
if err := fs . Remove ( config . DefaultMarkerName ) ; err != nil {
moved := config . DefaultMarkerName + time . Now ( ) . Format ( ".removed-20060102-150405" )
_ = fs . Rename ( config . DefaultMarkerName , moved )
}
2019-10-30 15:11:07 +00:00
}
2017-01-07 17:05:30 +00:00
2020-05-06 06:34:54 +00:00
m . cleanupFolderLocked ( cfg )
2023-09-02 14:42:46 +00:00
m . indexHandlers . Each ( func ( _ protocol . DeviceID , r * indexHandlerRegistry ) error {
2021-06-03 12:58:50 +00:00
r . Remove ( cfg . ID )
2023-09-02 14:42:46 +00:00
return nil
2023-08-21 16:39:13 +00:00
} )
2019-11-18 20:15:26 +00:00
m . fmut . Unlock ( )
2020-10-21 09:51:53 +00:00
m . pmut . RUnlock ( )
2019-11-18 20:15:26 +00:00
2016-08-07 16:21:59 +00:00
// Remove it from the database
2017-10-03 22:53:02 +00:00
db . DropFolder ( m . db , cfg . ID )
2016-08-07 16:21:59 +00:00
}
2019-11-18 20:15:26 +00:00
// Need to hold lock on m.fmut when calling this.
2020-05-06 06:34:54 +00:00
func ( m * model ) cleanupFolderLocked ( cfg config . FolderConfiguration ) {
2020-06-23 08:47:15 +00:00
// clear up our config maps
2018-06-06 21:34:11 +00:00
delete ( m . folderCfgs , cfg . ID )
delete ( m . folderFiles , cfg . ID )
delete ( m . folderIgnores , cfg . ID )
2019-11-26 07:39:31 +00:00
delete ( m . folderVersioners , cfg . ID )
2021-09-03 07:54:47 +00:00
delete ( m . folderEncryptionPasswordTokens , cfg . ID )
delete ( m . folderEncryptionFailures , cfg . ID )
2016-08-07 16:21:59 +00:00
}
2015-11-13 12:30:52 +00:00
2020-12-21 11:59:22 +00:00
func ( m * model ) restartFolder ( from , to config . FolderConfiguration , cacheIgnoredFiles bool ) error {
2022-07-28 14:51:03 +00:00
if to . ID == "" {
2018-10-05 08:26:25 +00:00
panic ( "bug: cannot restart empty folder ID" )
}
if to . ID != from . ID {
2019-07-10 08:57:49 +00:00
l . Warnf ( "bug: folder restart cannot change ID %q -> %q" , from . ID , to . ID )
panic ( "bug: folder restart cannot change ID" )
2016-08-07 16:21:59 +00:00
}
2020-08-27 16:33:27 +00:00
folder := to . ID
2016-08-07 16:21:59 +00:00
2018-10-05 08:26:25 +00:00
// This mutex protects the entirety of the restart operation, preventing
// there from being more than one folder restart operation in progress
// at any given time. The usual fmut/pmut stuff doesn't cover this,
// because those locks are released while we are waiting for the folder
// to shut down (and must be so because the folder might need them as
// part of its operations before shutting down).
2020-08-27 16:33:27 +00:00
restartMut := m . folderRestartMuts . Get ( folder )
2018-10-05 08:26:25 +00:00
restartMut . Lock ( )
defer restartMut . Unlock ( )
2020-10-02 09:49:51 +00:00
m . fmut . RLock ( )
2023-09-02 14:42:46 +00:00
wait := m . folderRunners . RemoveAndWaitChan ( from . ID , 0 )
2020-10-02 09:49:51 +00:00
m . fmut . RUnlock ( )
2023-09-02 14:42:46 +00:00
<- wait
2019-01-09 16:31:09 +00:00
2019-04-28 10:58:51 +00:00
m . fmut . Lock ( )
defer m . fmut . Unlock ( )
2020-08-27 16:33:27 +00:00
// Cache the (maybe) existing fset before it's removed by cleanupFolderLocked
fset := m . folderFiles [ folder ]
2020-11-09 07:58:46 +00:00
fsetNil := fset == nil
2020-08-27 16:33:27 +00:00
2020-05-06 06:34:54 +00:00
m . cleanupFolderLocked ( from )
2020-11-09 07:58:46 +00:00
if ! to . Paused {
2020-10-21 09:51:53 +00:00
if fsetNil {
2020-08-27 16:33:27 +00:00
// Create a new fset. Might take a while and we do it under
// locking, but it's unsafe to create fset:s concurrently so
// that's the price we pay.
2020-12-21 11:59:22 +00:00
var err error
2022-01-31 09:12:52 +00:00
fset , err = db . NewFileSet ( folder , m . db )
2020-12-21 11:59:22 +00:00
if err != nil {
return fmt . Errorf ( "restarting %v: %w" , to . Description ( ) , err )
}
2020-08-27 16:33:27 +00:00
}
2020-08-18 07:26:33 +00:00
m . addAndStartFolderLocked ( to , fset , cacheIgnoredFiles )
2020-11-09 07:58:46 +00:00
}
// Care needs to be taken because we already hold fmut and the lock order
// must be the same everywhere. As fmut is acquired first, this is fine.
m . pmut . RLock ( )
2023-09-02 14:42:46 +00:00
runner , _ := m . folderRunners . Get ( to . ID )
m . indexHandlers . Each ( func ( _ protocol . DeviceID , r * indexHandlerRegistry ) error {
r . RegisterFolderState ( to , fset , runner )
return nil
2023-08-21 16:39:13 +00:00
} )
2020-11-09 07:58:46 +00:00
m . pmut . RUnlock ( )
2020-10-02 09:49:51 +00:00
var infoMsg string
switch {
case to . Paused :
infoMsg = "Paused"
case from . Paused :
infoMsg = "Unpaused"
default :
infoMsg = "Restarted"
}
2019-01-09 16:31:09 +00:00
l . Infof ( "%v folder %v (%v)" , infoMsg , to . Description ( ) , to . Type )
2020-12-21 11:59:22 +00:00
return nil
2015-11-13 12:30:52 +00:00
}
2020-12-21 11:59:22 +00:00
func ( m * model ) newFolder ( cfg config . FolderConfiguration , cacheIgnoredFiles bool ) error {
2019-11-18 20:15:26 +00:00
// Creating the fileset can take a long time (metadata calculation) so
// we do it outside of the lock.
2022-01-31 09:12:52 +00:00
fset , err := db . NewFileSet ( cfg . ID , m . db )
2020-12-21 11:59:22 +00:00
if err != nil {
return fmt . Errorf ( "adding %v: %w" , cfg . Description ( ) , err )
}
2019-11-18 20:15:26 +00:00
m . fmut . Lock ( )
defer m . fmut . Unlock ( )
2020-11-09 07:58:46 +00:00
2021-06-03 12:58:50 +00:00
m . addAndStartFolderLocked ( cfg , fset , cacheIgnoredFiles )
2020-11-17 14:30:21 +00:00
// Cluster configs might be received and processed before reaching this
// point, i.e. before the folder is started. If that's the case, start
// index senders here.
2021-06-03 12:58:50 +00:00
// Care needs to be taken because we already hold fmut and the lock order
// must be the same everywhere. As fmut is acquired first, this is fine.
2020-11-17 14:30:21 +00:00
m . pmut . RLock ( )
2023-09-02 14:42:46 +00:00
m . indexHandlers . Each ( func ( _ protocol . DeviceID , r * indexHandlerRegistry ) error {
runner , _ := m . folderRunners . Get ( cfg . ID )
r . RegisterFolderState ( cfg , fset , runner )
return nil
2023-08-21 16:39:13 +00:00
} )
2020-11-17 14:30:21 +00:00
m . pmut . RUnlock ( )
2020-11-09 07:58:46 +00:00
2020-12-21 11:59:22 +00:00
return nil
2019-11-18 20:15:26 +00:00
}
2020-06-23 08:47:15 +00:00
func ( m * model ) UsageReportingStats ( report * contract . Report , version int , preview bool ) {
2017-10-12 06:16:46 +00:00
if version >= 3 {
// Block stats
2017-11-09 21:16:29 +00:00
blockStatsMut . Lock ( )
for k , v := range blockStats {
2020-06-23 08:47:15 +00:00
switch k {
case "total" :
report . BlockStats . Total = v
case "renamed" :
report . BlockStats . Renamed = v
case "reused" :
report . BlockStats . Reused = v
case "pulled" :
report . BlockStats . Pulled = v
case "copyOrigin" :
report . BlockStats . CopyOrigin = v
case "copyOriginShifted" :
report . BlockStats . CopyOriginShifted = v
case "copyElsewhere" :
report . BlockStats . CopyElsewhere = v
}
// Reset counts, as these are incremental
2017-11-09 21:16:29 +00:00
if ! preview {
blockStats [ k ] = 0
2017-10-12 06:16:46 +00:00
}
}
2017-11-09 21:16:29 +00:00
blockStatsMut . Unlock ( )
2017-10-12 06:16:46 +00:00
// Transport stats
2019-04-21 12:21:36 +00:00
m . pmut . RLock ( )
2023-09-06 10:52:01 +00:00
for _ , conn := range m . connections {
2020-06-23 08:47:15 +00:00
report . TransportStats [ conn . Transport ( ) ] ++
2017-10-12 06:16:46 +00:00
}
2019-04-21 12:21:36 +00:00
m . pmut . RUnlock ( )
2017-10-12 06:16:46 +00:00
// Ignore stats
var seenPrefix [ 3 ] bool
for folder := range m . cfg . Folders ( ) {
2021-01-12 15:25:21 +00:00
lines , _ , err := m . CurrentIgnores ( folder )
2017-10-12 06:16:46 +00:00
if err != nil {
continue
}
2020-06-23 08:47:15 +00:00
report . IgnoreStats . Lines += len ( lines )
2017-10-12 06:16:46 +00:00
for _ , line := range lines {
// Allow prefixes to be specified in any order, but only once.
for {
if strings . HasPrefix ( line , "!" ) && ! seenPrefix [ 0 ] {
seenPrefix [ 0 ] = true
line = line [ 1 : ]
2020-06-23 08:47:15 +00:00
report . IgnoreStats . Inverts ++
2017-10-12 06:16:46 +00:00
} else if strings . HasPrefix ( line , "(?i)" ) && ! seenPrefix [ 1 ] {
seenPrefix [ 1 ] = true
line = line [ 4 : ]
2020-06-23 08:47:15 +00:00
report . IgnoreStats . Folded ++
2017-10-12 06:16:46 +00:00
} else if strings . HasPrefix ( line , "(?d)" ) && ! seenPrefix [ 2 ] {
seenPrefix [ 2 ] = true
line = line [ 4 : ]
2020-06-23 08:47:15 +00:00
report . IgnoreStats . Deletable ++
2017-10-12 06:16:46 +00:00
} else {
seenPrefix [ 0 ] = false
seenPrefix [ 1 ] = false
seenPrefix [ 2 ] = false
break
}
}
// Noops, remove
2019-02-02 11:09:07 +00:00
line = strings . TrimSuffix ( line , "**" )
line = strings . TrimPrefix ( line , "**/" )
2017-10-12 06:16:46 +00:00
if strings . HasPrefix ( line , "/" ) {
2020-06-23 08:47:15 +00:00
report . IgnoreStats . Rooted ++
2017-10-12 06:16:46 +00:00
} else if strings . HasPrefix ( line , "#include " ) {
2020-06-23 08:47:15 +00:00
report . IgnoreStats . Includes ++
2017-10-12 06:16:46 +00:00
if strings . Contains ( line , ".." ) {
2020-06-23 08:47:15 +00:00
report . IgnoreStats . EscapedIncludes ++
2017-10-12 06:16:46 +00:00
}
}
if strings . Contains ( line , "**" ) {
2020-06-23 08:47:15 +00:00
report . IgnoreStats . DoubleStars ++
2017-10-12 06:16:46 +00:00
// Remove not to trip up star checks.
2021-03-17 22:12:26 +00:00
line = strings . ReplaceAll ( line , "**" , "" )
2017-10-12 06:16:46 +00:00
}
if strings . Contains ( line , "*" ) {
2020-06-23 08:47:15 +00:00
report . IgnoreStats . Stars ++
2017-10-12 06:16:46 +00:00
}
}
}
}
}
2023-09-06 10:52:01 +00:00
type ConnectionStats struct {
protocol . Statistics // Total for primary + secondaries
2022-01-10 09:26:45 +00:00
Connected bool ` json:"connected" `
Paused bool ` json:"paused" `
ClientVersion string ` json:"clientVersion" `
2023-09-06 10:52:01 +00:00
Address string ` json:"address" ` // mirror values from Primary, for compatibility with <1.24.0
Type string ` json:"type" ` // mirror values from Primary, for compatibility with <1.24.0
IsLocal bool ` json:"isLocal" ` // mirror values from Primary, for compatibility with <1.24.0
Crypto string ` json:"crypto" ` // mirror values from Primary, for compatibility with <1.24.0
Primary ConnectionInfo ` json:"primary,omitempty" `
Secondary [ ] ConnectionInfo ` json:"secondary,omitempty" `
2015-03-10 22:45:43 +00:00
}
2023-09-06 10:52:01 +00:00
type ConnectionInfo struct {
protocol . Statistics
Address string ` json:"address" `
Type string ` json:"type" `
IsLocal bool ` json:"isLocal" `
Crypto string ` json:"crypto" `
2021-01-11 14:14:44 +00:00
}
2015-11-09 22:48:58 +00:00
// ConnectionStats returns a map with connection statistics for each device.
2019-02-26 08:09:25 +00:00
func ( m * model ) ConnectionStats ( ) map [ string ] interface { } {
2016-09-14 19:38:55 +00:00
m . pmut . RLock ( )
2019-05-02 17:55:39 +00:00
defer m . pmut . RUnlock ( )
2014-01-05 15:16:37 +00:00
2015-08-23 19:56:10 +00:00
res := make ( map [ string ] interface { } )
devs := m . cfg . Devices ( )
2023-09-06 10:52:01 +00:00
conns := make ( map [ string ] ConnectionStats , len ( devs ) )
2016-12-21 18:41:25 +00:00
for device , deviceCfg := range devs {
2023-09-06 10:52:01 +00:00
if device == m . id {
continue
}
2016-03-25 20:29:07 +00:00
hello := m . helloMessages [ device ]
versionString := hello . ClientVersion
if hello . ClientName != "syncthing" {
versionString = hello . ClientName + " " + hello . ClientVersion
}
2023-09-06 10:52:01 +00:00
connIDs , ok := m . deviceConnIDs [ device ]
cs := ConnectionStats {
Connected : ok ,
2016-12-21 18:41:25 +00:00
Paused : deviceCfg . Paused ,
2023-09-06 10:52:01 +00:00
ClientVersion : strings . TrimSpace ( versionString ) ,
2014-01-05 22:54:57 +00:00
}
2023-09-06 10:52:01 +00:00
if ok {
conn := m . connections [ connIDs [ 0 ] ]
cs . Primary . Type = conn . Type ( )
cs . Primary . IsLocal = conn . IsLocal ( )
cs . Primary . Crypto = conn . Crypto ( )
cs . Primary . Statistics = conn . Statistics ( )
cs . Primary . Address = conn . RemoteAddr ( ) . String ( )
cs . Type = cs . Primary . Type
cs . IsLocal = cs . Primary . IsLocal
cs . Crypto = cs . Primary . Crypto
cs . Address = cs . Primary . Address
cs . Statistics = cs . Primary . Statistics
for _ , connID := range connIDs [ 1 : ] {
conn = m . connections [ connID ]
sec := ConnectionInfo {
Statistics : conn . Statistics ( ) ,
Address : conn . RemoteAddr ( ) . String ( ) ,
Type : conn . Type ( ) ,
IsLocal : conn . IsLocal ( ) ,
Crypto : conn . Crypto ( ) ,
}
if sec . At . After ( cs . At ) {
cs . At = sec . At
}
if sec . StartedAt . Before ( cs . StartedAt ) {
cs . StartedAt = sec . StartedAt
}
cs . InBytesTotal += sec . InBytesTotal
cs . OutBytesTotal += sec . OutBytesTotal
cs . Secondary = append ( cs . Secondary , sec )
2015-08-23 19:56:10 +00:00
}
2014-01-05 22:54:57 +00:00
}
2014-02-13 11:41:37 +00:00
2023-09-06 10:52:01 +00:00
conns [ device . String ( ) ] = cs
2013-12-30 14:30:29 +00:00
}
2014-01-18 03:06:44 +00:00
2015-04-07 12:20:40 +00:00
res [ "connections" ] = conns
2014-05-24 19:34:11 +00:00
in , out := protocol . TotalInOut ( )
2022-01-10 09:26:45 +00:00
res [ "total" ] = map [ string ] interface { } {
"at" : time . Now ( ) . Truncate ( time . Second ) ,
"inBytesTotal" : in ,
"outBytesTotal" : out ,
2014-05-24 19:34:11 +00:00
}
2014-01-05 15:16:37 +00:00
return res
2013-12-30 14:30:29 +00:00
}
2015-04-28 20:32:10 +00:00
// DeviceStatistics returns statistics about each device
2021-01-11 14:14:44 +00:00
func ( m * model ) DeviceStatistics ( ) ( map [ protocol . DeviceID ] stats . DeviceStatistics , error ) {
2019-09-12 04:55:24 +00:00
m . fmut . RLock ( )
defer m . fmut . RUnlock ( )
2021-01-11 14:14:44 +00:00
res := make ( map [ protocol . DeviceID ] stats . DeviceStatistics , len ( m . deviceStatRefs ) )
2019-09-12 04:55:24 +00:00
for id , sr := range m . deviceStatRefs {
2019-11-30 12:03:24 +00:00
stats , err := sr . GetStatistics ( )
if err != nil {
return nil , err
}
2021-01-11 14:14:44 +00:00
res [ id ] = stats
2014-08-21 22:45:40 +00:00
}
2019-11-30 12:03:24 +00:00
return res , nil
2014-08-21 22:45:40 +00:00
}
2015-04-28 20:32:10 +00:00
// FolderStatistics returns statistics about each folder
2019-11-30 12:03:24 +00:00
func ( m * model ) FolderStatistics ( ) ( map [ string ] stats . FolderStatistics , error ) {
2016-12-06 08:54:04 +00:00
res := make ( map [ string ] stats . FolderStatistics )
2019-03-11 16:57:21 +00:00
m . fmut . RLock ( )
defer m . fmut . RUnlock ( )
2023-09-02 14:42:46 +00:00
err := m . folderRunners . Each ( func ( id string , runner service ) error {
2019-11-30 12:03:24 +00:00
stats , err := runner . GetStatistics ( )
if err != nil {
2023-09-02 14:42:46 +00:00
return err
2019-11-30 12:03:24 +00:00
}
res [ id ] = stats
2023-09-02 14:42:46 +00:00
return nil
} )
if err != nil {
return nil , err
2014-12-07 20:21:12 +00:00
}
2019-11-30 12:03:24 +00:00
return res , nil
2014-12-07 20:21:12 +00:00
}
2016-08-12 06:41:43 +00:00
type FolderCompletion struct {
CompletionPct float64
GlobalBytes int64
2020-07-03 06:48:37 +00:00
NeedBytes int64
2020-10-02 06:07:05 +00:00
GlobalItems int
NeedItems int
NeedDeletes int
2020-09-07 07:35:37 +00:00
Sequence int64
2022-04-22 06:42:20 +00:00
RemoteState remoteFolderState
2016-08-12 06:41:43 +00:00
}
2022-04-22 06:42:20 +00:00
func newFolderCompletion ( global , need db . Counts , sequence int64 , state remoteFolderState ) FolderCompletion {
2020-07-03 06:48:37 +00:00
comp := FolderCompletion {
GlobalBytes : global . Bytes ,
NeedBytes : need . Bytes ,
GlobalItems : global . Files + global . Directories + global . Symlinks ,
NeedItems : need . Files + need . Directories + need . Symlinks ,
NeedDeletes : need . Deleted ,
2020-09-07 07:35:37 +00:00
Sequence : sequence ,
2022-04-22 06:42:20 +00:00
RemoteState : state ,
2020-07-03 06:48:37 +00:00
}
comp . setComplectionPct ( )
return comp
}
func ( comp * FolderCompletion ) add ( other FolderCompletion ) {
comp . GlobalBytes += other . GlobalBytes
comp . NeedBytes += other . NeedBytes
comp . GlobalItems += other . GlobalItems
comp . NeedItems += other . NeedItems
comp . NeedDeletes += other . NeedDeletes
comp . setComplectionPct ( )
}
func ( comp * FolderCompletion ) setComplectionPct ( ) {
if comp . GlobalBytes == 0 {
comp . CompletionPct = 100
} else {
needRatio := float64 ( comp . NeedBytes ) / float64 ( comp . GlobalBytes )
comp . CompletionPct = 100 * ( 1 - needRatio )
}
// If the completion is 100% but there are deletes we need to handle,
// drop it down a notch. Hack for consumers that look only at the
// percentage (our own GUI does the same calculation as here on its own
// and needs the same fixup).
if comp . NeedBytes == 0 && comp . NeedDeletes > 0 {
comp . CompletionPct = 95 // chosen by fair dice roll
}
}
2022-04-22 06:42:20 +00:00
// Map returns the members as a map, e.g. used in api to serialize as JSON.
2023-07-29 08:24:44 +00:00
func ( comp * FolderCompletion ) Map ( ) map [ string ] interface { } {
2019-03-26 19:53:58 +00:00
return map [ string ] interface { } {
"completion" : comp . CompletionPct ,
2020-07-03 06:48:37 +00:00
"globalBytes" : comp . GlobalBytes ,
2019-03-26 19:53:58 +00:00
"needBytes" : comp . NeedBytes ,
2020-07-03 06:48:37 +00:00
"globalItems" : comp . GlobalItems ,
2019-03-26 19:53:58 +00:00
"needItems" : comp . NeedItems ,
"needDeletes" : comp . NeedDeletes ,
2020-09-07 07:35:37 +00:00
"sequence" : comp . Sequence ,
2022-04-22 06:42:20 +00:00
"remoteState" : comp . RemoteState ,
2019-03-26 19:53:58 +00:00
}
}
2020-07-03 06:48:37 +00:00
// Completion returns the completion status, in percent with some counters,
// for the given device and folder. The device can be any known device ID
// (including the local device) or explicitly protocol.LocalDeviceID. An
// empty folder string means the aggregate of all folders shared with the
// given device.
2021-03-07 12:43:22 +00:00
func ( m * model ) Completion ( device protocol . DeviceID , folder string ) ( FolderCompletion , error ) {
2020-07-03 06:48:37 +00:00
// The user specifically asked for our own device ID. Internally that is
// known as protocol.LocalDeviceID so translate.
if device == m . id {
device = protocol . LocalDeviceID
}
if folder != "" {
// We want completion for a specific folder.
return m . folderCompletion ( device , folder )
}
// We want completion for all (shared) folders as an aggregate.
var comp FolderCompletion
for _ , fcfg := range m . cfg . FolderList ( ) {
2022-08-10 06:50:19 +00:00
if fcfg . Paused {
continue
}
2020-07-03 06:48:37 +00:00
if device == protocol . LocalDeviceID || fcfg . SharedWith ( device ) {
2021-03-07 12:43:22 +00:00
folderComp , err := m . folderCompletion ( device , fcfg . ID )
2022-08-10 06:50:19 +00:00
if errors . Is ( err , ErrFolderPaused ) {
continue
} else if err != nil {
2021-03-07 12:43:22 +00:00
return FolderCompletion { } , err
}
comp . add ( folderComp )
2020-07-03 06:48:37 +00:00
}
}
2021-03-07 12:43:22 +00:00
return comp , nil
2020-07-03 06:48:37 +00:00
}
2021-03-07 12:43:22 +00:00
func ( m * model ) folderCompletion ( device protocol . DeviceID , folder string ) ( FolderCompletion , error ) {
2014-09-28 11:39:39 +00:00
m . fmut . RLock ( )
2021-03-07 12:43:22 +00:00
err := m . checkFolderRunningLocked ( folder )
rf := m . folderFiles [ folder ]
2014-09-28 11:39:39 +00:00
m . fmut . RUnlock ( )
2021-03-07 12:43:22 +00:00
if err != nil {
return FolderCompletion { } , err
2014-08-05 18:16:25 +00:00
}
2021-03-07 12:43:22 +00:00
snap , err := rf . Snapshot ( )
if err != nil {
return FolderCompletion { } , err
}
2020-01-21 17:23:08 +00:00
defer snap . Release ( )
2016-05-26 06:53:27 +00:00
m . pmut . RLock ( )
2022-04-22 06:42:20 +00:00
state := m . remoteFolderStates [ device ] [ folder ]
2020-05-11 13:07:06 +00:00
downloaded := m . deviceDownloads [ device ] . BytesDownloaded ( folder )
2016-05-26 06:53:27 +00:00
m . pmut . RUnlock ( )
2020-05-11 13:07:06 +00:00
need := snap . NeedSize ( device )
need . Bytes -= downloaded
// This might might be more than it really is, because some blocks can be of a smaller size.
if need . Bytes < 0 {
need . Bytes = 0
}
2014-07-29 09:06:52 +00:00
2022-04-22 06:42:20 +00:00
comp := newFolderCompletion ( snap . GlobalSize ( ) , need , snap . Sequence ( device ) , state )
2016-09-02 06:45:46 +00:00
2020-07-03 06:48:37 +00:00
l . Debugf ( "%v Completion(%s, %q): %v" , m , device , folder , comp . Map ( ) )
2021-03-07 12:43:22 +00:00
return comp , nil
2014-07-29 09:06:52 +00:00
}
2020-01-21 17:23:08 +00:00
// DBSnapshot returns a snapshot of the database content relevant to the given folder.
func ( m * model ) DBSnapshot ( folder string ) ( * db . Snapshot , error ) {
2014-09-28 11:39:39 +00:00
m . fmut . RLock ( )
2020-02-12 06:35:24 +00:00
err := m . checkFolderRunningLocked ( folder )
rf := m . folderFiles [ folder ]
2019-08-13 07:04:43 +00:00
m . fmut . RUnlock ( )
2020-02-12 06:35:24 +00:00
if err != nil {
return nil , err
2014-03-29 17:53:48 +00:00
}
2021-03-07 12:43:22 +00:00
return rf . Snapshot ( )
2014-01-06 05:38:01 +00:00
}
2020-01-21 17:23:08 +00:00
func ( m * model ) FolderProgressBytesCompleted ( folder string ) int64 {
return m . progressEmitter . BytesCompleted ( folder )
2013-12-23 17:12:44 +00:00
}
2015-04-28 20:32:10 +00:00
// NeedFolderFiles returns paginated list of currently needed files in
2019-06-27 18:25:38 +00:00
// progress, queued, and to be queued on next puller iteration.
2020-11-27 10:26:36 +00:00
func ( m * model ) NeedFolderFiles ( folder string , page , perpage int ) ( [ ] db . FileInfoTruncated , [ ] db . FileInfoTruncated , [ ] db . FileInfoTruncated , error ) {
2014-09-28 11:39:39 +00:00
m . fmut . RLock ( )
2019-05-02 17:55:39 +00:00
rf , rfOk := m . folderFiles [ folder ]
2023-09-02 14:42:46 +00:00
runner , runnerOk := m . folderRunners . Get ( folder )
2019-05-02 17:55:39 +00:00
cfg := m . folderCfgs [ folder ]
m . fmut . RUnlock ( )
2015-01-17 20:51:46 +00:00
2019-05-02 17:55:39 +00:00
if ! rfOk {
2021-03-07 12:43:22 +00:00
return nil , nil , nil , ErrFolderMissing
2015-04-25 21:53:44 +00:00
}
2014-12-30 08:31:34 +00:00
2021-03-07 12:43:22 +00:00
snap , err := rf . Snapshot ( )
if err != nil {
return nil , nil , nil , err
}
2020-01-21 17:23:08 +00:00
defer snap . Release ( )
2015-04-25 21:53:44 +00:00
var progress , queued , rest [ ] db . FileInfoTruncated
var seen map [ string ] struct { }
2014-12-30 08:31:34 +00:00
2020-11-27 10:26:36 +00:00
p := newPager ( page , perpage )
2014-12-30 08:31:34 +00:00
2019-05-02 17:55:39 +00:00
if runnerOk {
2019-06-27 18:25:38 +00:00
progressNames , queuedNames , skipped := runner . Jobs ( page , perpage )
2015-04-25 21:53:44 +00:00
2019-07-19 17:39:52 +00:00
progress = make ( [ ] db . FileInfoTruncated , len ( progressNames ) )
queued = make ( [ ] db . FileInfoTruncated , len ( queuedNames ) )
seen = make ( map [ string ] struct { } , len ( progressNames ) + len ( queuedNames ) )
2015-04-25 21:53:44 +00:00
for i , name := range progressNames {
2020-01-21 17:23:08 +00:00
if f , ok := snap . GetGlobalTruncated ( name ) ; ok {
2015-04-25 21:53:44 +00:00
progress [ i ] = f
seen [ name ] = struct { } { }
Add job queue (fixes #629)
Request to terminate currently ongoing downloads and jump to the bumped file
incoming in 3, 2, 1.
Also, has a slightly strange effect where we pop a job off the queue, but
the copyChannel is still busy and blocks, though it gets moved to the
progress slice in the jobqueue, and looks like it's in progress which it isn't
as it's waiting to be picked up from the copyChan.
As a result, the progress emitter doesn't register on the task, and hence the file
doesn't have a progress bar, but cannot be replaced by a bump.
I guess I can fix progress bar issue by moving the progressEmiter.Register just
before passing the file to the copyChan, but then we are back to the initial
problem of a file with a progress bar, but no progress happening as it's stuck
on write to copyChan
I checked if there is a way to check for channel writeability (before popping)
but got struck by lightning just for bringing the idea up in #go-nuts.
My ideal scenario would be to check if copyChan is writeable, pop job from the
queue and shove it down handleFile. This way jobs would stay in the queue while
they cannot be handled, meaning that the `Bump` could bring your file up higher.
2014-12-01 19:23:06 +00:00
}
}
2015-04-25 21:53:44 +00:00
for i , name := range queuedNames {
2020-01-21 17:23:08 +00:00
if f , ok := snap . GetGlobalTruncated ( name ) ; ok {
2015-04-25 21:53:44 +00:00
queued [ i ] = f
seen [ name ] = struct { } { }
}
Add job queue (fixes #629)
Request to terminate currently ongoing downloads and jump to the bumped file
incoming in 3, 2, 1.
Also, has a slightly strange effect where we pop a job off the queue, but
the copyChannel is still busy and blocks, though it gets moved to the
progress slice in the jobqueue, and looks like it's in progress which it isn't
as it's waiting to be picked up from the copyChan.
As a result, the progress emitter doesn't register on the task, and hence the file
doesn't have a progress bar, but cannot be replaced by a bump.
I guess I can fix progress bar issue by moving the progressEmiter.Register just
before passing the file to the copyChan, but then we are back to the initial
problem of a file with a progress bar, but no progress happening as it's stuck
on write to copyChan
I checked if there is a way to check for channel writeability (before popping)
but got struck by lightning just for bringing the idea up in #go-nuts.
My ideal scenario would be to check if copyChan is writeable, pop job from the
queue and shove it down handleFile. This way jobs would stay in the queue while
they cannot be handled, meaning that the `Bump` could bring your file up higher.
2014-12-01 19:23:06 +00:00
}
2019-06-27 18:25:38 +00:00
2020-11-27 10:26:36 +00:00
p . get -= len ( seen )
if p . get == 0 {
return progress , queued , nil , nil
2019-06-27 18:25:38 +00:00
}
2020-11-27 10:26:36 +00:00
p . toSkip -= skipped
2014-04-09 20:03:30 +00:00
}
2015-04-25 21:53:44 +00:00
rest = make ( [ ] db . FileInfoTruncated , 0 , perpage )
2020-05-30 07:50:23 +00:00
snap . WithNeedTruncated ( protocol . LocalDeviceID , func ( f protocol . FileIntf ) bool {
2017-11-11 19:18:17 +00:00
if cfg . IgnoreDelete && f . IsDeleted ( ) {
2016-08-05 07:13:52 +00:00
return true
}
2020-11-27 10:26:36 +00:00
if p . skip ( ) {
2015-04-25 21:53:44 +00:00
return true
}
2018-12-11 08:59:04 +00:00
ft := f . ( db . FileInfoTruncated )
if _ , ok := seen [ ft . Name ] ; ! ok {
rest = append ( rest , ft )
2020-11-27 10:26:36 +00:00
p . get --
2015-04-25 21:53:44 +00:00
}
2020-11-27 10:26:36 +00:00
return p . get > 0
2015-04-25 21:53:44 +00:00
} )
2020-11-27 10:26:36 +00:00
return progress , queued , rest , nil
}
2022-02-07 07:51:09 +00:00
// RemoteNeedFolderFiles returns paginated list of currently needed files for a
// remote device to become synced with a folder.
2020-11-27 10:26:36 +00:00
func ( m * model ) RemoteNeedFolderFiles ( folder string , device protocol . DeviceID , page , perpage int ) ( [ ] db . FileInfoTruncated , error ) {
m . fmut . RLock ( )
rf , ok := m . folderFiles [ folder ]
m . fmut . RUnlock ( )
if ! ok {
2021-03-07 12:43:22 +00:00
return nil , ErrFolderMissing
2020-11-27 10:26:36 +00:00
}
2021-03-07 12:43:22 +00:00
snap , err := rf . Snapshot ( )
if err != nil {
return nil , err
}
2020-11-27 10:26:36 +00:00
defer snap . Release ( )
files := make ( [ ] db . FileInfoTruncated , 0 , perpage )
p := newPager ( page , perpage )
snap . WithNeedTruncated ( device , func ( f protocol . FileIntf ) bool {
if p . skip ( ) {
return true
}
files = append ( files , f . ( db . FileInfoTruncated ) )
return ! p . done ( )
} )
return files , nil
}
func ( m * model ) LocalChangedFolderFiles ( folder string , page , perpage int ) ( [ ] db . FileInfoTruncated , error ) {
m . fmut . RLock ( )
rf , ok := m . folderFiles [ folder ]
m . fmut . RUnlock ( )
if ! ok {
2021-03-07 12:43:22 +00:00
return nil , ErrFolderMissing
2020-11-27 10:26:36 +00:00
}
2021-03-07 12:43:22 +00:00
snap , err := rf . Snapshot ( )
if err != nil {
return nil , err
}
2020-11-27 10:26:36 +00:00
defer snap . Release ( )
if snap . ReceiveOnlyChangedSize ( ) . TotalItems ( ) == 0 {
return nil , nil
}
p := newPager ( page , perpage )
files := make ( [ ] db . FileInfoTruncated , 0 , perpage )
snap . WithHaveTruncated ( protocol . LocalDeviceID , func ( f protocol . FileIntf ) bool {
2021-11-10 08:46:21 +00:00
if ! f . IsReceiveOnlyChanged ( ) {
2020-11-27 10:26:36 +00:00
return true
}
if p . skip ( ) {
return true
}
ft := f . ( db . FileInfoTruncated )
files = append ( files , ft )
return ! p . done ( )
} )
return files , nil
}
type pager struct {
toSkip , get int
}
func newPager ( page , perpage int ) * pager {
return & pager {
toSkip : ( page - 1 ) * perpage ,
get : perpage ,
}
}
func ( p * pager ) skip ( ) bool {
if p . toSkip == 0 {
return false
}
p . toSkip --
return true
}
func ( p * pager ) done ( ) bool {
if p . get > 0 {
p . get --
}
return p . get == 0
2017-12-15 20:01:56 +00:00
}
2014-09-28 11:00:38 +00:00
// Index is called when a new device is connected and we receive their full index.
2014-01-06 10:11:18 +00:00
// Implements the protocol.Model interface.
2023-07-29 08:24:44 +00:00
func ( m * model ) Index ( conn protocol . Connection , folder string , fs [ ] protocol . FileInfo ) error {
return m . handleIndex ( conn , folder , fs , false )
2013-12-28 13:10:36 +00:00
}
2014-09-28 11:00:38 +00:00
// IndexUpdate is called for incremental updates to connected devices' indexes.
2014-01-06 10:11:18 +00:00
// Implements the protocol.Model interface.
2023-07-29 08:24:44 +00:00
func ( m * model ) IndexUpdate ( conn protocol . Connection , folder string , fs [ ] protocol . FileInfo ) error {
return m . handleIndex ( conn , folder , fs , true )
2018-01-19 14:33:16 +00:00
}
2023-07-29 08:24:44 +00:00
func ( m * model ) handleIndex ( conn protocol . Connection , folder string , fs [ ] protocol . FileInfo , update bool ) error {
2018-01-19 14:33:16 +00:00
op := "Index"
if update {
op += " update"
}
2023-07-29 08:24:44 +00:00
deviceID := conn . DeviceID ( )
2018-01-19 14:33:16 +00:00
l . Debugf ( "%v (in): %s / %q: %d files" , op , deviceID , folder , len ( fs ) )
2014-03-29 17:53:48 +00:00
2018-06-06 21:34:11 +00:00
if cfg , ok := m . cfg . Folder ( folder ) ; ! ok || ! cfg . SharedWith ( deviceID ) {
2023-07-22 21:17:32 +00:00
l . Warnf ( "%v for unexpected folder ID %q sent from device %q; ensure that the folder exists and that this device is selected under \"Share With\" in the folder configuration." , op , folder , deviceID )
2022-08-16 08:01:49 +00:00
return fmt . Errorf ( "%s: %w" , folder , ErrFolderMissing )
2018-06-06 21:34:11 +00:00
} else if cfg . Paused {
l . Debugf ( "%v for paused folder (ID %q) sent from device %q." , op , folder , deviceID )
2022-08-16 08:01:49 +00:00
return fmt . Errorf ( "%s: %w" , folder , ErrFolderPaused )
2014-06-06 19:48:29 +00:00
}
2016-05-01 06:49:29 +00:00
m . pmut . RLock ( )
2023-09-06 10:52:01 +00:00
indexHandler , ok := m . getIndexHandlerPRLocked ( conn )
2016-05-01 06:49:29 +00:00
m . pmut . RUnlock ( )
2021-06-03 12:58:50 +00:00
if ! ok {
2023-09-06 10:52:01 +00:00
// This should be impossible, as an index handler is registered when
// we send a cluster config, and that is what triggers index
// sending.
2021-06-03 12:58:50 +00:00
m . evLogger . Log ( events . Failure , "index sender does not exist for connection on which indexes were received" )
l . Debugf ( "%v for folder (ID %q) sent from device %q: missing index handler" , op , folder , deviceID )
2023-09-06 10:52:01 +00:00
return fmt . Errorf ( "%s: %w" , folder , ErrFolderNotRunning )
2021-06-03 12:58:50 +00:00
}
return indexHandler . ReceiveIndex ( folder , fs , update , op )
}
type clusterConfigDeviceInfo struct {
local , remote protocol . Device
2014-01-09 09:59:09 +00:00
}
2022-04-22 06:42:20 +00:00
type ClusterConfigReceivedEventData struct {
Device protocol . DeviceID ` json:"device" `
}
2023-07-29 08:24:44 +00:00
func ( m * model ) ClusterConfig ( conn protocol . Connection , cm protocol . ClusterConfig ) error {
2023-09-06 10:52:01 +00:00
deviceID := conn . DeviceID ( )
if cm . Secondary {
// No handling of secondary connection ClusterConfigs; they merely
// indicate the connection is ready to start.
l . Debugf ( "Skipping secondary ClusterConfig from %v at %s" , deviceID . Short ( ) , conn )
return nil
}
2016-01-01 19:11:12 +00:00
// Check the peer device's announced folders against our own. Emits events
// for folders that we don't expect (unknown or not shared).
2016-04-15 10:59:41 +00:00
// Also, collect a list of folders we do share, and if he's interested in
// temporary indexes, subscribe the connection.
2023-09-06 10:52:01 +00:00
l . Debugf ( "Handling ClusterConfig from %v at %s" , deviceID . Short ( ) , conn )
indexHandlerRegistry := m . ensureIndexHandler ( conn )
2016-07-23 12:46:31 +00:00
2020-11-24 21:20:50 +00:00
deviceCfg , ok := m . cfg . Device ( deviceID )
2020-05-20 09:13:55 +00:00
if ! ok {
2023-09-06 10:52:01 +00:00
l . Debugf ( "Device %s disappeared from config while processing cluster-config" , deviceID . Short ( ) )
2020-05-20 09:13:55 +00:00
return errDeviceUnknown
}
2016-07-27 21:38:43 +00:00
2020-11-09 14:33:32 +00:00
// Assemble the device information from the connected device about
// themselves and us for all folders.
2021-06-03 12:58:50 +00:00
ccDeviceInfos := make ( map [ string ] * clusterConfigDeviceInfo , len ( cm . Folders ) )
2020-11-09 14:33:32 +00:00
for _ , folder := range cm . Folders {
2021-06-03 12:58:50 +00:00
info := & clusterConfigDeviceInfo { }
2020-11-09 14:33:32 +00:00
for _ , dev := range folder . Devices {
if dev . ID == m . id {
info . local = dev
} else if dev . ID == deviceID {
info . remote = dev
}
if info . local . ID != protocol . EmptyDeviceID && info . remote . ID != protocol . EmptyDeviceID {
break
}
}
if info . remote . ID == protocol . EmptyDeviceID {
2023-09-06 10:52:01 +00:00
l . Infof ( "Device %v sent cluster-config without the device info for the remote on folder %v" , deviceID . Short ( ) , folder . Description ( ) )
2020-11-09 14:33:32 +00:00
return errMissingRemoteInClusterConfig
}
if info . local . ID == protocol . EmptyDeviceID {
2023-09-06 10:52:01 +00:00
l . Infof ( "Device %v sent cluster-config without the device info for us locally on folder %v" , deviceID . Short ( ) , folder . Description ( ) )
2020-11-09 14:33:32 +00:00
return errMissingLocalInClusterConfig
}
ccDeviceInfos [ folder . ID ] = info
}
2022-06-20 17:36:45 +00:00
for _ , info := range ccDeviceInfos {
if deviceCfg . Introducer && info . local . Introducer {
l . Warnf ( "Remote %v is an introducer to us, and we are to them - only one should be introducer to the other, see https://docs.syncthing.net/users/introducer.html" , deviceCfg . Description ( ) )
}
break
}
2017-12-07 07:08:24 +00:00
// Needs to happen outside of the fmut, as can cause CommitConfiguration
if deviceCfg . AutoAcceptFolders {
2021-01-15 14:43:34 +00:00
w , _ := m . cfg . Modify ( func ( cfg * config . Configuration ) {
changedFcfg := make ( map [ string ] config . FolderConfiguration )
haveFcfg := cfg . FolderMap ( )
for _ , folder := range cm . Folders {
from , ok := haveFcfg [ folder . ID ]
2023-05-03 08:25:36 +00:00
if to , changed := m . handleAutoAccepts ( deviceID , folder , ccDeviceInfos [ folder . ID ] , from , ok , cfg . Defaults . Folder ) ; changed {
2021-01-15 14:43:34 +00:00
changedFcfg [ folder . ID ] = to
}
2020-05-27 06:05:26 +00:00
}
2021-01-15 14:43:34 +00:00
if len ( changedFcfg ) == 0 {
return
}
for i := range cfg . Folders {
if fcfg , ok := changedFcfg [ cfg . Folders [ i ] . ID ] ; ok {
cfg . Folders [ i ] = fcfg
delete ( changedFcfg , cfg . Folders [ i ] . ID )
}
}
for _ , fcfg := range changedFcfg {
cfg . Folders = append ( cfg . Folders , fcfg )
}
} )
// Need to wait for the waiter, as this calls CommitConfiguration,
// which sets up the folder and as we return from this call,
// ClusterConfig starts poking at m.folderFiles and other things
// that might not exist until the config is committed.
w . Wait ( )
2017-12-07 07:08:24 +00:00
}
2022-04-10 20:47:57 +00:00
tempIndexFolders , states , err := m . ccHandleFolders ( cm . Folders , deviceCfg , ccDeviceInfos , indexHandlerRegistry )
2020-11-09 14:33:32 +00:00
if err != nil {
return err
}
m . pmut . Lock ( )
2022-04-10 20:47:57 +00:00
m . remoteFolderStates [ deviceID ] = states
2020-11-09 14:33:32 +00:00
m . pmut . Unlock ( )
2022-04-22 06:42:20 +00:00
m . evLogger . Log ( events . ClusterConfigReceived , ClusterConfigReceivedEventData {
Device : deviceID ,
} )
2020-11-09 14:33:32 +00:00
if len ( tempIndexFolders ) > 0 {
2023-09-06 10:52:01 +00:00
var connOK bool
var conn protocol . Connection
2020-11-09 14:33:32 +00:00
m . pmut . RLock ( )
2023-09-06 10:52:01 +00:00
if connIDs , connIDOK := m . deviceConnIDs [ deviceID ] ; connIDOK {
conn , connOK = m . connections [ connIDs [ 0 ] ]
}
2020-11-09 14:33:32 +00:00
m . pmut . RUnlock ( )
// In case we've got ClusterConfig, and the connection disappeared
// from infront of our nose.
2023-09-06 10:52:01 +00:00
if connOK {
2020-11-09 14:33:32 +00:00
m . progressEmitter . temporaryIndexSubscribe ( conn , tempIndexFolders )
}
}
if deviceCfg . Introducer {
2021-01-15 14:43:34 +00:00
m . cfg . Modify ( func ( cfg * config . Configuration ) {
folders , devices , foldersDevices , introduced := m . handleIntroductions ( deviceCfg , cm , cfg . FolderMap ( ) , cfg . DeviceMap ( ) )
folders , devices , deintroduced := m . handleDeintroductions ( deviceCfg , foldersDevices , folders , devices )
if ! introduced && ! deintroduced {
return
}
2020-11-09 14:33:32 +00:00
cfg . Folders = make ( [ ] config . FolderConfiguration , 0 , len ( folders ) )
for _ , fcfg := range folders {
cfg . Folders = append ( cfg . Folders , fcfg )
}
2022-12-06 20:22:35 +00:00
cfg . Devices = make ( [ ] config . DeviceConfiguration , 0 , len ( devices ) )
2020-11-09 14:33:32 +00:00
for _ , dcfg := range devices {
cfg . Devices = append ( cfg . Devices , dcfg )
}
2021-01-15 14:43:34 +00:00
} )
2020-11-09 14:33:32 +00:00
}
return nil
}
2023-09-06 10:52:01 +00:00
func ( m * model ) ensureIndexHandler ( conn protocol . Connection ) * indexHandlerRegistry {
deviceID := conn . DeviceID ( )
connID := conn . ConnectionID ( )
2023-11-20 07:12:25 +00:00
// We must acquire fmut first when acquiring both locks.
m . fmut . RLock ( )
defer m . fmut . RUnlock ( )
2023-09-06 10:52:01 +00:00
m . pmut . Lock ( )
defer m . pmut . Unlock ( )
indexHandlerRegistry , ok := m . indexHandlers . Get ( deviceID )
if ok && indexHandlerRegistry . conn . ConnectionID ( ) == connID {
// This is an existing and proper index handler for this connection.
return indexHandlerRegistry
}
if ok {
// A handler exists, but it's for another connection than the one we
// now got a ClusterConfig on. This should be unusual as it means
// the other side has decided to start using a new primary
// connection but we haven't seen it close yet. Ideally it will
// close shortly by itself...
l . Infof ( "Abandoning old index handler for %s (%s) in favour of %s" , deviceID . Short ( ) , indexHandlerRegistry . conn . ConnectionID ( ) , connID )
m . indexHandlers . RemoveAndWait ( deviceID , 0 )
}
// Create a new index handler for this device.
indexHandlerRegistry = newIndexHandlerRegistry ( conn , m . deviceDownloads [ deviceID ] , m . evLogger )
for id , fcfg := range m . folderCfgs {
l . Debugln ( "Registering folder" , id , "for" , deviceID . Short ( ) )
runner , _ := m . folderRunners . Get ( id )
indexHandlerRegistry . RegisterFolderState ( fcfg , m . folderFiles [ id ] , runner )
}
m . indexHandlers . Add ( deviceID , indexHandlerRegistry )
return indexHandlerRegistry
}
func ( m * model ) getIndexHandlerPRLocked ( conn protocol . Connection ) ( * indexHandlerRegistry , bool ) {
// Reads from index handlers, which requires pmut to be read locked
deviceID := conn . DeviceID ( )
connID := conn . ConnectionID ( )
indexHandlerRegistry , ok := m . indexHandlers . Get ( deviceID )
if ok && indexHandlerRegistry . conn . ConnectionID ( ) == connID {
// This is an existing and proper index handler for this connection.
return indexHandlerRegistry , true
}
// There is no index handler, or it's not registered for this connection.
return nil , false
}
2022-04-10 20:47:57 +00:00
func ( m * model ) ccHandleFolders ( folders [ ] protocol . Folder , deviceCfg config . DeviceConfiguration , ccDeviceInfos map [ string ] * clusterConfigDeviceInfo , indexHandlers * indexHandlerRegistry ) ( [ ] string , map [ string ] remoteFolderState , error ) {
2020-11-09 14:33:32 +00:00
var folderDevice config . FolderDeviceConfiguration
tempIndexFolders := make ( [ ] string , 0 , len ( folders ) )
2022-04-10 20:47:57 +00:00
seenFolders := make ( map [ string ] remoteFolderState , len ( folders ) )
2021-02-12 21:51:29 +00:00
updatedPending := make ( [ ] updatedPendingFolder , 0 , len ( folders ) )
2020-11-09 14:33:32 +00:00
deviceID := deviceCfg . DeviceID
2021-04-11 13:24:08 +00:00
expiredPending , err := m . db . PendingFoldersForDevice ( deviceID )
if err != nil {
l . Infof ( "Could not get pending folders for cleanup: %v" , err )
}
of := db . ObservedFolder { Time : time . Now ( ) . Truncate ( time . Second ) }
2020-11-09 14:33:32 +00:00
for _ , folder := range folders {
2022-04-22 06:42:20 +00:00
seenFolders [ folder . ID ] = remoteFolderValid
2020-10-21 09:51:53 +00:00
2018-06-06 21:34:11 +00:00
cfg , ok := m . cfg . Folder ( folder . ID )
2020-11-09 14:33:32 +00:00
if ok {
folderDevice , ok = cfg . Device ( deviceID )
}
if ! ok {
2021-06-03 12:58:50 +00:00
indexHandlers . Remove ( folder . ID )
2018-08-25 10:36:10 +00:00
if deviceCfg . IgnoredFolder ( folder . ID ) {
2018-06-06 21:34:11 +00:00
l . Infof ( "Ignoring folder %s from device %s since we are configured to" , folder . Description ( ) , deviceID )
continue
}
2021-04-11 13:24:08 +00:00
delete ( expiredPending , folder . ID )
of . Label = folder . Label
of . ReceiveEncrypted = len ( ccDeviceInfos [ folder . ID ] . local . EncryptionPasswordToken ) > 0
2021-06-17 11:53:02 +00:00
of . RemoteEncrypted = len ( ccDeviceInfos [ folder . ID ] . remote . EncryptionPasswordToken ) > 0
2021-04-11 13:24:08 +00:00
if err := m . db . AddOrUpdatePendingFolder ( folder . ID , of , deviceID ) ; err != nil {
2020-12-17 18:54:31 +00:00
l . Warnf ( "Failed to persist pending folder entry to database: %v" , err )
}
2021-06-05 14:27:15 +00:00
if ! folder . Paused {
indexHandlers . AddIndexInfo ( folder . ID , ccDeviceInfos [ folder . ID ] )
}
2021-02-12 21:51:29 +00:00
updatedPending = append ( updatedPending , updatedPendingFolder {
FolderID : folder . ID ,
FolderLabel : folder . Label ,
DeviceID : deviceID ,
2021-04-11 13:24:08 +00:00
ReceiveEncrypted : of . ReceiveEncrypted ,
2021-06-17 11:53:02 +00:00
RemoteEncrypted : of . RemoteEncrypted ,
lib/model: Forget pending folders no longer announced in ClusterConfig (fixes #5187) (#7205)
* lib/db: Add ExpirePendingFolders().
Use-case is to drop any no-longer-pending folders for a specific
device when parsing its ClusterConfig message where previously offered
folders are not mentioned any more.
The timestamp in ObservedFolder is stored with only second precision,
so round to seconds here as well. This allows calling the function
within the same second of adding or updating entries.
* lib/model: Weed out pending folders when receiving ClusterConfig.
Filter the entries by timestamp, which must be newer than or equal to
the reception time of the ClusterConfig. For just mentioned ones,
this assumption will hold as AddOrUpdatePendingFolder() updates the
timestamp.
* lib/model, gui: Notify when one or more pending folders expired.
Introduce new event type FolderOfferCancelled and use it to trigger a
complete refreshCluster() cycle. Listing individual entries would be
much more code and probably just as much work to answer the API
request.
* lib/model: Add comment and rename ExpirePendingFolders().
* lib/events: Rename FolderOfferCancelled to ClusterPendingChanged.
* lib/model: Reuse ClusterPendingChanged event for cleanPending()
Changing the config does not necessarily mean that the
/resut/cluster/pending endpoints need to be refreshed, but only if
something was actually removed. Detect this and indicate it through
the ClusterPendingChanged event, which is already hooked up to requery
respective endpoints within the GUI.
No more need for a separate refreshCluster() in reaction to
ConfigSaved event or calling refreshConfig().
* lib/model: Gofmt.
* lib/db: Warn instead of info log for failed removal.
* gui: Fix pending notifications not loading on GUI start.
* lib/db: Use short device ID in log message.
* lib/db: Return list of expired folder IDs after deleting them.
* lib/model: Refactor Pending...Changed events.
* lib/model: Adjust format of removed pending folders enumeration.
Use an array of objects with device / folder ID properties, matching
the other places where it's used.
* lib/db: Drop invalid entries in RemovePendingFoldersBeforeTime().
* lib/model: Gofmt.
My local gofmt did not complain here, strangely...
* gui: Handle PendingDevicesChanged event.
Even though it currently only holds one device at a time, wrap the
contents in an array under the "added" property name.
* lib/model: Fix null values in PendingFoldersChanged removed member.
* gui: Handle PendingFoldersChanged event.
* lib/model: Simplify construction of expiredPendingList.
* lib/model: Reduce code duplication in cleanPending().
Use goto and a label for the common parts of calling the DB removal
function and building the event data part.
* lib/events, gui: Mark ...Rejected events deprecated.
Extend comments explaining the conditions when the replacement event
types are emitted.
* lib/model: Wrap removed devices in array of objects as well.
* lib/db: Use iter.Value() instead of needless db.Get(iter.Key())
* lib/db: Add comment explaining RemovePendingFoldersBeforeTime().
* lib/model: Rename fields folderID and deviceID in event data.
* lib/db: Only list actually expired IDs as removed.
Skip entries where Delete() failed as well as invalid entries that got
removed automatically.
* lib/model: Gofmt
2021-01-25 10:58:10 +00:00
} )
// DEPRECATED: Only for backwards compatibility, should be removed.
2019-08-15 14:29:37 +00:00
m . evLogger . Log ( events . FolderRejected , map [ string ] string {
2016-03-11 09:48:46 +00:00
"folder" : folder . ID ,
"folderLabel" : folder . Label ,
"device" : deviceID . String ( ) ,
2016-01-01 19:11:12 +00:00
} )
2016-11-22 07:36:14 +00:00
l . Infof ( "Unexpected folder %s sent from device %q; ensure that the folder exists and that this device is selected under \"Share With\" in the folder configuration." , folder . Description ( ) , deviceID )
2016-01-01 19:11:12 +00:00
continue
}
2020-10-21 09:51:53 +00:00
2018-06-06 21:34:11 +00:00
if folder . Paused {
2021-06-03 12:58:50 +00:00
indexHandlers . Remove ( folder . ID )
2022-04-22 06:42:20 +00:00
seenFolders [ cfg . ID ] = remoteFolderPaused
2018-06-06 21:34:11 +00:00
continue
}
2020-10-21 09:51:53 +00:00
2018-06-06 21:34:11 +00:00
if cfg . Paused {
2021-06-03 12:58:50 +00:00
indexHandlers . AddIndexInfo ( folder . ID , ccDeviceInfos [ folder . ID ] )
2018-06-06 21:34:11 +00:00
continue
}
2020-10-21 09:51:53 +00:00
2020-11-09 14:33:32 +00:00
if err := m . ccCheckEncryption ( cfg , folderDevice , ccDeviceInfos [ folder . ID ] , deviceCfg . Untrusted ) ; err != nil {
sameError := false
2021-07-22 09:15:25 +00:00
m . fmut . Lock ( )
2020-11-09 14:33:32 +00:00
if devs , ok := m . folderEncryptionFailures [ folder . ID ] ; ok {
sameError = devs [ deviceID ] == err
} else {
m . folderEncryptionFailures [ folder . ID ] = make ( map [ protocol . DeviceID ] error )
}
m . folderEncryptionFailures [ folder . ID ] [ deviceID ] = err
2021-07-22 09:15:25 +00:00
m . fmut . Unlock ( )
2020-11-09 14:33:32 +00:00
msg := fmt . Sprintf ( "Failure checking encryption consistency with device %v for folder %v: %v" , deviceID , cfg . Description ( ) , err )
2021-03-23 09:38:40 +00:00
if sameError {
2020-11-09 14:33:32 +00:00
l . Debugln ( msg )
} else {
2021-04-29 17:21:07 +00:00
if rerr , ok := err . ( * redactedError ) ; ok {
err = rerr . redacted
}
m . evLogger . Log ( events . Failure , err . Error ( ) )
2020-11-09 14:33:32 +00:00
l . Warnln ( msg )
}
2022-04-10 20:47:57 +00:00
return tempIndexFolders , seenFolders , err
2020-11-09 14:33:32 +00:00
}
2021-07-22 09:15:25 +00:00
m . fmut . Lock ( )
2020-11-09 14:33:32 +00:00
if devErrs , ok := m . folderEncryptionFailures [ folder . ID ] ; ok {
if len ( devErrs ) == 1 {
delete ( m . folderEncryptionFailures , folder . ID )
} else {
delete ( m . folderEncryptionFailures [ folder . ID ] , deviceID )
}
}
2021-07-22 09:15:25 +00:00
m . fmut . Unlock ( )
2020-11-09 14:33:32 +00:00
// Handle indexes
2016-07-04 10:40:29 +00:00
if ! folder . DisableTempIndexes {
2016-04-15 10:59:41 +00:00
tempIndexFolders = append ( tempIndexFolders , folder . ID )
}
2016-07-23 12:46:31 +00:00
2021-06-03 12:58:50 +00:00
indexHandlers . AddIndexInfo ( folder . ID , ccDeviceInfos [ folder . ID ] )
2016-01-01 19:11:12 +00:00
}
2021-06-03 12:58:50 +00:00
indexHandlers . RemoveAllExcept ( seenFolders )
2022-04-10 20:47:57 +00:00
// Explicitly mark folders we offer, but the remote has not accepted
for folderID , cfg := range m . cfg . Folders ( ) {
if _ , seen := seenFolders [ folderID ] ; ! seen && cfg . SharedWith ( deviceID ) {
2022-04-22 06:42:20 +00:00
l . Debugf ( "Remote device %v has not accepted sharing folder %s" , deviceID . Short ( ) , cfg . Description ( ) )
seenFolders [ folderID ] = remoteFolderNotSharing
2022-04-10 20:47:57 +00:00
}
}
2021-06-07 08:29:24 +00:00
expiredPendingList := make ( [ ] map [ string ] string , 0 , len ( expiredPending ) )
2021-04-11 13:24:08 +00:00
for folder := range expiredPending {
2021-06-07 08:29:24 +00:00
if err = m . db . RemovePendingFolderForDevice ( folder , deviceID ) ; err != nil {
2021-06-09 11:35:17 +00:00
msg := "Failed to remove pending folder-device entry"
l . Warnf ( "%v (%v, %v): %v" , msg , folder , deviceID , err )
m . evLogger . Log ( events . Failure , msg )
2021-06-07 08:29:24 +00:00
continue
lib/model: Forget pending folders no longer announced in ClusterConfig (fixes #5187) (#7205)
* lib/db: Add ExpirePendingFolders().
Use-case is to drop any no-longer-pending folders for a specific
device when parsing its ClusterConfig message where previously offered
folders are not mentioned any more.
The timestamp in ObservedFolder is stored with only second precision,
so round to seconds here as well. This allows calling the function
within the same second of adding or updating entries.
* lib/model: Weed out pending folders when receiving ClusterConfig.
Filter the entries by timestamp, which must be newer than or equal to
the reception time of the ClusterConfig. For just mentioned ones,
this assumption will hold as AddOrUpdatePendingFolder() updates the
timestamp.
* lib/model, gui: Notify when one or more pending folders expired.
Introduce new event type FolderOfferCancelled and use it to trigger a
complete refreshCluster() cycle. Listing individual entries would be
much more code and probably just as much work to answer the API
request.
* lib/model: Add comment and rename ExpirePendingFolders().
* lib/events: Rename FolderOfferCancelled to ClusterPendingChanged.
* lib/model: Reuse ClusterPendingChanged event for cleanPending()
Changing the config does not necessarily mean that the
/resut/cluster/pending endpoints need to be refreshed, but only if
something was actually removed. Detect this and indicate it through
the ClusterPendingChanged event, which is already hooked up to requery
respective endpoints within the GUI.
No more need for a separate refreshCluster() in reaction to
ConfigSaved event or calling refreshConfig().
* lib/model: Gofmt.
* lib/db: Warn instead of info log for failed removal.
* gui: Fix pending notifications not loading on GUI start.
* lib/db: Use short device ID in log message.
* lib/db: Return list of expired folder IDs after deleting them.
* lib/model: Refactor Pending...Changed events.
* lib/model: Adjust format of removed pending folders enumeration.
Use an array of objects with device / folder ID properties, matching
the other places where it's used.
* lib/db: Drop invalid entries in RemovePendingFoldersBeforeTime().
* lib/model: Gofmt.
My local gofmt did not complain here, strangely...
* gui: Handle PendingDevicesChanged event.
Even though it currently only holds one device at a time, wrap the
contents in an array under the "added" property name.
* lib/model: Fix null values in PendingFoldersChanged removed member.
* gui: Handle PendingFoldersChanged event.
* lib/model: Simplify construction of expiredPendingList.
* lib/model: Reduce code duplication in cleanPending().
Use goto and a label for the common parts of calling the DB removal
function and building the event data part.
* lib/events, gui: Mark ...Rejected events deprecated.
Extend comments explaining the conditions when the replacement event
types are emitted.
* lib/model: Wrap removed devices in array of objects as well.
* lib/db: Use iter.Value() instead of needless db.Get(iter.Key())
* lib/db: Add comment explaining RemovePendingFoldersBeforeTime().
* lib/model: Rename fields folderID and deviceID in event data.
* lib/db: Only list actually expired IDs as removed.
Skip entries where Delete() failed as well as invalid entries that got
removed automatically.
* lib/model: Gofmt
2021-01-25 10:58:10 +00:00
}
2021-06-07 08:29:24 +00:00
expiredPendingList = append ( expiredPendingList , map [ string ] string {
"folderID" : folder ,
"deviceID" : deviceID . String ( ) ,
} )
}
if len ( updatedPending ) > 0 || len ( expiredPendingList ) > 0 {
lib/model: Forget pending folders no longer announced in ClusterConfig (fixes #5187) (#7205)
* lib/db: Add ExpirePendingFolders().
Use-case is to drop any no-longer-pending folders for a specific
device when parsing its ClusterConfig message where previously offered
folders are not mentioned any more.
The timestamp in ObservedFolder is stored with only second precision,
so round to seconds here as well. This allows calling the function
within the same second of adding or updating entries.
* lib/model: Weed out pending folders when receiving ClusterConfig.
Filter the entries by timestamp, which must be newer than or equal to
the reception time of the ClusterConfig. For just mentioned ones,
this assumption will hold as AddOrUpdatePendingFolder() updates the
timestamp.
* lib/model, gui: Notify when one or more pending folders expired.
Introduce new event type FolderOfferCancelled and use it to trigger a
complete refreshCluster() cycle. Listing individual entries would be
much more code and probably just as much work to answer the API
request.
* lib/model: Add comment and rename ExpirePendingFolders().
* lib/events: Rename FolderOfferCancelled to ClusterPendingChanged.
* lib/model: Reuse ClusterPendingChanged event for cleanPending()
Changing the config does not necessarily mean that the
/resut/cluster/pending endpoints need to be refreshed, but only if
something was actually removed. Detect this and indicate it through
the ClusterPendingChanged event, which is already hooked up to requery
respective endpoints within the GUI.
No more need for a separate refreshCluster() in reaction to
ConfigSaved event or calling refreshConfig().
* lib/model: Gofmt.
* lib/db: Warn instead of info log for failed removal.
* gui: Fix pending notifications not loading on GUI start.
* lib/db: Use short device ID in log message.
* lib/db: Return list of expired folder IDs after deleting them.
* lib/model: Refactor Pending...Changed events.
* lib/model: Adjust format of removed pending folders enumeration.
Use an array of objects with device / folder ID properties, matching
the other places where it's used.
* lib/db: Drop invalid entries in RemovePendingFoldersBeforeTime().
* lib/model: Gofmt.
My local gofmt did not complain here, strangely...
* gui: Handle PendingDevicesChanged event.
Even though it currently only holds one device at a time, wrap the
contents in an array under the "added" property name.
* lib/model: Fix null values in PendingFoldersChanged removed member.
* gui: Handle PendingFoldersChanged event.
* lib/model: Simplify construction of expiredPendingList.
* lib/model: Reduce code duplication in cleanPending().
Use goto and a label for the common parts of calling the DB removal
function and building the event data part.
* lib/events, gui: Mark ...Rejected events deprecated.
Extend comments explaining the conditions when the replacement event
types are emitted.
* lib/model: Wrap removed devices in array of objects as well.
* lib/db: Use iter.Value() instead of needless db.Get(iter.Key())
* lib/db: Add comment explaining RemovePendingFoldersBeforeTime().
* lib/model: Rename fields folderID and deviceID in event data.
* lib/db: Only list actually expired IDs as removed.
Skip entries where Delete() failed as well as invalid entries that got
removed automatically.
* lib/model: Gofmt
2021-01-25 10:58:10 +00:00
m . evLogger . Log ( events . PendingFoldersChanged , map [ string ] interface { } {
"added" : updatedPending ,
"removed" : expiredPendingList ,
} )
}
2020-10-21 09:51:53 +00:00
2022-04-10 20:47:57 +00:00
return tempIndexFolders , seenFolders , nil
2020-11-09 14:33:32 +00:00
}
2016-12-21 18:41:25 +00:00
2021-06-03 12:58:50 +00:00
func ( m * model ) ccCheckEncryption ( fcfg config . FolderConfiguration , folderDevice config . FolderDeviceConfiguration , ccDeviceInfos * clusterConfigDeviceInfo , deviceUntrusted bool ) error {
2020-11-09 14:33:32 +00:00
hasTokenRemote := len ( ccDeviceInfos . remote . EncryptionPasswordToken ) > 0
hasTokenLocal := len ( ccDeviceInfos . local . EncryptionPasswordToken ) > 0
isEncryptedRemote := folderDevice . EncryptionPassword != ""
isEncryptedLocal := fcfg . Type == config . FolderTypeReceiveEncrypted
if ! isEncryptedRemote && ! isEncryptedLocal && deviceUntrusted {
return errEncryptionNotEncryptedUntrusted
}
if ! ( hasTokenRemote || hasTokenLocal || isEncryptedRemote || isEncryptedLocal ) {
2022-08-23 13:44:11 +00:00
// No one cares about encryption here
2020-11-09 14:33:32 +00:00
return nil
}
if isEncryptedRemote && isEncryptedLocal {
// Should never happen, but config racyness and be safe.
return errEncryptionInvConfigLocal
}
if hasTokenRemote && hasTokenLocal {
return errEncryptionInvConfigRemote
}
if ! ( hasTokenRemote || hasTokenLocal ) {
2021-05-11 05:55:44 +00:00
if isEncryptedRemote {
return errEncryptionPlainForRemoteEncrypted
2021-05-28 20:20:18 +00:00
} else {
return errEncryptionPlainForReceiveEncrypted
2021-05-11 05:55:44 +00:00
}
2020-11-09 14:33:32 +00:00
}
if ! ( isEncryptedRemote || isEncryptedLocal ) {
return errEncryptionNotEncryptedLocal
}
if isEncryptedRemote {
2023-03-12 19:06:59 +00:00
passwordToken := protocol . PasswordToken ( m . keyGen , fcfg . ID , folderDevice . EncryptionPassword )
2020-11-09 14:33:32 +00:00
match := false
if hasTokenLocal {
match = bytes . Equal ( passwordToken , ccDeviceInfos . local . EncryptionPasswordToken )
} else {
// hasTokenRemote == true
match = bytes . Equal ( passwordToken , ccDeviceInfos . remote . EncryptionPasswordToken )
2016-04-15 10:59:41 +00:00
}
2020-11-09 14:33:32 +00:00
if ! match {
return errEncryptionPassword
}
return nil
2016-04-15 10:59:41 +00:00
}
2020-11-09 14:33:32 +00:00
// isEncryptedLocal == true
var ccToken [ ] byte
if hasTokenLocal {
ccToken = ccDeviceInfos . local . EncryptionPasswordToken
} else {
// hasTokenRemote == true
ccToken = ccDeviceInfos . remote . EncryptionPasswordToken
}
m . fmut . RLock ( )
token , ok := m . folderEncryptionPasswordTokens [ fcfg . ID ]
m . fmut . RUnlock ( )
if ! ok {
var err error
token , err = readEncryptionToken ( fcfg )
if err != nil && ! fs . IsNotExist ( err ) {
2021-04-29 17:21:07 +00:00
if rerr , ok := redactPathError ( err ) ; ok {
return rerr
}
return & redactedError {
error : err ,
redacted : errEncryptionTokenRead ,
}
2020-11-09 14:33:32 +00:00
}
if err == nil {
m . fmut . Lock ( )
m . folderEncryptionPasswordTokens [ fcfg . ID ] = token
m . fmut . Unlock ( )
} else {
if err := writeEncryptionToken ( ccToken , fcfg ) ; err != nil {
2021-04-29 17:21:07 +00:00
if rerr , ok := redactPathError ( err ) ; ok {
return rerr
} else {
return & redactedError {
error : err ,
redacted : errEncryptionTokenWrite ,
}
}
2019-11-21 07:41:41 +00:00
}
2020-11-09 14:33:32 +00:00
m . fmut . Lock ( )
m . folderEncryptionPasswordTokens [ fcfg . ID ] = ccToken
m . fmut . Unlock ( )
2022-08-23 13:44:11 +00:00
// We can only announce ourselves once we have the token,
2020-11-09 14:33:32 +00:00
// thus we need to resend CCs now that we have it.
2021-03-22 20:50:19 +00:00
m . sendClusterConfig ( fcfg . DeviceIDs ( ) )
2020-11-09 14:33:32 +00:00
return nil
2016-11-07 16:40:48 +00:00
}
}
2020-11-09 14:33:32 +00:00
if ! bytes . Equal ( token , ccToken ) {
return errEncryptionPassword
}
return nil
}
2014-09-23 14:04:20 +00:00
2021-03-22 20:50:19 +00:00
func ( m * model ) sendClusterConfig ( ids [ ] protocol . DeviceID ) {
2020-11-09 14:33:32 +00:00
if len ( ids ) == 0 {
return
}
ccConns := make ( [ ] protocol . Connection , 0 , len ( ids ) )
m . pmut . RLock ( )
for _ , id := range ids {
2023-09-06 10:52:01 +00:00
if connIDs , ok := m . deviceConnIDs [ id ] ; ok {
ccConns = append ( ccConns , m . connections [ connIDs [ 0 ] ] )
2016-11-07 16:40:48 +00:00
}
}
2020-11-09 14:33:32 +00:00
m . pmut . RUnlock ( )
// Generating cluster-configs acquires fmut -> must happen outside of pmut.
for _ , conn := range ccConns {
2023-07-29 08:24:44 +00:00
cm , passwords := m . generateClusterConfig ( conn . DeviceID ( ) )
2021-03-22 20:50:19 +00:00
conn . SetFolderPasswords ( passwords )
2020-11-09 14:33:32 +00:00
go conn . ClusterConfig ( cm )
}
2016-11-07 16:40:48 +00:00
}
2014-09-23 14:04:20 +00:00
2019-11-21 07:41:41 +00:00
// handleIntroductions handles adding devices/folders that are shared by an introducer device
2021-01-15 14:43:34 +00:00
func ( m * model ) handleIntroductions ( introducerCfg config . DeviceConfiguration , cm protocol . ClusterConfig , folders map [ string ] config . FolderConfiguration , devices map [ protocol . DeviceID ] config . DeviceConfiguration ) ( map [ string ] config . FolderConfiguration , map [ protocol . DeviceID ] config . DeviceConfiguration , folderDeviceSet , bool ) {
2016-11-07 16:40:48 +00:00
changed := false
2014-09-23 14:04:20 +00:00
2016-11-07 16:40:48 +00:00
foldersDevices := make ( folderDeviceSet )
2014-09-23 14:04:20 +00:00
2016-11-07 16:40:48 +00:00
for _ , folder := range cm . Folders {
// Adds devices which we do not have, but the introducer has
// for the folders that we have in common. Also, shares folders
// with devices that we have in common, yet are currently not sharing
// the folder.
2018-01-03 07:42:25 +00:00
2019-11-21 07:41:41 +00:00
fcfg , ok := folders [ folder . ID ]
2018-01-03 07:42:25 +00:00
if ! ok {
// Don't have this folder, carry on.
continue
}
2019-11-21 07:41:41 +00:00
folderChanged := false
2016-11-07 16:40:48 +00:00
for _ , device := range folder . Devices {
2017-12-07 07:08:24 +00:00
// No need to share with self.
if device . ID == m . id {
continue
}
2016-11-07 16:40:48 +00:00
foldersDevices . set ( device . ID , folder . ID )
2016-11-07 16:40:48 +00:00
2021-01-15 14:43:34 +00:00
if _ , ok := devices [ device . ID ] ; ! ok {
2016-11-07 16:40:48 +00:00
// The device is currently unknown. Add it to the config.
2019-11-21 07:41:41 +00:00
devices [ device . ID ] = m . introduceDevice ( device , introducerCfg )
2018-06-06 21:34:11 +00:00
} else if fcfg . SharedWith ( device . ID ) {
// We already share the folder with this device, so
// nothing to do.
continue
2016-11-07 16:40:48 +00:00
}
2016-11-07 16:40:48 +00:00
2021-06-03 13:02:57 +00:00
if fcfg . Type != config . FolderTypeReceiveEncrypted && device . EncryptionPasswordToken != nil {
l . Infof ( "Cannot share folder %s with %v because the introducer %v encrypts data, which requires a password" , folder . Description ( ) , device . ID , introducerCfg . DeviceID )
continue
}
2016-11-07 16:40:48 +00:00
// We don't yet share this folder with this device. Add the device
// to sharing list of the folder.
2017-12-07 07:08:24 +00:00
l . Infof ( "Sharing folder %s with %v (vouched for by introducer %v)" , folder . Description ( ) , device . ID , introducerCfg . DeviceID )
2018-01-03 07:42:25 +00:00
fcfg . Devices = append ( fcfg . Devices , config . FolderDeviceConfiguration {
DeviceID : device . ID ,
IntroducedBy : introducerCfg . DeviceID ,
} )
2019-11-21 07:41:41 +00:00
folderChanged = true
2016-11-07 16:40:48 +00:00
}
2018-01-03 07:42:25 +00:00
2019-11-21 07:41:41 +00:00
if folderChanged {
folders [ fcfg . ID ] = fcfg
changed = true
2018-01-03 07:42:25 +00:00
}
2016-11-07 16:40:48 +00:00
}
2016-11-07 16:40:48 +00:00
2019-11-21 07:41:41 +00:00
return folders , devices , foldersDevices , changed
2016-11-07 16:40:48 +00:00
}
2016-11-07 16:40:48 +00:00
2017-12-07 07:08:24 +00:00
// handleDeintroductions handles removals of devices/shares that are removed by an introducer device
2022-07-28 15:32:45 +00:00
func ( * model ) handleDeintroductions ( introducerCfg config . DeviceConfiguration , foldersDevices folderDeviceSet , folders map [ string ] config . FolderConfiguration , devices map [ protocol . DeviceID ] config . DeviceConfiguration ) ( map [ string ] config . FolderConfiguration , map [ protocol . DeviceID ] config . DeviceConfiguration , bool ) {
2019-11-21 07:41:41 +00:00
if introducerCfg . SkipIntroductionRemovals {
return folders , devices , false
}
2016-11-07 16:40:48 +00:00
changed := false
2018-06-06 21:34:11 +00:00
devicesNotIntroduced := make ( map [ protocol . DeviceID ] struct { } )
2016-11-07 16:40:48 +00:00
2016-11-07 16:40:48 +00:00
// Check if we should unshare some folders, if the introducer has unshared them.
2019-11-21 07:41:41 +00:00
for folderID , folderCfg := range folders {
for k := 0 ; k < len ( folderCfg . Devices ) ; k ++ {
if folderCfg . Devices [ k ] . IntroducedBy != introducerCfg . DeviceID {
devicesNotIntroduced [ folderCfg . Devices [ k ] . DeviceID ] = struct { } { }
2018-06-06 21:34:11 +00:00
continue
}
2019-11-21 07:41:41 +00:00
if ! foldersDevices . has ( folderCfg . Devices [ k ] . DeviceID , folderCfg . ID ) {
2018-06-06 21:34:11 +00:00
// We could not find that folder shared on the
// introducer with the device that was introduced to us.
// We should follow and unshare as well.
2019-11-21 07:41:41 +00:00
l . Infof ( "Unsharing folder %s with %v as introducer %v no longer shares the folder with that device" , folderCfg . Description ( ) , folderCfg . Devices [ k ] . DeviceID , folderCfg . Devices [ k ] . IntroducedBy )
folderCfg . Devices = append ( folderCfg . Devices [ : k ] , folderCfg . Devices [ k + 1 : ] ... )
folders [ folderID ] = folderCfg
2018-06-06 21:34:11 +00:00
k --
changed = true
2016-11-07 16:40:48 +00:00
}
}
}
2016-11-07 16:40:48 +00:00
2016-11-17 08:56:55 +00:00
// Check if we should remove some devices, if the introducer no longer
// shares any folder with them. Yet do not remove if we share other
// folders that haven't been introduced by the introducer.
2019-11-21 07:41:41 +00:00
for deviceID , device := range devices {
2016-11-17 08:56:55 +00:00
if device . IntroducedBy == introducerCfg . DeviceID {
2018-06-06 21:34:11 +00:00
if ! foldersDevices . hasDevice ( deviceID ) {
if _ , ok := devicesNotIntroduced [ deviceID ] ; ! ok {
// The introducer no longer shares any folder with the
// device, remove the device.
l . Infof ( "Removing device %v as introducer %v no longer shares any folders with that device" , deviceID , device . IntroducedBy )
changed = true
2019-11-21 07:41:41 +00:00
delete ( devices , deviceID )
2016-11-07 16:40:48 +00:00
continue
}
2018-06-06 21:34:11 +00:00
l . Infof ( "Would have removed %v as %v no longer shares any folders, yet there are other folders that are shared with this device that haven't been introduced by this introducer." , deviceID , device . IntroducedBy )
2016-11-12 07:38:29 +00:00
}
}
2016-11-07 16:40:48 +00:00
}
2019-11-21 07:41:41 +00:00
return folders , devices , changed
2016-11-07 16:40:48 +00:00
}
2017-12-07 07:08:24 +00:00
// handleAutoAccepts handles adding and sharing folders for devices that have
// AutoAcceptFolders set to true.
2023-05-03 08:25:36 +00:00
func ( m * model ) handleAutoAccepts ( deviceID protocol . DeviceID , folder protocol . Folder , ccDeviceInfos * clusterConfigDeviceInfo , cfg config . FolderConfiguration , haveCfg bool , defaultFolderCfg config . FolderConfiguration ) ( config . FolderConfiguration , bool ) {
2021-01-15 14:43:34 +00:00
if ! haveCfg {
2023-05-03 08:25:36 +00:00
defaultPathFs := fs . NewFilesystem ( defaultFolderCfg . FilesystemType , defaultFolderCfg . Path )
2023-04-11 11:07:22 +00:00
var pathAlternatives [ ] string
if alt := fs . SanitizePath ( folder . Label ) ; alt != "" {
pathAlternatives = append ( pathAlternatives , alt )
}
if alt := fs . SanitizePath ( folder . ID ) ; alt != "" {
pathAlternatives = append ( pathAlternatives , alt )
}
if len ( pathAlternatives ) == 0 {
l . Infof ( "Failed to auto-accept folder %s from %s due to lack of path alternatives" , folder . Description ( ) , deviceID )
return config . FolderConfiguration { } , false
2019-01-05 17:10:02 +00:00
}
for _ , path := range pathAlternatives {
2023-04-11 11:07:22 +00:00
// Make sure the folder path doesn't already exist.
2017-12-07 07:08:24 +00:00
if _ , err := defaultPathFs . Lstat ( path ) ; ! fs . IsNotExist ( err ) {
continue
}
2023-04-11 11:07:22 +00:00
// Attempt to create it to make sure it does, now.
2023-05-03 08:25:36 +00:00
fullPath := filepath . Join ( defaultFolderCfg . Path , path )
2023-04-11 11:07:22 +00:00
if err := defaultPathFs . MkdirAll ( path , 0 o700 ) ; err != nil {
l . Warnf ( "Failed to create path for auto-accepted folder %s at path %s: %v" , folder . Description ( ) , fullPath , err )
continue
}
2023-05-03 08:25:36 +00:00
fcfg := newFolderConfiguration ( m . cfg , folder . ID , folder . Label , defaultFolderCfg . FilesystemType , fullPath )
2018-01-03 07:42:25 +00:00
fcfg . Devices = append ( fcfg . Devices , config . FolderDeviceConfiguration {
2020-11-09 14:33:32 +00:00
DeviceID : deviceID ,
2018-01-03 07:42:25 +00:00
} )
2017-12-07 07:08:24 +00:00
2020-11-09 14:33:32 +00:00
if len ( ccDeviceInfos . remote . EncryptionPasswordToken ) > 0 || len ( ccDeviceInfos . local . EncryptionPasswordToken ) > 0 {
fcfg . Type = config . FolderTypeReceiveEncrypted
2022-07-22 09:27:58 +00:00
// Override the user-configured defaults, as normally done by the GUI
fcfg . FSWatcherEnabled = false
2022-12-21 18:52:22 +00:00
if fcfg . RescanIntervalS != 0 {
minRescanInterval := 3600 * 24
if fcfg . RescanIntervalS < minRescanInterval {
fcfg . RescanIntervalS = minRescanInterval
}
}
2022-07-22 09:27:58 +00:00
fcfg . Versioning . Reset ( )
// Other necessary settings are ensured by FolderConfiguration itself
2022-01-13 22:38:21 +00:00
} else {
ignores := m . cfg . DefaultIgnores ( )
if err := m . setIgnores ( fcfg , ignores . Lines ) ; err != nil {
l . Warnf ( "Failed to apply default ignores to auto-accepted folder %s at path %s: %v" , folder . Description ( ) , fcfg . Path , err )
}
2020-11-09 14:33:32 +00:00
}
l . Infof ( "Auto-accepted %s folder %s at path %s" , deviceID , folder . Description ( ) , fcfg . Path )
2020-05-27 06:05:26 +00:00
return fcfg , true
2017-12-07 07:08:24 +00:00
}
2020-11-09 14:33:32 +00:00
l . Infof ( "Failed to auto-accept folder %s from %s due to path conflict" , folder . Description ( ) , deviceID )
2020-05-27 06:05:26 +00:00
return config . FolderConfiguration { } , false
2018-01-03 07:42:25 +00:00
} else {
for _ , device := range cfg . DeviceIDs ( ) {
2020-11-09 14:33:32 +00:00
if device == deviceID {
2018-01-03 07:42:25 +00:00
// Already shared nothing todo.
2020-05-27 06:05:26 +00:00
return config . FolderConfiguration { } , false
2018-01-03 07:42:25 +00:00
}
}
2020-11-09 14:33:32 +00:00
if cfg . Type == config . FolderTypeReceiveEncrypted {
if len ( ccDeviceInfos . remote . EncryptionPasswordToken ) == 0 && len ( ccDeviceInfos . local . EncryptionPasswordToken ) == 0 {
l . Infof ( "Failed to auto-accept device %s on existing folder %s as the remote wants to send us unencrypted data, but the folder type is receive-encrypted" , folder . Description ( ) , deviceID )
return config . FolderConfiguration { } , false
}
} else {
if len ( ccDeviceInfos . remote . EncryptionPasswordToken ) > 0 || len ( ccDeviceInfos . local . EncryptionPasswordToken ) > 0 {
l . Infof ( "Failed to auto-accept device %s on existing folder %s as the remote wants to send us encrypted data, but the folder type is not receive-encrypted" , folder . Description ( ) , deviceID )
return config . FolderConfiguration { } , false
}
}
2018-01-03 07:42:25 +00:00
cfg . Devices = append ( cfg . Devices , config . FolderDeviceConfiguration {
2020-11-09 14:33:32 +00:00
DeviceID : deviceID ,
2018-01-03 07:42:25 +00:00
} )
2020-11-09 14:33:32 +00:00
l . Infof ( "Shared %s with %s due to auto-accept" , folder . ID , deviceID )
2020-05-27 06:05:26 +00:00
return cfg , true
2017-12-07 07:08:24 +00:00
}
}
2019-11-21 07:41:41 +00:00
func ( m * model ) introduceDevice ( device protocol . Device , introducerCfg config . DeviceConfiguration ) config . DeviceConfiguration {
2016-11-07 16:40:48 +00:00
addresses := [ ] string { "dynamic" }
for _ , addr := range device . Addresses {
if addr != "dynamic" {
addresses = append ( addresses , addr )
}
}
l . Infof ( "Adding device %v to config (vouched for by introducer %v)" , device . ID , introducerCfg . DeviceID )
2021-02-04 20:10:41 +00:00
newDeviceCfg := m . cfg . DefaultDevice ( )
newDeviceCfg . DeviceID = device . ID
newDeviceCfg . Name = device . Name
newDeviceCfg . Compression = introducerCfg . Compression
newDeviceCfg . Addresses = addresses
newDeviceCfg . CertName = device . CertName
newDeviceCfg . IntroducedBy = introducerCfg . DeviceID
2016-11-07 16:40:48 +00:00
// The introducers' introducers are also our introducers.
if device . Introducer {
l . Infof ( "Device %v is now also an introducer" , device . ID )
newDeviceCfg . Introducer = true
newDeviceCfg . SkipIntroductionRemovals = device . SkipIntroductionRemovals
2014-09-23 14:04:20 +00:00
}
2016-11-07 16:40:48 +00:00
2019-11-21 07:41:41 +00:00
return newDeviceCfg
2016-11-07 16:40:48 +00:00
}
2016-08-10 09:37:32 +00:00
// Closed is called when a connection has been closed
2023-07-29 08:24:44 +00:00
func ( m * model ) Closed ( conn protocol . Connection , err error ) {
2023-09-06 10:52:01 +00:00
connID := conn . ConnectionID ( )
deviceID := conn . DeviceID ( )
2014-07-15 11:04:37 +00:00
m . pmut . Lock ( )
2023-09-06 10:52:01 +00:00
conn , ok := m . connections [ connID ]
2019-05-02 12:24:55 +00:00
if ! ok {
2019-08-16 07:35:19 +00:00
m . pmut . Unlock ( )
2019-05-02 12:24:55 +00:00
return
2013-12-31 02:21:57 +00:00
}
2021-01-05 16:45:07 +00:00
2023-09-06 10:52:01 +00:00
closed := m . closed [ connID ]
delete ( m . closed , connID )
delete ( m . connections , connID )
removedIsPrimary := m . promotedConnID [ deviceID ] == connID
remainingConns := without ( m . deviceConnIDs [ deviceID ] , connID )
var wait <- chan error
if removedIsPrimary {
m . progressEmitter . temporaryIndexUnsubscribe ( conn )
if idxh , ok := m . indexHandlers . Get ( deviceID ) ; ok && idxh . conn . ConnectionID ( ) == connID {
wait = m . indexHandlers . RemoveAndWaitChan ( deviceID , 0 )
}
m . scheduleConnectionPromotion ( )
}
if len ( remainingConns ) == 0 {
// All device connections closed
delete ( m . deviceConnIDs , deviceID )
delete ( m . promotedConnID , deviceID )
delete ( m . connRequestLimiters , deviceID )
delete ( m . helloMessages , deviceID )
delete ( m . remoteFolderStates , deviceID )
delete ( m . deviceDownloads , deviceID )
} else {
// Some connections remain
m . deviceConnIDs [ deviceID ] = remainingConns
}
2019-08-16 07:35:19 +00:00
m . pmut . Unlock ( )
2023-09-06 10:52:01 +00:00
if wait != nil {
<- wait
}
m . fmut . RLock ( )
m . deviceDidCloseFRLocked ( deviceID , time . Since ( conn . EstablishedAt ( ) ) )
m . fmut . RUnlock ( )
2019-08-16 07:35:19 +00:00
2023-09-06 10:52:01 +00:00
k := map [ bool ] string { false : "secondary" , true : "primary" } [ removedIsPrimary ]
l . Infof ( "Lost %s connection to %s at %s: %v (%d remain)" , k , deviceID . Short ( ) , conn , err , len ( remainingConns ) )
2016-08-10 09:37:32 +00:00
2023-09-06 10:52:01 +00:00
if len ( remainingConns ) == 0 {
l . Infof ( "Connection to %s at %s closed: %v" , deviceID . Short ( ) , conn , err )
m . evLogger . Log ( events . DeviceDisconnected , map [ string ] string {
"id" : deviceID . String ( ) ,
"error" : err . Error ( ) ,
} )
}
2016-08-10 09:37:32 +00:00
close ( closed )
}
2018-11-13 07:53:55 +00:00
// Implements protocol.RequestResponse
type requestResponse struct {
data [ ] byte
closed chan struct { }
once stdsync . Once
}
func newRequestResponse ( size int ) * requestResponse {
return & requestResponse {
data : protocol . BufferPool . Get ( size ) ,
closed : make ( chan struct { } ) ,
2015-01-18 01:12:06 +00:00
}
2018-11-13 07:53:55 +00:00
}
2015-01-18 01:12:06 +00:00
2018-11-13 07:53:55 +00:00
func ( r * requestResponse ) Data ( ) [ ] byte {
return r . data
}
func ( r * requestResponse ) Close ( ) {
r . once . Do ( func ( ) {
protocol . BufferPool . Put ( r . data )
close ( r . closed )
} )
}
func ( r * requestResponse ) Wait ( ) {
<- r . closed
}
// Request returns the specified data segment by reading it from local disk.
// Implements the protocol.Model interface.
2023-07-29 08:24:44 +00:00
func ( m * model ) Request ( conn protocol . Connection , folder , name string , _ , size int32 , offset int64 , hash [ ] byte , weakHash uint32 , fromTemporary bool ) ( out protocol . RequestResponse , err error ) {
2018-11-13 07:53:55 +00:00
if size < 0 || offset < 0 {
return nil , protocol . ErrInvalid
2018-04-09 19:55:52 +00:00
}
2023-07-29 08:24:44 +00:00
deviceID := conn . DeviceID ( )
2018-08-15 14:33:03 +00:00
m . fmut . RLock ( )
folderCfg , ok := m . folderCfgs [ folder ]
2018-04-09 19:55:52 +00:00
folderIgnores := m . folderIgnores [ folder ]
m . fmut . RUnlock ( )
2018-08-15 14:33:03 +00:00
if ! ok {
// The folder might be already unpaused in the config, but not yet
// in the model.
2023-09-06 10:52:01 +00:00
l . Debugf ( "Request from %s for file %s in unstarted folder %q" , deviceID . Short ( ) , name , folder )
2019-04-10 09:47:24 +00:00
return nil , protocol . ErrGeneric
2018-11-13 07:53:55 +00:00
}
if ! folderCfg . SharedWith ( deviceID ) {
2023-09-06 10:52:01 +00:00
l . Warnf ( "Request from %s for file %s in unshared folder %q" , deviceID . Short ( ) , name , folder )
2019-04-10 09:47:24 +00:00
return nil , protocol . ErrGeneric
2018-11-13 07:53:55 +00:00
}
if folderCfg . Paused {
2023-09-06 10:52:01 +00:00
l . Debugf ( "Request from %s for file %s in paused folder %q" , deviceID . Short ( ) , name , folder )
2019-04-10 09:47:24 +00:00
return nil , protocol . ErrGeneric
2018-08-15 14:33:03 +00:00
}
2018-04-09 19:55:52 +00:00
2018-06-06 21:34:11 +00:00
// Make sure the path is valid and in canonical form
if name , err = fs . Canonicalize ( name ) ; err != nil {
2023-09-06 10:52:01 +00:00
l . Debugf ( "Request from %s in folder %q for invalid filename %s" , deviceID . Short ( ) , folder , name )
2019-04-10 09:47:24 +00:00
return nil , protocol . ErrGeneric
2018-04-09 19:55:52 +00:00
}
2015-10-03 15:25:21 +00:00
if deviceID != protocol . LocalDeviceID {
2023-09-06 10:52:01 +00:00
l . Debugf ( "%v REQ(in): %s: %q / %q o=%d s=%d t=%v" , m , deviceID . Short ( ) , folder , name , offset , size , fromTemporary )
2013-12-15 10:43:31 +00:00
}
2014-11-09 04:26:52 +00:00
2017-09-02 05:52:38 +00:00
if fs . IsInternal ( name ) {
2023-09-06 10:52:01 +00:00
l . Debugf ( "%v REQ(in) for internal file: %s: %q / %q o=%d s=%d" , m , deviceID . Short ( ) , folder , name , offset , size )
2019-04-10 09:47:24 +00:00
return nil , protocol . ErrInvalid
2016-12-01 14:00:11 +00:00
}
if folderIgnores . Match ( name ) . IsIgnored ( ) {
2023-09-06 10:52:01 +00:00
l . Debugf ( "%v REQ(in) for ignored file: %s: %q / %q o=%d s=%d" , m , deviceID . Short ( ) , folder , name , offset , size )
2019-04-10 09:47:24 +00:00
return nil , protocol . ErrInvalid
2015-10-13 13:59:31 +00:00
}
2018-11-13 07:53:55 +00:00
// Restrict parallel requests by connection/device
m . pmut . RLock ( )
limiter := m . connRequestLimiters [ deviceID ]
m . pmut . RUnlock ( )
2020-02-01 07:02:18 +00:00
// The requestResponse releases the bytes to the buffer pool and the
// limiters when its Close method is called.
res := newLimitedRequestResponse ( int ( size ) , limiter , m . globalRequestLimiter )
2018-11-13 07:53:55 +00:00
defer func ( ) {
// Close it ourselves if it isn't returned due to an error
if err != nil {
res . Close ( )
}
} ( )
2021-01-12 15:22:21 +00:00
// Grab the FS after limiting, as it causes I/O and we want to minimize
// the race time between the symlink check and the read.
2022-04-10 18:55:05 +00:00
folderFs := folderCfg . Filesystem ( nil )
2021-01-12 15:22:21 +00:00
if err := osutil . TraversesSymlink ( folderFs , filepath . Dir ( name ) ) ; err != nil {
2023-09-06 10:52:01 +00:00
l . Debugf ( "%v REQ(in) traversal check: %s - %s: %q / %q o=%d s=%d" , m , err , deviceID . Short ( ) , folder , name , offset , size )
2021-01-12 15:22:21 +00:00
return nil , protocol . ErrNoSuchFile
}
2016-04-15 10:59:41 +00:00
// Only check temp files if the flag is set, and if we are set to advertise
// the temp indexes.
2016-07-04 10:40:29 +00:00
if fromTemporary && ! folderCfg . DisableTempIndexes {
2017-09-02 05:52:38 +00:00
tempFn := fs . TempName ( name )
2016-12-13 10:24:10 +00:00
2017-08-19 14:36:56 +00:00
if info , err := folderFs . Lstat ( tempFn ) ; err != nil || ! info . IsRegular ( ) {
2016-12-13 10:24:10 +00:00
// Reject reads for anything that doesn't exist or is something
// other than a regular file.
2023-09-06 10:52:01 +00:00
l . Debugf ( "%v REQ(in) failed stating temp file (%v): %s: %q / %q o=%d s=%d" , m , err , deviceID . Short ( ) , folder , name , offset , size )
2018-11-13 07:53:55 +00:00
return nil , protocol . ErrNoSuchFile
2016-12-13 10:24:10 +00:00
}
2021-02-05 15:07:21 +00:00
_ , err := readOffsetIntoBuf ( folderFs , tempFn , offset , res . data )
2018-11-13 07:53:55 +00:00
if err == nil && scanner . Validate ( res . data , hash , weakHash ) {
return res , nil
2016-04-15 10:59:41 +00:00
}
2023-03-21 07:07:28 +00:00
// Fall through to reading from a non-temp file, just in case the temp
2016-04-15 10:59:41 +00:00
// file has finished downloading.
2013-12-15 10:43:31 +00:00
}
2017-08-19 14:36:56 +00:00
if info , err := folderFs . Lstat ( name ) ; err != nil || ! info . IsRegular ( ) {
2016-12-13 10:24:10 +00:00
// Reject reads for anything that doesn't exist or is something
// other than a regular file.
2023-09-06 10:52:01 +00:00
l . Debugf ( "%v REQ(in) failed stating file (%v): %s: %q / %q o=%d s=%d" , m , err , deviceID . Short ( ) , folder , name , offset , size )
2018-11-13 07:53:55 +00:00
return nil , protocol . ErrNoSuchFile
2016-12-13 10:24:10 +00:00
}
2021-02-05 15:07:21 +00:00
n , err := readOffsetIntoBuf ( folderFs , name , offset , res . data )
if fs . IsNotExist ( err ) {
2023-09-06 10:52:01 +00:00
l . Debugf ( "%v REQ(in) file doesn't exist: %s: %q / %q o=%d s=%d" , m , deviceID . Short ( ) , folder , name , offset , size )
2018-11-13 07:53:55 +00:00
return nil , protocol . ErrNoSuchFile
2021-02-05 15:07:21 +00:00
} else if err == io . EOF {
// Read beyond end of file. This might indicate a problem, or it
// might be a short block that gets padded when read for encrypted
// folders. We ignore the error and let the hash validation in the
// next step take care of it, by only hashing the part we actually
// managed to read.
2016-04-15 10:59:41 +00:00
} else if err != nil {
2023-09-06 10:52:01 +00:00
l . Debugf ( "%v REQ(in) failed reading file (%v): %s: %q / %q o=%d s=%d" , m , err , deviceID . Short ( ) , folder , name , offset , size )
2018-11-13 07:53:55 +00:00
return nil , protocol . ErrGeneric
2013-12-15 10:43:31 +00:00
}
2018-04-09 19:55:52 +00:00
2021-05-22 19:38:49 +00:00
if folderCfg . Type != config . FolderTypeReceiveEncrypted && len ( hash ) > 0 && ! scanner . Validate ( res . data [ : n ] , hash , weakHash ) {
2020-06-26 14:47:03 +00:00
m . recheckFile ( deviceID , folder , name , offset , hash , weakHash )
2023-09-06 10:52:01 +00:00
l . Debugf ( "%v REQ(in) failed validating data: %s: %q / %q o=%d s=%d" , m , deviceID . Short ( ) , folder , name , offset , size )
2018-11-13 07:53:55 +00:00
return nil , protocol . ErrNoSuchFile
2018-05-05 08:24:44 +00:00
}
2018-11-13 07:53:55 +00:00
return res , nil
2013-12-15 10:43:31 +00:00
}
2020-02-01 07:02:18 +00:00
// newLimitedRequestResponse takes size bytes from the limiters in order,
// skipping nil limiters, then returns a requestResponse of the given size.
// When the requestResponse is closed the limiters are given back the bytes,
// in reverse order.
2023-08-21 17:44:33 +00:00
func newLimitedRequestResponse ( size int , limiters ... * semaphore . Semaphore ) * requestResponse {
multi := semaphore . MultiSemaphore ( limiters )
2021-06-25 09:38:04 +00:00
multi . Take ( size )
2020-02-01 07:02:18 +00:00
res := newRequestResponse ( size )
go func ( ) {
res . Wait ( )
2021-06-25 09:38:04 +00:00
multi . Give ( size )
2020-02-01 07:02:18 +00:00
} ( )
return res
}
2020-06-26 14:47:03 +00:00
func ( m * model ) recheckFile ( deviceID protocol . DeviceID , folder , name string , offset int64 , hash [ ] byte , weakHash uint32 ) {
2021-03-07 12:43:22 +00:00
cf , ok , err := m . CurrentFolderFile ( folder , name )
if err != nil {
l . Debugf ( "%v recheckFile: %s: %q / %q: current file error: %v" , m , deviceID , folder , name , err )
return
}
2018-05-05 08:24:44 +00:00
if ! ok {
l . Debugf ( "%v recheckFile: %s: %q / %q: no current file" , m , deviceID , folder , name )
return
}
if cf . IsDeleted ( ) || cf . IsInvalid ( ) || cf . IsSymlink ( ) || cf . IsDirectory ( ) {
l . Debugf ( "%v recheckFile: %s: %q / %q: not a regular file" , m , deviceID , folder , name )
return
}
2019-07-12 14:37:12 +00:00
blockIndex := int ( offset / int64 ( cf . BlockSize ( ) ) )
2018-06-13 17:07:52 +00:00
if blockIndex >= len ( cf . Blocks ) {
2018-05-05 08:24:44 +00:00
l . Debugf ( "%v recheckFile: %s: %q / %q i=%d: block index too far" , m , deviceID , folder , name , blockIndex )
return
}
block := cf . Blocks [ blockIndex ]
// Seems to want a different version of the file, whatever.
if ! bytes . Equal ( block . Hash , hash ) {
l . Debugf ( "%v recheckFile: %s: %q / %q i=%d: hash mismatch %x != %x" , m , deviceID , folder , name , blockIndex , block . Hash , hash )
return
}
2020-06-26 14:47:03 +00:00
if weakHash != 0 && block . WeakHash != weakHash {
l . Debugf ( "%v recheckFile: %s: %q / %q i=%d: weak hash mismatch %v != %v" , m , deviceID , folder , name , blockIndex , block . WeakHash , weakHash )
return
}
2018-05-05 08:24:44 +00:00
// The hashes provided part of the request match what we expect to find according
// to what we have in the database, yet the content we've read off the filesystem doesn't
// Something is fishy, invalidate the file and rescan it.
// The file will temporarily become invalid, which is ok as the content is messed up.
2019-08-11 14:10:30 +00:00
m . fmut . RLock ( )
2023-09-02 14:42:46 +00:00
runner , ok := m . folderRunners . Get ( folder )
2019-08-11 14:10:30 +00:00
m . fmut . RUnlock ( )
2019-04-07 11:29:17 +00:00
if ! ok {
l . Debugf ( "%v recheckFile: %s: %q / %q: Folder stopped before rescan could be scheduled" , m , deviceID , folder , name )
return
}
2020-05-01 09:08:59 +00:00
runner . ScheduleForceRescan ( name )
2019-04-07 11:29:17 +00:00
l . Debugf ( "%v recheckFile: %s: %q / %q" , m , deviceID , folder , name )
2018-05-05 08:24:44 +00:00
}
2021-03-07 12:43:22 +00:00
func ( m * model ) CurrentFolderFile ( folder string , file string ) ( protocol . FileInfo , bool , error ) {
2014-09-28 11:39:39 +00:00
m . fmut . RLock ( )
2015-04-18 13:41:47 +00:00
fs , ok := m . folderFiles [ folder ]
2014-09-28 11:39:39 +00:00
m . fmut . RUnlock ( )
2015-04-18 13:41:47 +00:00
if ! ok {
2021-03-07 12:43:22 +00:00
return protocol . FileInfo { } , false , ErrFolderMissing
2015-04-18 13:41:47 +00:00
}
2021-03-07 12:43:22 +00:00
snap , err := fs . Snapshot ( )
if err != nil {
return protocol . FileInfo { } , false , err
}
f , ok := snap . Get ( protocol . LocalDeviceID , file )
snap . Release ( )
return f , ok , nil
2014-04-01 21:18:32 +00:00
}
2021-03-07 12:43:22 +00:00
func ( m * model ) CurrentGlobalFile ( folder string , file string ) ( protocol . FileInfo , bool , error ) {
2014-09-28 11:39:39 +00:00
m . fmut . RLock ( )
2021-05-03 10:28:25 +00:00
ffs , ok := m . folderFiles [ folder ]
2014-09-28 11:39:39 +00:00
m . fmut . RUnlock ( )
2015-04-18 13:41:47 +00:00
if ! ok {
2021-03-07 12:43:22 +00:00
return protocol . FileInfo { } , false , ErrFolderMissing
2015-04-18 13:41:47 +00:00
}
2021-05-03 10:28:25 +00:00
snap , err := ffs . Snapshot ( )
2021-03-07 12:43:22 +00:00
if err != nil {
return protocol . FileInfo { } , false , err
}
f , ok := snap . GetGlobal ( file )
snap . Release ( )
return f , ok , nil
2014-04-01 21:18:32 +00:00
}
2021-05-03 10:28:25 +00:00
func ( m * model ) GetMtimeMapping ( folder string , file string ) ( fs . MtimeMapping , error ) {
m . fmut . RLock ( )
ffs , ok := m . folderFiles [ folder ]
2022-01-31 09:12:52 +00:00
fcfg := m . folderCfgs [ folder ]
2021-05-03 10:28:25 +00:00
m . fmut . RUnlock ( )
if ! ok {
return fs . MtimeMapping { } , ErrFolderMissing
}
2022-04-10 18:55:05 +00:00
return fs . GetMtimeMapping ( fcfg . Filesystem ( ffs ) , file )
2021-05-03 10:28:25 +00:00
}
2023-09-06 10:52:01 +00:00
// Connection returns if we are connected to the given device.
func ( m * model ) ConnectedTo ( deviceID protocol . DeviceID ) bool {
2014-01-18 03:06:44 +00:00
m . pmut . RLock ( )
2023-09-06 10:52:01 +00:00
_ , ok := m . deviceConnIDs [ deviceID ]
2014-09-20 17:14:45 +00:00
m . pmut . RUnlock ( )
2023-09-06 10:52:01 +00:00
return ok
2014-01-06 10:11:18 +00:00
}
2021-01-12 15:25:21 +00:00
// LoadIgnores loads or refreshes the ignore patterns from disk, if the
// folder is healthy, and returns the refreshed lines and patterns.
func ( m * model ) LoadIgnores ( folder string ) ( [ ] string , [ ] string , error ) {
2014-11-03 21:02:55 +00:00
m . fmut . RLock ( )
2019-05-02 17:55:39 +00:00
cfg , cfgOk := m . folderCfgs [ folder ]
ignores , ignoresOk := m . folderIgnores [ folder ]
m . fmut . RUnlock ( )
2017-08-12 17:10:43 +00:00
2019-05-02 17:55:39 +00:00
if ! cfgOk {
2020-11-24 21:20:50 +00:00
cfg , cfgOk = m . cfg . Folder ( folder )
2019-05-02 17:55:39 +00:00
if ! cfgOk {
2020-03-03 21:40:00 +00:00
return nil , nil , fmt . Errorf ( "folder %s does not exist" , folder )
2017-04-01 09:58:06 +00:00
}
2017-08-22 06:48:25 +00:00
}
2014-09-14 22:03:53 +00:00
2021-02-12 21:51:29 +00:00
if cfg . Type == config . FolderTypeReceiveEncrypted {
return nil , nil , nil
}
2019-05-02 17:55:39 +00:00
if ! ignoresOk {
2022-04-10 18:55:05 +00:00
ignores = ignore . New ( cfg . Filesystem ( nil ) )
2014-09-14 22:03:53 +00:00
}
2020-06-18 09:04:00 +00:00
err := ignores . Load ( ".stignore" )
if fs . IsNotExist ( err ) {
// Having no ignores is not an error.
return nil , nil , nil
2014-09-14 22:03:53 +00:00
}
2020-06-18 09:04:00 +00:00
// Return lines and patterns, which may have some meaning even when err
// != nil, depending on the specific error.
return ignores . Lines ( ) , ignores . Patterns ( ) , err
2014-09-14 22:03:53 +00:00
}
2021-01-12 15:25:21 +00:00
// CurrentIgnores returns the currently loaded set of ignore patterns,
// whichever it may be. No attempt is made to load or refresh ignore
// patterns from disk.
func ( m * model ) CurrentIgnores ( folder string ) ( [ ] string , [ ] string , error ) {
m . fmut . RLock ( )
_ , cfgOk := m . folderCfgs [ folder ]
ignores , ignoresOk := m . folderIgnores [ folder ]
m . fmut . RUnlock ( )
if ! cfgOk {
return nil , nil , fmt . Errorf ( "folder %s does not exist" , folder )
}
if ! ignoresOk {
// Empty ignore patterns
return [ ] string { } , [ ] string { } , nil
}
return ignores . Lines ( ) , ignores . Patterns ( ) , nil
}
2019-02-26 08:09:25 +00:00
func ( m * model ) SetIgnores ( folder string , content [ ] string ) error {
2020-11-24 21:20:50 +00:00
cfg , ok := m . cfg . Folder ( folder )
2014-09-14 22:03:53 +00:00
if ! ok {
2018-05-05 08:30:39 +00:00
return fmt . Errorf ( "folder %s does not exist" , cfg . Description ( ) )
}
2022-01-13 22:38:21 +00:00
return m . setIgnores ( cfg , content )
}
2018-05-05 08:30:39 +00:00
2022-01-13 22:38:21 +00:00
func ( m * model ) setIgnores ( cfg config . FolderConfiguration , content [ ] string ) error {
2018-05-05 08:30:39 +00:00
err := cfg . CheckPath ( )
if err == config . ErrPathMissing {
if err = cfg . CreateRoot ( ) ; err != nil {
2022-08-16 08:01:49 +00:00
return fmt . Errorf ( "failed to create folder root: %w" , err )
2018-05-05 08:30:39 +00:00
}
err = cfg . CheckPath ( )
}
if err != nil && err != config . ErrMarkerMissing {
return err
2014-09-14 22:03:53 +00:00
}
2022-04-10 18:55:05 +00:00
if err := ignore . WriteIgnores ( cfg . Filesystem ( nil ) , ".stignore" , content ) ; err != nil {
2014-09-14 22:03:53 +00:00
l . Warnln ( "Saving .stignore:" , err )
return err
}
2017-04-01 09:58:06 +00:00
m . fmut . RLock ( )
2023-09-02 14:42:46 +00:00
runner , ok := m . folderRunners . Get ( cfg . ID )
2017-04-01 09:58:06 +00:00
m . fmut . RUnlock ( )
if ok {
2021-08-17 07:23:33 +00:00
runner . ScheduleScan ( )
2014-09-14 22:03:53 +00:00
}
2017-04-01 09:58:06 +00:00
return nil
2014-09-14 22:03:53 +00:00
}
2016-03-25 20:29:07 +00:00
// OnHello is called when an device connects to us.
// This allows us to extract some information from the Hello message
// and add it to a list of known devices ahead of any checks.
2020-09-29 11:17:38 +00:00
func ( m * model ) OnHello ( remoteID protocol . DeviceID , addr net . Addr , hello protocol . Hello ) error {
2022-04-07 15:35:33 +00:00
if _ , ok := m . cfg . Device ( remoteID ) ; ! ok {
2020-12-17 18:54:31 +00:00
if err := m . db . AddOrUpdatePendingDevice ( remoteID , hello . DeviceName , addr . String ( ) ) ; err != nil {
l . Warnf ( "Failed to persist pending device entry to database: %v" , err )
}
lib/model: Forget pending folders no longer announced in ClusterConfig (fixes #5187) (#7205)
* lib/db: Add ExpirePendingFolders().
Use-case is to drop any no-longer-pending folders for a specific
device when parsing its ClusterConfig message where previously offered
folders are not mentioned any more.
The timestamp in ObservedFolder is stored with only second precision,
so round to seconds here as well. This allows calling the function
within the same second of adding or updating entries.
* lib/model: Weed out pending folders when receiving ClusterConfig.
Filter the entries by timestamp, which must be newer than or equal to
the reception time of the ClusterConfig. For just mentioned ones,
this assumption will hold as AddOrUpdatePendingFolder() updates the
timestamp.
* lib/model, gui: Notify when one or more pending folders expired.
Introduce new event type FolderOfferCancelled and use it to trigger a
complete refreshCluster() cycle. Listing individual entries would be
much more code and probably just as much work to answer the API
request.
* lib/model: Add comment and rename ExpirePendingFolders().
* lib/events: Rename FolderOfferCancelled to ClusterPendingChanged.
* lib/model: Reuse ClusterPendingChanged event for cleanPending()
Changing the config does not necessarily mean that the
/resut/cluster/pending endpoints need to be refreshed, but only if
something was actually removed. Detect this and indicate it through
the ClusterPendingChanged event, which is already hooked up to requery
respective endpoints within the GUI.
No more need for a separate refreshCluster() in reaction to
ConfigSaved event or calling refreshConfig().
* lib/model: Gofmt.
* lib/db: Warn instead of info log for failed removal.
* gui: Fix pending notifications not loading on GUI start.
* lib/db: Use short device ID in log message.
* lib/db: Return list of expired folder IDs after deleting them.
* lib/model: Refactor Pending...Changed events.
* lib/model: Adjust format of removed pending folders enumeration.
Use an array of objects with device / folder ID properties, matching
the other places where it's used.
* lib/db: Drop invalid entries in RemovePendingFoldersBeforeTime().
* lib/model: Gofmt.
My local gofmt did not complain here, strangely...
* gui: Handle PendingDevicesChanged event.
Even though it currently only holds one device at a time, wrap the
contents in an array under the "added" property name.
* lib/model: Fix null values in PendingFoldersChanged removed member.
* gui: Handle PendingFoldersChanged event.
* lib/model: Simplify construction of expiredPendingList.
* lib/model: Reduce code duplication in cleanPending().
Use goto and a label for the common parts of calling the DB removal
function and building the event data part.
* lib/events, gui: Mark ...Rejected events deprecated.
Extend comments explaining the conditions when the replacement event
types are emitted.
* lib/model: Wrap removed devices in array of objects as well.
* lib/db: Use iter.Value() instead of needless db.Get(iter.Key())
* lib/db: Add comment explaining RemovePendingFoldersBeforeTime().
* lib/model: Rename fields folderID and deviceID in event data.
* lib/db: Only list actually expired IDs as removed.
Skip entries where Delete() failed as well as invalid entries that got
removed automatically.
* lib/model: Gofmt
2021-01-25 10:58:10 +00:00
m . evLogger . Log ( events . PendingDevicesChanged , map [ string ] [ ] interface { } {
"added" : { map [ string ] string {
"deviceID" : remoteID . String ( ) ,
"name" : hello . DeviceName ,
"address" : addr . String ( ) ,
} } ,
} )
// DEPRECATED: Only for backwards compatibility, should be removed.
2019-08-15 14:29:37 +00:00
m . evLogger . Log ( events . DeviceRejected , map [ string ] string {
2017-04-01 09:52:31 +00:00
"name" : hello . DeviceName ,
"device" : remoteID . String ( ) ,
"address" : addr . String ( ) ,
} )
return errDeviceUnknown
2016-03-25 20:29:07 +00:00
}
2017-04-01 09:52:31 +00:00
return nil
2016-03-25 20:29:07 +00:00
}
2014-01-06 10:11:18 +00:00
// AddConnection adds a new peer connection to the model. An initial index will
// be sent to the connected peer, thereafter index updates whenever the local
2014-09-28 11:00:38 +00:00
// folder changes.
2020-12-21 10:40:51 +00:00
func ( m * model ) AddConnection ( conn protocol . Connection , hello protocol . Hello ) {
2023-07-29 08:24:44 +00:00
deviceID := conn . DeviceID ( )
2023-09-06 10:52:01 +00:00
deviceCfg , ok := m . cfg . Device ( deviceID )
2018-11-13 07:53:55 +00:00
if ! ok {
l . Infoln ( "Trying to add connection to unknown device" )
return
}
2014-07-15 11:04:37 +00:00
2023-09-06 10:52:01 +00:00
connID := conn . ConnectionID ( )
2020-10-21 09:51:53 +00:00
closed := make ( chan struct { } )
2014-01-06 10:11:18 +00:00
2023-09-06 10:52:01 +00:00
m . pmut . Lock ( )
m . connections [ connID ] = conn
m . closed [ connID ] = closed
2016-03-25 20:29:07 +00:00
m . helloMessages [ deviceID ] = hello
2023-09-06 10:52:01 +00:00
m . deviceConnIDs [ deviceID ] = append ( m . deviceConnIDs [ deviceID ] , connID )
if m . deviceDownloads [ deviceID ] == nil {
m . deviceDownloads [ deviceID ] = newDeviceDownloadState ( )
}
2016-03-25 20:29:07 +00:00
event := map [ string ] string {
"id" : deviceID . String ( ) ,
"deviceName" : hello . DeviceName ,
"clientName" : hello . ClientName ,
"clientVersion" : hello . ClientVersion ,
2016-11-30 07:54:20 +00:00
"type" : conn . Type ( ) ,
2016-03-25 20:29:07 +00:00
}
addr := conn . RemoteAddr ( )
if addr != nil {
event [ "addr" ] = addr . String ( )
}
2019-08-15 14:29:37 +00:00
m . evLogger . Log ( events . DeviceConnected , event )
2016-03-25 20:29:07 +00:00
2023-09-06 10:52:01 +00:00
if len ( m . deviceConnIDs [ deviceID ] ) == 1 {
l . Infof ( ` Device %s client is "%s %s" named "%s" at %s ` , deviceID . Short ( ) , hello . ClientName , hello . ClientVersion , hello . DeviceName , conn )
} else {
l . Infof ( ` Additional connection (+%d) for device %s at %s ` , len ( m . deviceConnIDs [ deviceID ] ) - 1 , deviceID . Short ( ) , conn )
}
2016-03-25 20:29:07 +00:00
2016-12-21 12:22:18 +00:00
m . pmut . Unlock ( )
2015-07-10 06:37:57 +00:00
2023-09-06 10:52:01 +00:00
if ( deviceCfg . Name == "" || m . cfg . Options ( ) . OverwriteRemoteDevNames ) && hello . DeviceName != "" {
2021-01-15 14:43:34 +00:00
m . cfg . Modify ( func ( cfg * config . Configuration ) {
for i := range cfg . Devices {
if cfg . Devices [ i ] . DeviceID == deviceID {
if cfg . Devices [ i ] . Name == "" || cfg . Options . OverwriteRemoteDevNames {
cfg . Devices [ i ] . Name = hello . DeviceName
}
return
}
}
} )
2016-04-18 20:25:31 +00:00
}
2014-09-28 11:00:38 +00:00
m . deviceWasSeen ( deviceID )
2023-09-06 10:52:01 +00:00
m . scheduleConnectionPromotion ( )
}
func ( m * model ) scheduleConnectionPromotion ( ) {
// Keeps deferring to prevent multiple executions in quick succession,
// e.g. if multiple connections to a single device are closed.
m . promotionTimer . Reset ( time . Second )
}
// promoteConnections checks for devices that have connections, but where
// the primary connection hasn't started index handlers etc. yet, and
// promotes the primary connection to be the index handling one. This should
// be called after adding new connections, and after closing a primary
// device connection.
func ( m * model ) promoteConnections ( ) {
m . fmut . RLock ( ) // for generateClusterConfigFRLocked
defer m . fmut . RUnlock ( )
m . pmut . Lock ( ) // for most other things
defer m . pmut . Unlock ( )
for deviceID , connIDs := range m . deviceConnIDs {
cm , passwords := m . generateClusterConfigFRLocked ( deviceID )
if m . promotedConnID [ deviceID ] != connIDs [ 0 ] {
// The previously promoted connection is not the current
// primary; we should promote the primary connection to be the
// index handling one. We do this by sending a ClusterConfig on
// it, which will cause the other side to start sending us index
// messages there. (On our side, we manage index handlers based
// on where we get ClusterConfigs from the peer.)
conn := m . connections [ connIDs [ 0 ] ]
l . Debugf ( "Promoting connection to %s at %s" , deviceID . Short ( ) , conn )
if conn . Statistics ( ) . StartedAt . IsZero ( ) {
conn . SetFolderPasswords ( passwords )
conn . Start ( )
}
conn . ClusterConfig ( cm )
m . promotedConnID [ deviceID ] = connIDs [ 0 ]
}
// Make sure any other new connections also get started, and that
// they get a secondary-marked ClusterConfig.
for _ , connID := range connIDs [ 1 : ] {
conn := m . connections [ connID ]
if conn . Statistics ( ) . StartedAt . IsZero ( ) {
conn . SetFolderPasswords ( passwords )
conn . Start ( )
conn . ClusterConfig ( protocol . ClusterConfig { Secondary : true } )
}
}
}
2014-09-20 17:14:45 +00:00
}
2023-07-29 08:24:44 +00:00
func ( m * model ) DownloadProgress ( conn protocol . Connection , folder string , updates [ ] protocol . FileDownloadProgressUpdate ) error {
2023-09-06 10:52:01 +00:00
deviceID := conn . DeviceID ( )
2016-04-15 10:59:41 +00:00
m . fmut . RLock ( )
cfg , ok := m . folderCfgs [ folder ]
m . fmut . RUnlock ( )
2023-09-06 10:52:01 +00:00
if ! ok || cfg . DisableTempIndexes || ! cfg . SharedWith ( deviceID ) {
2019-12-04 09:46:55 +00:00
return nil
2016-04-15 10:59:41 +00:00
}
m . pmut . RLock ( )
2023-09-06 10:52:01 +00:00
downloads := m . deviceDownloads [ deviceID ]
2016-04-15 10:59:41 +00:00
m . pmut . RUnlock ( )
2019-08-13 07:04:43 +00:00
downloads . Update ( folder , updates )
state := downloads . GetBlockCounts ( folder )
2016-05-22 07:52:08 +00:00
2019-08-15 14:29:37 +00:00
m . evLogger . Log ( events . RemoteDownloadProgress , map [ string ] interface { } {
2023-09-06 10:52:01 +00:00
"device" : deviceID . String ( ) ,
2016-05-22 07:52:08 +00:00
"folder" : folder ,
2016-05-26 06:53:27 +00:00
"state" : state ,
2016-05-22 07:52:08 +00:00
} )
2019-12-04 09:46:55 +00:00
return nil
2016-04-15 10:59:41 +00:00
}
2019-02-26 08:09:25 +00:00
func ( m * model ) deviceWasSeen ( deviceID protocol . DeviceID ) {
2019-09-12 04:55:24 +00:00
m . fmut . RLock ( )
sr , ok := m . deviceStatRefs [ deviceID ]
m . fmut . RUnlock ( )
if ok {
2021-01-05 16:45:07 +00:00
_ = sr . WasSeen ( )
}
}
2023-09-06 10:52:01 +00:00
func ( m * model ) deviceDidCloseFRLocked ( deviceID protocol . DeviceID , duration time . Duration ) {
if sr , ok := m . deviceStatRefs [ deviceID ] ; ok {
2021-01-05 16:45:07 +00:00
_ = sr . LastConnectionDuration ( duration )
2019-09-12 04:55:24 +00:00
}
2014-07-15 11:04:37 +00:00
}
2020-11-09 14:33:32 +00:00
func ( m * model ) requestGlobal ( ctx context . Context , deviceID protocol . DeviceID , folder , name string , blockNo int , offset int64 , size int , hash [ ] byte , weakHash uint32 , fromTemporary bool ) ( [ ] byte , error ) {
2023-09-06 10:52:01 +00:00
conn , connOK := m . requestConnectionForDevice ( deviceID )
if ! connOK {
return nil , fmt . Errorf ( "requestGlobal: no connection to device: %s" , deviceID . Short ( ) )
}
l . Debugf ( "%v REQ(out): %s (%s): %q / %q b=%d o=%d s=%d h=%x wh=%x ft=%t" , m , deviceID . Short ( ) , conn , folder , name , blockNo , offset , size , hash , weakHash , fromTemporary )
return conn . Request ( ctx , folder , name , blockNo , offset , size , hash , weakHash , fromTemporary )
}
// requestConnectionForDevice returns a connection to the given device, to
// be used for sending a request. If there is only one device connection,
// this is the one to use. If there are multiple then we avoid the first
// ("primary") connection, which is dedicated to index data, and pick a
// random one of the others.
func ( m * model ) requestConnectionForDevice ( deviceID protocol . DeviceID ) ( protocol . Connection , bool ) {
2014-01-18 03:06:44 +00:00
m . pmut . RLock ( )
2023-09-06 10:52:01 +00:00
defer m . pmut . RUnlock ( )
2014-01-18 03:06:44 +00:00
2023-09-06 10:52:01 +00:00
connIDs , ok := m . deviceConnIDs [ deviceID ]
2014-01-06 10:11:18 +00:00
if ! ok {
2023-09-06 10:52:01 +00:00
return nil , false
2014-01-06 10:11:18 +00:00
}
2023-09-06 10:52:01 +00:00
// If there is an entry in deviceConns, it always contains at least one
// connection.
connID := connIDs [ 0 ]
if len ( connIDs ) > 1 {
// Pick a random connection of the non-primary ones
idx := rand . Intn ( len ( connIDs ) - 1 ) + 1
connID = connIDs [ idx ]
}
2014-01-06 10:11:18 +00:00
2023-09-06 10:52:01 +00:00
conn , connOK := m . connections [ connID ]
return conn , connOK
2014-01-06 10:11:18 +00:00
}
2019-02-26 08:09:25 +00:00
func ( m * model ) ScanFolders ( ) map [ string ] error {
2014-09-28 11:39:39 +00:00
m . fmut . RLock ( )
2015-04-12 20:12:01 +00:00
folders := make ( [ ] string , 0 , len ( m . folderCfgs ) )
2014-09-28 11:00:38 +00:00
for folder := range m . folderCfgs {
folders = append ( folders , folder )
2014-03-29 17:53:48 +00:00
}
2014-09-28 11:39:39 +00:00
m . fmut . RUnlock ( )
2014-04-14 07:58:17 +00:00
2015-04-12 20:12:01 +00:00
errors := make ( map [ string ] error , len ( m . folderCfgs ) )
2015-04-22 22:54:31 +00:00
errorsMut := sync . NewMutex ( )
2015-02-11 18:52:59 +00:00
2015-04-22 22:54:31 +00:00
wg := sync . NewWaitGroup ( )
2014-09-28 11:00:38 +00:00
wg . Add ( len ( folders ) )
for _ , folder := range folders {
folder := folder
2014-05-13 23:42:12 +00:00
go func ( ) {
2014-09-28 11:00:38 +00:00
err := m . ScanFolder ( folder )
2014-05-28 04:55:30 +00:00
if err != nil {
2015-02-11 18:52:59 +00:00
errorsMut . Lock ( )
errors [ folder ] = err
errorsMut . Unlock ( )
2014-05-28 04:55:30 +00:00
}
2014-05-13 23:42:12 +00:00
wg . Done ( )
} ( )
2014-04-14 07:58:17 +00:00
}
2014-05-13 23:42:12 +00:00
wg . Wait ( )
2015-02-11 18:52:59 +00:00
return errors
2014-03-29 17:53:48 +00:00
}
2013-12-15 10:43:31 +00:00
2019-02-26 08:09:25 +00:00
func ( m * model ) ScanFolder ( folder string ) error {
2016-06-29 06:37:34 +00:00
return m . ScanFolderSubdirs ( folder , nil )
2014-08-11 18:20:01 +00:00
}
2019-02-26 08:09:25 +00:00
func ( m * model ) ScanFolderSubdirs ( folder string , subs [ ] string ) error {
2017-12-15 20:01:56 +00:00
m . fmut . RLock ( )
2019-05-02 17:55:39 +00:00
err := m . checkFolderRunningLocked ( folder )
2023-09-02 14:42:46 +00:00
runner , _ := m . folderRunners . Get ( folder )
2017-12-15 20:01:56 +00:00
m . fmut . RUnlock ( )
2015-06-20 17:26:25 +00:00
2019-05-02 17:55:39 +00:00
if err != nil {
return err
}
2015-06-20 17:26:25 +00:00
return runner . Scan ( subs )
}
2019-02-26 08:09:25 +00:00
func ( m * model ) DelayScan ( folder string , next time . Duration ) {
2019-09-12 04:55:24 +00:00
m . fmut . RLock ( )
2023-09-02 14:42:46 +00:00
runner , ok := m . folderRunners . Get ( folder )
2019-09-12 04:55:24 +00:00
m . fmut . RUnlock ( )
2015-05-01 12:30:17 +00:00
if ! ok {
return
}
runner . DelayScan ( next )
}
2015-04-29 18:46:32 +00:00
// numHashers returns the number of hasher routines to use for a given folder,
// taking into account configuration and available CPU cores.
2019-02-26 08:09:25 +00:00
func ( m * model ) numHashers ( folder string ) int {
2019-09-12 04:55:24 +00:00
m . fmut . RLock ( )
2015-04-29 18:46:32 +00:00
folderCfg := m . folderCfgs [ folder ]
numFolders := len ( m . folderCfgs )
2019-09-12 04:55:24 +00:00
m . fmut . RUnlock ( )
2015-04-29 18:46:32 +00:00
if folderCfg . Hashers > 0 {
// Specific value set in the config, use that.
return folderCfg . Hashers
}
2022-07-28 17:36:39 +00:00
if build . IsWindows || build . IsDarwin || build . IsAndroid {
2015-09-01 08:05:06 +00:00
// Interactive operating systems; don't load the system too heavily by
// default.
return 1
}
// For other operating systems and architectures, lets try to get some
// work done... Divide the available CPU cores among the configured
// folders.
2015-04-29 18:46:32 +00:00
if perFolder := runtime . GOMAXPROCS ( - 1 ) / numFolders ; perFolder > 0 {
return perFolder
}
return 1
}
2021-03-22 20:50:19 +00:00
// generateClusterConfig returns a ClusterConfigMessage that is correct and the
// set of folder passwords for the given peer device
2021-06-03 13:39:49 +00:00
func ( m * model ) generateClusterConfig ( device protocol . DeviceID ) ( protocol . ClusterConfig , map [ string ] string ) {
2014-09-28 11:39:39 +00:00
m . fmut . RLock ( )
2018-05-08 07:19:34 +00:00
defer m . fmut . RUnlock ( )
2023-09-06 10:52:01 +00:00
return m . generateClusterConfigFRLocked ( device )
}
2016-07-23 12:46:31 +00:00
2023-09-06 10:52:01 +00:00
func ( m * model ) generateClusterConfigFRLocked ( device protocol . DeviceID ) ( protocol . ClusterConfig , map [ string ] string ) {
var message protocol . ClusterConfig
2021-03-22 20:50:19 +00:00
folders := m . cfg . FolderList ( )
passwords := make ( map [ string ] string , len ( folders ) )
for _ , folderCfg := range folders {
2018-06-06 21:34:11 +00:00
if ! folderCfg . SharedWith ( device ) {
2018-05-08 07:19:34 +00:00
continue
}
2021-06-03 13:39:49 +00:00
encryptionToken , hasEncryptionToken := m . folderEncryptionPasswordTokens [ folderCfg . ID ]
if folderCfg . Type == config . FolderTypeReceiveEncrypted && ! hasEncryptionToken {
// We haven't gotten a token for us yet and without one the other
// side can't validate us - pretend we don't have the folder yet.
continue
2020-11-09 14:33:32 +00:00
}
2016-03-11 09:48:46 +00:00
protocolFolder := protocol . Folder {
2018-04-22 16:01:52 +00:00
ID : folderCfg . ID ,
2016-07-04 10:40:29 +00:00
Label : folderCfg . Label ,
2016-12-16 22:23:35 +00:00
ReadOnly : folderCfg . Type == config . FolderTypeSendOnly ,
2016-07-04 10:40:29 +00:00
IgnorePermissions : folderCfg . IgnorePerms ,
IgnoreDelete : folderCfg . IgnoreDelete ,
DisableTempIndexes : folderCfg . DisableTempIndexes ,
2015-09-27 11:11:34 +00:00
}
2016-07-04 10:40:29 +00:00
2020-11-20 13:13:50 +00:00
fs := m . folderFiles [ folderCfg . ID ]
// Even if we aren't paused, if we haven't started the folder yet
// pretend we are. Otherwise the remote might get confused about
// the missing index info (and drop all the info). We will send
// another cluster config once the folder is started.
protocolFolder . Paused = folderCfg . Paused || fs == nil
2016-07-23 12:46:31 +00:00
2021-03-22 20:50:19 +00:00
for _ , folderDevice := range folderCfg . Devices {
deviceCfg , _ := m . cfg . Device ( folderDevice . DeviceID )
2016-07-23 12:46:31 +00:00
2016-03-11 09:48:46 +00:00
protocolDevice := protocol . Device {
2018-04-22 16:01:52 +00:00
ID : deviceCfg . DeviceID ,
2016-07-29 19:54:24 +00:00
Name : deviceCfg . Name ,
Addresses : deviceCfg . Addresses ,
Compression : deviceCfg . Compression ,
CertName : deviceCfg . CertName ,
Introducer : deviceCfg . Introducer ,
2018-04-22 16:01:52 +00:00
}
2020-11-09 14:33:32 +00:00
if deviceCfg . DeviceID == m . id && hasEncryptionToken {
protocolDevice . EncryptionPasswordToken = encryptionToken
2021-03-22 20:50:19 +00:00
} else if folderDevice . EncryptionPassword != "" {
2023-03-12 19:06:59 +00:00
protocolDevice . EncryptionPasswordToken = protocol . PasswordToken ( m . keyGen , folderCfg . ID , folderDevice . EncryptionPassword )
2021-03-22 20:50:19 +00:00
if folderDevice . DeviceID == device {
passwords [ folderCfg . ID ] = folderDevice . EncryptionPassword
}
2020-11-09 14:33:32 +00:00
}
2018-05-08 07:19:34 +00:00
if fs != nil {
2018-04-22 16:01:52 +00:00
if deviceCfg . DeviceID == m . id {
protocolDevice . IndexID = fs . IndexID ( protocol . LocalDeviceID )
protocolDevice . MaxSequence = fs . Sequence ( protocol . LocalDeviceID )
} else {
protocolDevice . IndexID = fs . IndexID ( deviceCfg . DeviceID )
protocolDevice . MaxSequence = fs . Sequence ( deviceCfg . DeviceID )
}
2014-09-23 14:04:20 +00:00
}
2015-09-27 10:39:02 +00:00
2016-03-11 09:48:46 +00:00
protocolFolder . Devices = append ( protocolFolder . Devices , protocolDevice )
2014-01-09 12:58:35 +00:00
}
2018-04-22 16:01:52 +00:00
2016-03-11 09:48:46 +00:00
message . Folders = append ( message . Folders , protocolFolder )
2013-12-30 01:33:57 +00:00
}
2014-04-13 13:28:26 +00:00
2021-06-03 13:39:49 +00:00
return message , passwords
2013-12-30 01:33:57 +00:00
}
2014-04-14 07:58:17 +00:00
2019-02-26 08:09:25 +00:00
func ( m * model ) State ( folder string ) ( string , time . Time , error ) {
2015-03-16 20:14:19 +00:00
m . fmut . RLock ( )
2023-09-02 14:42:46 +00:00
runner , ok := m . folderRunners . Get ( folder )
2015-03-16 20:14:19 +00:00
m . fmut . RUnlock ( )
if ! ok {
2015-04-12 20:12:01 +00:00
// The returned error should be an actual folder error, so returning
// errors.New("does not exist") or similar here would be
// inappropriate.
return "" , time . Time { } , nil
2015-03-16 20:14:19 +00:00
}
2015-04-12 20:12:01 +00:00
state , changed , err := runner . getState ( )
return state . String ( ) , changed , err
2014-04-14 07:58:17 +00:00
}
2014-06-16 08:47:02 +00:00
2019-02-26 08:09:25 +00:00
func ( m * model ) FolderErrors ( folder string ) ( [ ] FileError , error ) {
2018-01-14 17:01:06 +00:00
m . fmut . RLock ( )
2019-08-13 07:04:43 +00:00
err := m . checkFolderRunningLocked ( folder )
2023-09-02 14:42:46 +00:00
runner , _ := m . folderRunners . Get ( folder )
2019-08-13 07:04:43 +00:00
m . fmut . RUnlock ( )
if err != nil {
2018-01-14 17:01:06 +00:00
return nil , err
}
2019-08-13 07:04:43 +00:00
return runner . Errors ( ) , nil
2018-01-14 17:01:06 +00:00
}
2019-02-26 08:09:25 +00:00
func ( m * model ) WatchError ( folder string ) error {
2018-02-04 21:46:24 +00:00
m . fmut . RLock ( )
2019-08-13 07:04:43 +00:00
err := m . checkFolderRunningLocked ( folder )
2023-09-02 14:42:46 +00:00
runner , _ := m . folderRunners . Get ( folder )
2019-08-13 07:04:43 +00:00
m . fmut . RUnlock ( )
if err != nil {
2019-07-19 17:41:16 +00:00
return nil // If the folder isn't running, there's no error to report.
2018-02-04 21:46:24 +00:00
}
2019-08-13 07:04:43 +00:00
return runner . WatchError ( )
2018-02-04 21:46:24 +00:00
}
2019-02-26 08:09:25 +00:00
func ( m * model ) Override ( folder string ) {
2018-05-21 06:56:24 +00:00
// Grab the runner and the file set.
2014-09-28 11:39:39 +00:00
m . fmut . RLock ( )
2023-09-02 14:42:46 +00:00
runner , ok := m . folderRunners . Get ( folder )
2014-09-28 11:39:39 +00:00
m . fmut . RUnlock ( )
2019-04-07 11:29:17 +00:00
if ! ok {
2015-04-18 13:41:47 +00:00
return
}
2014-06-23 09:52:13 +00:00
2018-05-21 06:56:24 +00:00
// Run the override, taking updates as if they came from scanning.
2014-07-15 15:54:00 +00:00
2019-04-07 11:29:17 +00:00
runner . Override ( )
2014-06-16 08:47:02 +00:00
}
2014-06-19 22:27:54 +00:00
2019-02-26 08:09:25 +00:00
func ( m * model ) Revert ( folder string ) {
2018-07-12 08:15:57 +00:00
// Grab the runner and the file set.
m . fmut . RLock ( )
2023-09-02 14:42:46 +00:00
runner , ok := m . folderRunners . Get ( folder )
2018-07-12 08:15:57 +00:00
m . fmut . RUnlock ( )
2019-04-07 11:29:17 +00:00
if ! ok {
2018-07-12 08:15:57 +00:00
return
}
// Run the revert, taking updates as if they came from scanning.
2019-04-07 11:29:17 +00:00
runner . Revert ( )
2018-07-12 08:15:57 +00:00
}
2021-02-01 08:27:34 +00:00
type TreeEntry struct {
Name string ` json:"name" `
ModTime time . Time ` json:"modTime" `
Size int64 ` json:"size" `
Type protocol . FileInfoType ` json:"type" `
Children [ ] * TreeEntry ` json:"children,omitempty" `
}
func findByName ( slice [ ] * TreeEntry , name string ) * TreeEntry {
for _ , child := range slice {
if child . Name == name {
return child
}
}
return nil
}
func ( m * model ) GlobalDirectoryTree ( folder , prefix string , levels int , dirsOnly bool ) ( [ ] * TreeEntry , error ) {
2015-02-07 10:52:42 +00:00
m . fmut . RLock ( )
files , ok := m . folderFiles [ folder ]
m . fmut . RUnlock ( )
if ! ok {
2021-03-07 12:43:22 +00:00
return nil , ErrFolderMissing
2015-02-07 10:52:42 +00:00
}
2021-02-01 08:27:34 +00:00
root := & TreeEntry {
Children : make ( [ ] * TreeEntry , 0 ) ,
}
2015-02-07 10:52:42 +00:00
sep := string ( filepath . Separator )
prefix = osutil . NativeFilename ( prefix )
if prefix != "" && ! strings . HasSuffix ( prefix , sep ) {
prefix = prefix + sep
}
2021-03-07 12:43:22 +00:00
snap , err := files . Snapshot ( )
if err != nil {
return nil , err
}
2020-01-21 17:23:08 +00:00
defer snap . Release ( )
2020-05-30 07:50:23 +00:00
snap . WithPrefixedGlobalTruncated ( prefix , func ( fi protocol . FileIntf ) bool {
2015-02-07 10:52:42 +00:00
f := fi . ( db . FileInfoTruncated )
2018-05-17 07:26:40 +00:00
// Don't include the prefix itself.
if f . IsInvalid ( ) || f . IsDeleted ( ) || strings . HasPrefix ( prefix , f . Name ) {
2015-02-07 10:52:42 +00:00
return true
}
f . Name = strings . Replace ( f . Name , prefix , "" , 1 )
2021-02-01 08:27:34 +00:00
dir := filepath . Dir ( f . Name )
base := filepath . Base ( f . Name )
2015-02-07 10:52:42 +00:00
if levels > - 1 && strings . Count ( f . Name , sep ) > levels {
return true
}
2021-02-01 08:27:34 +00:00
parent := root
2015-02-07 10:52:42 +00:00
if dir != "." {
for _ , path := range strings . Split ( dir , sep ) {
2021-02-01 08:27:34 +00:00
child := findByName ( parent . Children , path )
if child == nil {
err = fmt . Errorf ( "could not find child '%s' for path '%s' in parent '%s'" , path , f . Name , parent . Name )
return false
2015-02-07 10:52:42 +00:00
}
2021-02-01 08:27:34 +00:00
parent = child
2015-02-07 10:52:42 +00:00
}
}
2021-02-01 08:27:34 +00:00
if dirsOnly && ! f . IsDirectory ( ) {
return true
2015-02-07 10:52:42 +00:00
}
2021-02-01 08:27:34 +00:00
parent . Children = append ( parent . Children , & TreeEntry {
Name : base ,
Type : f . Type ,
ModTime : f . ModTime ( ) ,
Size : f . FileSize ( ) ,
} )
2015-02-07 10:52:42 +00:00
return true
} )
2021-02-01 08:27:34 +00:00
if err != nil {
return nil , err
}
2015-02-07 10:52:42 +00:00
2021-02-01 08:27:34 +00:00
return root . Children , nil
2015-02-07 10:52:42 +00:00
}
2019-02-26 08:09:25 +00:00
func ( m * model ) GetFolderVersions ( folder string ) ( map [ string ] [ ] versioner . FileVersion , error ) {
2019-11-26 07:39:31 +00:00
m . fmut . RLock ( )
2020-02-12 06:35:24 +00:00
err := m . checkFolderRunningLocked ( folder )
ver := m . folderVersioners [ folder ]
2019-11-26 07:39:31 +00:00
m . fmut . RUnlock ( )
2020-02-12 06:35:24 +00:00
if err != nil {
return nil , err
2018-01-01 14:39:23 +00:00
}
2019-04-28 22:30:16 +00:00
if ver == nil {
2019-11-26 07:39:31 +00:00
return nil , errNoVersioner
2018-01-01 14:39:23 +00:00
}
2019-04-28 22:30:16 +00:00
return ver . GetVersions ( )
2018-01-01 14:39:23 +00:00
}
2021-02-12 19:30:51 +00:00
func ( m * model ) RestoreFolderVersions ( folder string , versions map [ string ] time . Time ) ( map [ string ] error , error ) {
2019-11-26 07:39:31 +00:00
m . fmut . RLock ( )
2020-02-12 06:35:24 +00:00
err := m . checkFolderRunningLocked ( folder )
fcfg := m . folderCfgs [ folder ]
2019-11-26 07:39:31 +00:00
ver := m . folderVersioners [ folder ]
m . fmut . RUnlock ( )
2020-02-12 06:35:24 +00:00
if err != nil {
return nil , err
2019-11-26 07:39:31 +00:00
}
if ver == nil {
return nil , errNoVersioner
}
2018-01-01 14:39:23 +00:00
2021-02-12 19:30:51 +00:00
restoreErrors := make ( map [ string ] error )
2018-01-01 14:39:23 +00:00
for file , version := range versions {
2019-04-28 22:30:16 +00:00
if err := ver . Restore ( file , version ) ; err != nil {
2021-02-12 19:30:51 +00:00
restoreErrors [ file ] = err
2018-01-01 14:39:23 +00:00
}
}
// Trigger scan
if ! fcfg . FSWatcherEnabled {
2019-04-28 22:30:16 +00:00
go func ( ) { _ = m . ScanFolder ( folder ) } ( )
2018-01-01 14:39:23 +00:00
}
2019-04-28 22:30:16 +00:00
return restoreErrors , nil
2018-01-01 14:39:23 +00:00
}
2021-03-07 12:43:22 +00:00
func ( m * model ) Availability ( folder string , file protocol . FileInfo , block protocol . BlockInfo ) ( [ ] Availability , error ) {
2016-11-08 06:38:50 +00:00
// The slightly unusual locking sequence here is because we need to hold
// pmut for the duration (as the value returned from foldersFiles can
// get heavily modified on Close()), but also must acquire fmut before
// pmut. (The locks can be *released* in any order.)
m . fmut . RLock ( )
2014-10-31 23:41:18 +00:00
m . pmut . RLock ( )
defer m . pmut . RUnlock ( )
2014-09-28 11:00:38 +00:00
fs , ok := m . folderFiles [ folder ]
2018-06-06 21:34:11 +00:00
cfg := m . folderCfgs [ folder ]
2014-11-03 21:02:55 +00:00
m . fmut . RUnlock ( )
2016-11-08 06:38:50 +00:00
2014-09-27 12:44:15 +00:00
if ! ok {
2021-03-07 12:43:22 +00:00
return nil , ErrFolderMissing
2014-09-27 12:44:15 +00:00
}
2021-03-07 12:43:22 +00:00
snap , err := fs . Snapshot ( )
if err != nil {
return nil , err
}
2020-01-21 17:23:08 +00:00
defer snap . Release ( )
2021-03-07 12:43:22 +00:00
2021-03-17 19:46:13 +00:00
return m . availabilityInSnapshotPRlocked ( cfg , snap , file , block ) , nil
2021-03-07 12:43:22 +00:00
}
func ( m * model ) availabilityInSnapshot ( cfg config . FolderConfiguration , snap * db . Snapshot , file protocol . FileInfo , block protocol . BlockInfo ) [ ] Availability {
2021-03-17 19:46:13 +00:00
m . pmut . RLock ( )
defer m . pmut . RUnlock ( )
return m . availabilityInSnapshotPRlocked ( cfg , snap , file , block )
}
func ( m * model ) availabilityInSnapshotPRlocked ( cfg config . FolderConfiguration , snap * db . Snapshot , file protocol . FileInfo , block protocol . BlockInfo ) [ ] Availability {
2021-03-07 12:43:22 +00:00
var availabilities [ ] Availability
2020-01-21 17:23:08 +00:00
for _ , device := range snap . Availability ( file . Name ) {
2022-04-10 20:47:57 +00:00
if _ , ok := m . remoteFolderStates [ device ] ; ! ok {
2020-10-21 09:51:53 +00:00
continue
}
2022-04-22 06:42:20 +00:00
if state := m . remoteFolderStates [ device ] [ cfg . ID ] ; state != remoteFolderValid {
2020-10-21 09:51:53 +00:00
continue
2016-12-21 18:41:25 +00:00
}
2023-09-06 10:52:01 +00:00
_ , ok := m . deviceConnIDs [ device ]
2014-10-31 23:41:18 +00:00
if ok {
2016-04-15 10:59:41 +00:00
availabilities = append ( availabilities , Availability { ID : device , FromTemporary : false } )
2014-10-31 23:41:18 +00:00
}
}
2016-04-15 10:59:41 +00:00
2018-06-06 21:34:11 +00:00
for _ , device := range cfg . Devices {
2021-03-07 12:43:22 +00:00
if m . deviceDownloads [ device . DeviceID ] . Has ( cfg . ID , file . Name , file . Version , int ( block . Offset / int64 ( file . BlockSize ( ) ) ) ) {
2018-06-06 21:34:11 +00:00
availabilities = append ( availabilities , Availability { ID : device . DeviceID , FromTemporary : true } )
2016-04-15 10:59:41 +00:00
}
}
return availabilities
2014-09-27 12:44:15 +00:00
}
2015-04-28 20:32:10 +00:00
// BringToFront bumps the given files priority in the job queue.
2019-02-26 08:09:25 +00:00
func ( m * model ) BringToFront ( folder , file string ) {
2019-05-02 12:09:42 +00:00
m . fmut . RLock ( )
2023-09-02 14:42:46 +00:00
runner , ok := m . folderRunners . Get ( folder )
2019-05-02 17:55:39 +00:00
m . fmut . RUnlock ( )
Add job queue (fixes #629)
Request to terminate currently ongoing downloads and jump to the bumped file
incoming in 3, 2, 1.
Also, has a slightly strange effect where we pop a job off the queue, but
the copyChannel is still busy and blocks, though it gets moved to the
progress slice in the jobqueue, and looks like it's in progress which it isn't
as it's waiting to be picked up from the copyChan.
As a result, the progress emitter doesn't register on the task, and hence the file
doesn't have a progress bar, but cannot be replaced by a bump.
I guess I can fix progress bar issue by moving the progressEmiter.Register just
before passing the file to the copyChan, but then we are back to the initial
problem of a file with a progress bar, but no progress happening as it's stuck
on write to copyChan
I checked if there is a way to check for channel writeability (before popping)
but got struck by lightning just for bringing the idea up in #go-nuts.
My ideal scenario would be to check if copyChan is writeable, pop job from the
queue and shove it down handleFile. This way jobs would stay in the queue while
they cannot be handled, meaning that the `Bump` could bring your file up higher.
2014-12-01 19:23:06 +00:00
if ok {
2014-12-30 08:35:21 +00:00
runner . BringToFront ( file )
Add job queue (fixes #629)
Request to terminate currently ongoing downloads and jump to the bumped file
incoming in 3, 2, 1.
Also, has a slightly strange effect where we pop a job off the queue, but
the copyChannel is still busy and blocks, though it gets moved to the
progress slice in the jobqueue, and looks like it's in progress which it isn't
as it's waiting to be picked up from the copyChan.
As a result, the progress emitter doesn't register on the task, and hence the file
doesn't have a progress bar, but cannot be replaced by a bump.
I guess I can fix progress bar issue by moving the progressEmiter.Register just
before passing the file to the copyChan, but then we are back to the initial
problem of a file with a progress bar, but no progress happening as it's stuck
on write to copyChan
I checked if there is a way to check for channel writeability (before popping)
but got struck by lightning just for bringing the idea up in #go-nuts.
My ideal scenario would be to check if copyChan is writeable, pop job from the
queue and shove it down handleFile. This way jobs would stay in the queue while
they cannot be handled, meaning that the `Bump` could bring your file up higher.
2014-12-01 19:23:06 +00:00
}
}
2021-09-11 15:14:47 +00:00
func ( m * model ) ResetFolder ( folder string ) error {
m . fmut . RLock ( )
defer m . fmut . RUnlock ( )
2023-09-02 14:42:46 +00:00
_ , ok := m . folderRunners . Get ( folder )
2021-09-11 15:14:47 +00:00
if ok {
return errors . New ( "folder must be paused when resetting" )
}
2021-09-10 07:56:44 +00:00
l . Infof ( "Cleaning metadata for reset folder %q" , folder )
2015-06-21 07:35:41 +00:00
db . DropFolder ( m . db , folder )
2021-09-11 15:14:47 +00:00
return nil
2015-04-03 18:06:03 +00:00
}
2019-02-26 08:09:25 +00:00
func ( m * model ) String ( ) string {
2014-09-27 12:44:15 +00:00
return fmt . Sprintf ( "model@%p" , m )
}
2014-10-13 12:43:01 +00:00
2022-07-28 15:32:45 +00:00
func ( * model ) VerifyConfiguration ( from , to config . Configuration ) error {
2021-07-03 11:47:04 +00:00
toFolders := to . FolderMap ( )
for _ , from := range from . Folders {
to , ok := toFolders [ from . ID ]
if ok && from . Type != to . Type && ( from . Type == config . FolderTypeReceiveEncrypted || to . Type == config . FolderTypeReceiveEncrypted ) {
return errors . New ( "folder type must not be changed from/to receive-encrypted" )
}
}
2023-10-07 02:09:51 +00:00
// Verify that any requested versioning is possible to construct, or we
// will panic later when starting the folder.
for _ , to := range to . Folders {
if to . Versioning . Type != "" {
if _ , err := versioner . New ( to ) ; err != nil {
return err
}
}
}
2015-06-03 07:47:39 +00:00
return nil
}
2019-02-26 08:09:25 +00:00
func ( m * model ) CommitConfiguration ( from , to config . Configuration ) bool {
2015-06-03 07:47:39 +00:00
// TODO: This should not use reflect, and should take more care to try to handle stuff without restart.
2021-01-15 14:43:34 +00:00
// Delay processing config changes until after the initial setup
<- m . started
2015-07-22 07:02:55 +00:00
// Go through the folder configs and figure out if we need to restart or not.
2020-10-02 09:49:51 +00:00
// Tracks devices affected by any configuration change to resend ClusterConfig.
2020-11-20 14:53:13 +00:00
clusterConfigDevices := make ( deviceIDSet , len ( from . Devices ) + len ( to . Devices ) )
2021-06-03 13:39:49 +00:00
closeDevices := make ( [ ] protocol . DeviceID , 0 , len ( to . Devices ) )
2020-10-02 09:49:51 +00:00
2015-07-22 07:02:55 +00:00
fromFolders := mapFolders ( from . Folders )
toFolders := mapFolders ( to . Folders )
2015-07-23 14:13:53 +00:00
for folderID , cfg := range toFolders {
2015-07-22 07:02:55 +00:00
if _ , ok := fromFolders [ folderID ] ; ! ok {
2015-07-23 14:13:53 +00:00
// A folder was added.
2017-04-01 09:58:06 +00:00
if cfg . Paused {
2017-12-07 07:08:24 +00:00
l . Infoln ( "Paused folder" , cfg . Description ( ) )
2017-04-01 09:58:06 +00:00
} else {
2017-12-07 07:08:24 +00:00
l . Infoln ( "Adding folder" , cfg . Description ( ) )
2020-12-21 11:59:22 +00:00
if err := m . newFolder ( cfg , to . Options . CacheIgnoredFiles ) ; err != nil {
m . fatal ( err )
return true
}
2017-04-01 09:58:06 +00:00
}
2020-11-20 14:53:13 +00:00
clusterConfigDevices . add ( cfg . DeviceIDs ( ) )
2015-07-22 07:02:55 +00:00
}
2015-06-03 07:47:39 +00:00
}
2020-12-17 18:54:31 +00:00
removedFolders := make ( map [ string ] struct { } )
2015-07-22 07:02:55 +00:00
for folderID , fromCfg := range fromFolders {
toCfg , ok := toFolders [ folderID ]
if ! ok {
2015-11-13 12:30:52 +00:00
// The folder was removed.
2019-11-08 09:56:16 +00:00
m . removeFolder ( fromCfg )
2020-11-20 14:53:13 +00:00
clusterConfigDevices . add ( fromCfg . DeviceIDs ( ) )
2020-12-17 18:54:31 +00:00
removedFolders [ fromCfg . ID ] = struct { } { }
2015-11-13 12:30:52 +00:00
continue
2015-07-22 07:02:55 +00:00
}
2019-01-09 16:31:09 +00:00
if fromCfg . Paused && toCfg . Paused {
continue
}
2016-08-07 16:21:59 +00:00
// This folder exists on both sides. Settings might have changed.
2017-12-07 08:33:32 +00:00
// Check if anything differs that requires a restart.
2020-08-18 07:26:33 +00:00
if ! reflect . DeepEqual ( fromCfg . RequiresRestartOnly ( ) , toCfg . RequiresRestartOnly ( ) ) || from . Options . CacheIgnoredFiles != to . Options . CacheIgnoredFiles {
2020-12-21 11:59:22 +00:00
if err := m . restartFolder ( fromCfg , toCfg , to . Options . CacheIgnoredFiles ) ; err != nil {
m . fatal ( err )
return true
}
2020-11-20 14:53:13 +00:00
clusterConfigDevices . add ( fromCfg . DeviceIDs ( ) )
2021-06-03 13:39:49 +00:00
if toCfg . Type != config . FolderTypeReceiveEncrypted {
clusterConfigDevices . add ( toCfg . DeviceIDs ( ) )
} else {
// If we don't have the encryption token yet, we need to drop
// the connection to make the remote re-send the cluster-config
// and with it the token.
m . fmut . RLock ( )
_ , ok := m . folderEncryptionPasswordTokens [ toCfg . ID ]
m . fmut . RUnlock ( )
if ! ok {
2022-07-28 14:57:09 +00:00
closeDevices = append ( closeDevices , toCfg . DeviceIDs ( ) ... )
2021-06-03 13:39:49 +00:00
} else {
clusterConfigDevices . add ( toCfg . DeviceIDs ( ) )
}
}
2015-07-22 07:02:55 +00:00
}
2016-12-21 18:41:25 +00:00
// Emit the folder pause/resume event
if fromCfg . Paused != toCfg . Paused {
eventType := events . FolderResumed
if toCfg . Paused {
eventType = events . FolderPaused
}
2019-08-15 14:29:37 +00:00
m . evLogger . Log ( eventType , map [ string ] string { "id" : toCfg . ID , "label" : toCfg . Label } )
2016-12-21 18:41:25 +00:00
}
2015-06-03 07:47:39 +00:00
}
2015-07-22 07:02:55 +00:00
2016-12-21 18:41:25 +00:00
// Pausing a device, unpausing is handled by the connection service.
2018-03-26 10:01:59 +00:00
fromDevices := from . DeviceMap ( )
toDevices := to . DeviceMap ( )
2016-12-21 18:41:25 +00:00
for deviceID , toCfg := range toDevices {
fromCfg , ok := fromDevices [ deviceID ]
2019-09-12 04:55:24 +00:00
if ! ok {
2021-01-05 16:45:07 +00:00
sr := stats . NewDeviceStatisticsReference ( m . db , deviceID )
2019-09-12 04:55:24 +00:00
m . fmut . Lock ( )
m . deviceStatRefs [ deviceID ] = sr
m . fmut . Unlock ( )
continue
}
delete ( fromDevices , deviceID )
if fromCfg . Paused == toCfg . Paused {
2016-12-21 18:41:25 +00:00
continue
}
if toCfg . Paused {
l . Infoln ( "Pausing" , deviceID )
2020-10-02 09:49:51 +00:00
closeDevices = append ( closeDevices , deviceID )
2019-08-15 14:29:37 +00:00
m . evLogger . Log ( events . DevicePaused , map [ string ] string { "device" : deviceID . String ( ) } )
2016-12-21 18:41:25 +00:00
} else {
2021-06-03 13:39:49 +00:00
// Ignored folder was removed, reconnect to retrigger the prompt.
if len ( fromCfg . IgnoredFolders ) > len ( toCfg . IgnoredFolders ) {
closeDevices = append ( closeDevices , deviceID )
}
l . Infoln ( "Resuming" , deviceID )
2019-08-15 14:29:37 +00:00
m . evLogger . Log ( events . DeviceResumed , map [ string ] string { "device" : deviceID . String ( ) } )
2016-12-21 18:41:25 +00:00
}
2023-09-06 10:52:01 +00:00
if toCfg . MaxRequestKiB != fromCfg . MaxRequestKiB {
m . pmut . Lock ( )
m . setConnRequestLimitersPLocked ( toCfg )
m . pmut . Unlock ( )
}
2016-12-21 18:41:25 +00:00
}
2023-09-06 10:52:01 +00:00
2020-12-17 18:54:31 +00:00
// Clean up after removed devices
2020-05-14 05:50:53 +00:00
removedDevices := make ( [ ] protocol . DeviceID , 0 , len ( fromDevices ) )
2019-09-12 04:55:24 +00:00
m . fmut . Lock ( )
for deviceID := range fromDevices {
delete ( m . deviceStatRefs , deviceID )
2020-05-14 05:50:53 +00:00
removedDevices = append ( removedDevices , deviceID )
2020-10-02 09:49:51 +00:00
delete ( clusterConfigDevices , deviceID )
2019-09-12 04:55:24 +00:00
}
m . fmut . Unlock ( )
2020-10-02 09:49:51 +00:00
m . pmut . RLock ( )
for _ , id := range closeDevices {
2021-06-03 13:39:49 +00:00
delete ( clusterConfigDevices , id )
2023-09-06 10:52:01 +00:00
if conns , ok := m . deviceConnIDs [ id ] ; ok {
for _ , connID := range conns {
go m . connections [ connID ] . Close ( errDevicePaused )
}
2020-10-02 09:49:51 +00:00
}
}
for _ , id := range removedDevices {
2021-06-03 13:39:49 +00:00
delete ( clusterConfigDevices , id )
2023-09-06 10:52:01 +00:00
if conns , ok := m . deviceConnIDs [ id ] ; ok {
for _ , connID := range conns {
go m . connections [ connID ] . Close ( errDevicePaused )
}
2020-10-02 09:49:51 +00:00
}
}
m . pmut . RUnlock ( )
2020-11-09 13:05:21 +00:00
// Generating cluster-configs acquires fmut -> must happen outside of pmut.
2021-03-22 20:50:19 +00:00
m . sendClusterConfig ( clusterConfigDevices . AsSlice ( ) )
2016-12-21 18:41:25 +00:00
2020-12-17 18:54:31 +00:00
ignoredDevices := observedDeviceSet ( to . IgnoredDevices )
m . cleanPending ( toDevices , toFolders , ignoredDevices , removedFolders )
2021-06-25 09:38:04 +00:00
m . globalRequestLimiter . SetCapacity ( 1024 * to . Options . MaxConcurrentIncomingRequestKiB ( ) )
m . folderIOLimiter . SetCapacity ( to . Options . MaxFolderConcurrency ( ) )
2018-12-05 07:40:05 +00:00
2016-01-18 18:06:31 +00:00
// Some options don't require restart as those components handle it fine
2017-12-07 08:33:32 +00:00
// by themselves. Compare the options structs containing only the
// attributes that require restart and act apprioriately.
if ! reflect . DeepEqual ( from . Options . RequiresRestartOnly ( ) , to . Options . RequiresRestartOnly ( ) ) {
2015-10-03 15:25:21 +00:00
l . Debugln ( m , "requires restart, options differ" )
2015-06-03 07:47:39 +00:00
return false
}
return true
}
2023-09-06 10:52:01 +00:00
func ( m * model ) setConnRequestLimitersPLocked ( cfg config . DeviceConfiguration ) {
// Touches connRequestLimiters which is protected by pmut.
// 0: default, <0: no limiting
switch {
case cfg . MaxRequestKiB > 0 :
m . connRequestLimiters [ cfg . DeviceID ] = semaphore . New ( 1024 * cfg . MaxRequestKiB )
case cfg . MaxRequestKiB == 0 :
m . connRequestLimiters [ cfg . DeviceID ] = semaphore . New ( 1024 * defaultPullerPendingKiB )
}
}
2020-12-17 18:54:31 +00:00
func ( m * model ) cleanPending ( existingDevices map [ protocol . DeviceID ] config . DeviceConfiguration , existingFolders map [ string ] config . FolderConfiguration , ignoredDevices deviceIDSet , removedFolders map [ string ] struct { } ) {
lib/model: Forget pending folders no longer announced in ClusterConfig (fixes #5187) (#7205)
* lib/db: Add ExpirePendingFolders().
Use-case is to drop any no-longer-pending folders for a specific
device when parsing its ClusterConfig message where previously offered
folders are not mentioned any more.
The timestamp in ObservedFolder is stored with only second precision,
so round to seconds here as well. This allows calling the function
within the same second of adding or updating entries.
* lib/model: Weed out pending folders when receiving ClusterConfig.
Filter the entries by timestamp, which must be newer than or equal to
the reception time of the ClusterConfig. For just mentioned ones,
this assumption will hold as AddOrUpdatePendingFolder() updates the
timestamp.
* lib/model, gui: Notify when one or more pending folders expired.
Introduce new event type FolderOfferCancelled and use it to trigger a
complete refreshCluster() cycle. Listing individual entries would be
much more code and probably just as much work to answer the API
request.
* lib/model: Add comment and rename ExpirePendingFolders().
* lib/events: Rename FolderOfferCancelled to ClusterPendingChanged.
* lib/model: Reuse ClusterPendingChanged event for cleanPending()
Changing the config does not necessarily mean that the
/resut/cluster/pending endpoints need to be refreshed, but only if
something was actually removed. Detect this and indicate it through
the ClusterPendingChanged event, which is already hooked up to requery
respective endpoints within the GUI.
No more need for a separate refreshCluster() in reaction to
ConfigSaved event or calling refreshConfig().
* lib/model: Gofmt.
* lib/db: Warn instead of info log for failed removal.
* gui: Fix pending notifications not loading on GUI start.
* lib/db: Use short device ID in log message.
* lib/db: Return list of expired folder IDs after deleting them.
* lib/model: Refactor Pending...Changed events.
* lib/model: Adjust format of removed pending folders enumeration.
Use an array of objects with device / folder ID properties, matching
the other places where it's used.
* lib/db: Drop invalid entries in RemovePendingFoldersBeforeTime().
* lib/model: Gofmt.
My local gofmt did not complain here, strangely...
* gui: Handle PendingDevicesChanged event.
Even though it currently only holds one device at a time, wrap the
contents in an array under the "added" property name.
* lib/model: Fix null values in PendingFoldersChanged removed member.
* gui: Handle PendingFoldersChanged event.
* lib/model: Simplify construction of expiredPendingList.
* lib/model: Reduce code duplication in cleanPending().
Use goto and a label for the common parts of calling the DB removal
function and building the event data part.
* lib/events, gui: Mark ...Rejected events deprecated.
Extend comments explaining the conditions when the replacement event
types are emitted.
* lib/model: Wrap removed devices in array of objects as well.
* lib/db: Use iter.Value() instead of needless db.Get(iter.Key())
* lib/db: Add comment explaining RemovePendingFoldersBeforeTime().
* lib/model: Rename fields folderID and deviceID in event data.
* lib/db: Only list actually expired IDs as removed.
Skip entries where Delete() failed as well as invalid entries that got
removed automatically.
* lib/model: Gofmt
2021-01-25 10:58:10 +00:00
var removedPendingFolders [ ] map [ string ] string
2020-12-17 18:54:31 +00:00
pendingFolders , err := m . db . PendingFolders ( )
if err != nil {
2021-06-09 11:35:17 +00:00
msg := "Could not iterate through pending folder entries for cleanup"
l . Warnf ( "%v: %v" , msg , err )
m . evLogger . Log ( events . Failure , msg )
2020-12-17 18:54:31 +00:00
// Continue with pending devices below, loop is skipped.
}
for folderID , pf := range pendingFolders {
if _ , ok := removedFolders [ folderID ] ; ok {
// Forget pending folder device associations for recently removed
// folders as well, assuming the folder is no longer of interest
// at all (but might become pending again).
l . Debugf ( "Discarding pending removed folder %v from all devices" , folderID )
2021-06-07 08:29:24 +00:00
if err := m . db . RemovePendingFolder ( folderID ) ; err != nil {
2021-06-09 11:35:17 +00:00
msg := "Failed to remove pending folder entry"
l . Warnf ( "%v (%v): %v" , msg , folderID , err )
m . evLogger . Log ( events . Failure , msg )
2021-06-07 08:29:24 +00:00
} else {
removedPendingFolders = append ( removedPendingFolders , map [ string ] string {
"folderID" : folderID ,
} )
}
2020-12-17 18:54:31 +00:00
continue
}
for deviceID := range pf . OfferedBy {
if dev , ok := existingDevices [ deviceID ] ; ! ok {
l . Debugf ( "Discarding pending folder %v from unknown device %v" , folderID , deviceID )
lib/model: Forget pending folders no longer announced in ClusterConfig (fixes #5187) (#7205)
* lib/db: Add ExpirePendingFolders().
Use-case is to drop any no-longer-pending folders for a specific
device when parsing its ClusterConfig message where previously offered
folders are not mentioned any more.
The timestamp in ObservedFolder is stored with only second precision,
so round to seconds here as well. This allows calling the function
within the same second of adding or updating entries.
* lib/model: Weed out pending folders when receiving ClusterConfig.
Filter the entries by timestamp, which must be newer than or equal to
the reception time of the ClusterConfig. For just mentioned ones,
this assumption will hold as AddOrUpdatePendingFolder() updates the
timestamp.
* lib/model, gui: Notify when one or more pending folders expired.
Introduce new event type FolderOfferCancelled and use it to trigger a
complete refreshCluster() cycle. Listing individual entries would be
much more code and probably just as much work to answer the API
request.
* lib/model: Add comment and rename ExpirePendingFolders().
* lib/events: Rename FolderOfferCancelled to ClusterPendingChanged.
* lib/model: Reuse ClusterPendingChanged event for cleanPending()
Changing the config does not necessarily mean that the
/resut/cluster/pending endpoints need to be refreshed, but only if
something was actually removed. Detect this and indicate it through
the ClusterPendingChanged event, which is already hooked up to requery
respective endpoints within the GUI.
No more need for a separate refreshCluster() in reaction to
ConfigSaved event or calling refreshConfig().
* lib/model: Gofmt.
* lib/db: Warn instead of info log for failed removal.
* gui: Fix pending notifications not loading on GUI start.
* lib/db: Use short device ID in log message.
* lib/db: Return list of expired folder IDs after deleting them.
* lib/model: Refactor Pending...Changed events.
* lib/model: Adjust format of removed pending folders enumeration.
Use an array of objects with device / folder ID properties, matching
the other places where it's used.
* lib/db: Drop invalid entries in RemovePendingFoldersBeforeTime().
* lib/model: Gofmt.
My local gofmt did not complain here, strangely...
* gui: Handle PendingDevicesChanged event.
Even though it currently only holds one device at a time, wrap the
contents in an array under the "added" property name.
* lib/model: Fix null values in PendingFoldersChanged removed member.
* gui: Handle PendingFoldersChanged event.
* lib/model: Simplify construction of expiredPendingList.
* lib/model: Reduce code duplication in cleanPending().
Use goto and a label for the common parts of calling the DB removal
function and building the event data part.
* lib/events, gui: Mark ...Rejected events deprecated.
Extend comments explaining the conditions when the replacement event
types are emitted.
* lib/model: Wrap removed devices in array of objects as well.
* lib/db: Use iter.Value() instead of needless db.Get(iter.Key())
* lib/db: Add comment explaining RemovePendingFoldersBeforeTime().
* lib/model: Rename fields folderID and deviceID in event data.
* lib/db: Only list actually expired IDs as removed.
Skip entries where Delete() failed as well as invalid entries that got
removed automatically.
* lib/model: Gofmt
2021-01-25 10:58:10 +00:00
goto removeFolderForDevice
2020-12-17 18:54:31 +00:00
} else if dev . IgnoredFolder ( folderID ) {
l . Debugf ( "Discarding now ignored pending folder %v for device %v" , folderID , deviceID )
lib/model: Forget pending folders no longer announced in ClusterConfig (fixes #5187) (#7205)
* lib/db: Add ExpirePendingFolders().
Use-case is to drop any no-longer-pending folders for a specific
device when parsing its ClusterConfig message where previously offered
folders are not mentioned any more.
The timestamp in ObservedFolder is stored with only second precision,
so round to seconds here as well. This allows calling the function
within the same second of adding or updating entries.
* lib/model: Weed out pending folders when receiving ClusterConfig.
Filter the entries by timestamp, which must be newer than or equal to
the reception time of the ClusterConfig. For just mentioned ones,
this assumption will hold as AddOrUpdatePendingFolder() updates the
timestamp.
* lib/model, gui: Notify when one or more pending folders expired.
Introduce new event type FolderOfferCancelled and use it to trigger a
complete refreshCluster() cycle. Listing individual entries would be
much more code and probably just as much work to answer the API
request.
* lib/model: Add comment and rename ExpirePendingFolders().
* lib/events: Rename FolderOfferCancelled to ClusterPendingChanged.
* lib/model: Reuse ClusterPendingChanged event for cleanPending()
Changing the config does not necessarily mean that the
/resut/cluster/pending endpoints need to be refreshed, but only if
something was actually removed. Detect this and indicate it through
the ClusterPendingChanged event, which is already hooked up to requery
respective endpoints within the GUI.
No more need for a separate refreshCluster() in reaction to
ConfigSaved event or calling refreshConfig().
* lib/model: Gofmt.
* lib/db: Warn instead of info log for failed removal.
* gui: Fix pending notifications not loading on GUI start.
* lib/db: Use short device ID in log message.
* lib/db: Return list of expired folder IDs after deleting them.
* lib/model: Refactor Pending...Changed events.
* lib/model: Adjust format of removed pending folders enumeration.
Use an array of objects with device / folder ID properties, matching
the other places where it's used.
* lib/db: Drop invalid entries in RemovePendingFoldersBeforeTime().
* lib/model: Gofmt.
My local gofmt did not complain here, strangely...
* gui: Handle PendingDevicesChanged event.
Even though it currently only holds one device at a time, wrap the
contents in an array under the "added" property name.
* lib/model: Fix null values in PendingFoldersChanged removed member.
* gui: Handle PendingFoldersChanged event.
* lib/model: Simplify construction of expiredPendingList.
* lib/model: Reduce code duplication in cleanPending().
Use goto and a label for the common parts of calling the DB removal
function and building the event data part.
* lib/events, gui: Mark ...Rejected events deprecated.
Extend comments explaining the conditions when the replacement event
types are emitted.
* lib/model: Wrap removed devices in array of objects as well.
* lib/db: Use iter.Value() instead of needless db.Get(iter.Key())
* lib/db: Add comment explaining RemovePendingFoldersBeforeTime().
* lib/model: Rename fields folderID and deviceID in event data.
* lib/db: Only list actually expired IDs as removed.
Skip entries where Delete() failed as well as invalid entries that got
removed automatically.
* lib/model: Gofmt
2021-01-25 10:58:10 +00:00
goto removeFolderForDevice
2020-12-17 18:54:31 +00:00
}
if folderCfg , ok := existingFolders [ folderID ] ; ok {
if folderCfg . SharedWith ( deviceID ) {
l . Debugf ( "Discarding now shared pending folder %v for device %v" , folderID , deviceID )
lib/model: Forget pending folders no longer announced in ClusterConfig (fixes #5187) (#7205)
* lib/db: Add ExpirePendingFolders().
Use-case is to drop any no-longer-pending folders for a specific
device when parsing its ClusterConfig message where previously offered
folders are not mentioned any more.
The timestamp in ObservedFolder is stored with only second precision,
so round to seconds here as well. This allows calling the function
within the same second of adding or updating entries.
* lib/model: Weed out pending folders when receiving ClusterConfig.
Filter the entries by timestamp, which must be newer than or equal to
the reception time of the ClusterConfig. For just mentioned ones,
this assumption will hold as AddOrUpdatePendingFolder() updates the
timestamp.
* lib/model, gui: Notify when one or more pending folders expired.
Introduce new event type FolderOfferCancelled and use it to trigger a
complete refreshCluster() cycle. Listing individual entries would be
much more code and probably just as much work to answer the API
request.
* lib/model: Add comment and rename ExpirePendingFolders().
* lib/events: Rename FolderOfferCancelled to ClusterPendingChanged.
* lib/model: Reuse ClusterPendingChanged event for cleanPending()
Changing the config does not necessarily mean that the
/resut/cluster/pending endpoints need to be refreshed, but only if
something was actually removed. Detect this and indicate it through
the ClusterPendingChanged event, which is already hooked up to requery
respective endpoints within the GUI.
No more need for a separate refreshCluster() in reaction to
ConfigSaved event or calling refreshConfig().
* lib/model: Gofmt.
* lib/db: Warn instead of info log for failed removal.
* gui: Fix pending notifications not loading on GUI start.
* lib/db: Use short device ID in log message.
* lib/db: Return list of expired folder IDs after deleting them.
* lib/model: Refactor Pending...Changed events.
* lib/model: Adjust format of removed pending folders enumeration.
Use an array of objects with device / folder ID properties, matching
the other places where it's used.
* lib/db: Drop invalid entries in RemovePendingFoldersBeforeTime().
* lib/model: Gofmt.
My local gofmt did not complain here, strangely...
* gui: Handle PendingDevicesChanged event.
Even though it currently only holds one device at a time, wrap the
contents in an array under the "added" property name.
* lib/model: Fix null values in PendingFoldersChanged removed member.
* gui: Handle PendingFoldersChanged event.
* lib/model: Simplify construction of expiredPendingList.
* lib/model: Reduce code duplication in cleanPending().
Use goto and a label for the common parts of calling the DB removal
function and building the event data part.
* lib/events, gui: Mark ...Rejected events deprecated.
Extend comments explaining the conditions when the replacement event
types are emitted.
* lib/model: Wrap removed devices in array of objects as well.
* lib/db: Use iter.Value() instead of needless db.Get(iter.Key())
* lib/db: Add comment explaining RemovePendingFoldersBeforeTime().
* lib/model: Rename fields folderID and deviceID in event data.
* lib/db: Only list actually expired IDs as removed.
Skip entries where Delete() failed as well as invalid entries that got
removed automatically.
* lib/model: Gofmt
2021-01-25 10:58:10 +00:00
goto removeFolderForDevice
2020-12-17 18:54:31 +00:00
}
}
lib/model: Forget pending folders no longer announced in ClusterConfig (fixes #5187) (#7205)
* lib/db: Add ExpirePendingFolders().
Use-case is to drop any no-longer-pending folders for a specific
device when parsing its ClusterConfig message where previously offered
folders are not mentioned any more.
The timestamp in ObservedFolder is stored with only second precision,
so round to seconds here as well. This allows calling the function
within the same second of adding or updating entries.
* lib/model: Weed out pending folders when receiving ClusterConfig.
Filter the entries by timestamp, which must be newer than or equal to
the reception time of the ClusterConfig. For just mentioned ones,
this assumption will hold as AddOrUpdatePendingFolder() updates the
timestamp.
* lib/model, gui: Notify when one or more pending folders expired.
Introduce new event type FolderOfferCancelled and use it to trigger a
complete refreshCluster() cycle. Listing individual entries would be
much more code and probably just as much work to answer the API
request.
* lib/model: Add comment and rename ExpirePendingFolders().
* lib/events: Rename FolderOfferCancelled to ClusterPendingChanged.
* lib/model: Reuse ClusterPendingChanged event for cleanPending()
Changing the config does not necessarily mean that the
/resut/cluster/pending endpoints need to be refreshed, but only if
something was actually removed. Detect this and indicate it through
the ClusterPendingChanged event, which is already hooked up to requery
respective endpoints within the GUI.
No more need for a separate refreshCluster() in reaction to
ConfigSaved event or calling refreshConfig().
* lib/model: Gofmt.
* lib/db: Warn instead of info log for failed removal.
* gui: Fix pending notifications not loading on GUI start.
* lib/db: Use short device ID in log message.
* lib/db: Return list of expired folder IDs after deleting them.
* lib/model: Refactor Pending...Changed events.
* lib/model: Adjust format of removed pending folders enumeration.
Use an array of objects with device / folder ID properties, matching
the other places where it's used.
* lib/db: Drop invalid entries in RemovePendingFoldersBeforeTime().
* lib/model: Gofmt.
My local gofmt did not complain here, strangely...
* gui: Handle PendingDevicesChanged event.
Even though it currently only holds one device at a time, wrap the
contents in an array under the "added" property name.
* lib/model: Fix null values in PendingFoldersChanged removed member.
* gui: Handle PendingFoldersChanged event.
* lib/model: Simplify construction of expiredPendingList.
* lib/model: Reduce code duplication in cleanPending().
Use goto and a label for the common parts of calling the DB removal
function and building the event data part.
* lib/events, gui: Mark ...Rejected events deprecated.
Extend comments explaining the conditions when the replacement event
types are emitted.
* lib/model: Wrap removed devices in array of objects as well.
* lib/db: Use iter.Value() instead of needless db.Get(iter.Key())
* lib/db: Add comment explaining RemovePendingFoldersBeforeTime().
* lib/model: Rename fields folderID and deviceID in event data.
* lib/db: Only list actually expired IDs as removed.
Skip entries where Delete() failed as well as invalid entries that got
removed automatically.
* lib/model: Gofmt
2021-01-25 10:58:10 +00:00
continue
removeFolderForDevice :
2021-06-07 08:29:24 +00:00
if err := m . db . RemovePendingFolderForDevice ( folderID , deviceID ) ; err != nil {
2021-06-09 11:35:17 +00:00
msg := "Failed to remove pending folder-device entry"
l . Warnf ( "%v (%v, %v): %v" , msg , folderID , deviceID , err )
m . evLogger . Log ( events . Failure , msg )
2021-06-07 08:29:24 +00:00
continue
}
lib/model: Forget pending folders no longer announced in ClusterConfig (fixes #5187) (#7205)
* lib/db: Add ExpirePendingFolders().
Use-case is to drop any no-longer-pending folders for a specific
device when parsing its ClusterConfig message where previously offered
folders are not mentioned any more.
The timestamp in ObservedFolder is stored with only second precision,
so round to seconds here as well. This allows calling the function
within the same second of adding or updating entries.
* lib/model: Weed out pending folders when receiving ClusterConfig.
Filter the entries by timestamp, which must be newer than or equal to
the reception time of the ClusterConfig. For just mentioned ones,
this assumption will hold as AddOrUpdatePendingFolder() updates the
timestamp.
* lib/model, gui: Notify when one or more pending folders expired.
Introduce new event type FolderOfferCancelled and use it to trigger a
complete refreshCluster() cycle. Listing individual entries would be
much more code and probably just as much work to answer the API
request.
* lib/model: Add comment and rename ExpirePendingFolders().
* lib/events: Rename FolderOfferCancelled to ClusterPendingChanged.
* lib/model: Reuse ClusterPendingChanged event for cleanPending()
Changing the config does not necessarily mean that the
/resut/cluster/pending endpoints need to be refreshed, but only if
something was actually removed. Detect this and indicate it through
the ClusterPendingChanged event, which is already hooked up to requery
respective endpoints within the GUI.
No more need for a separate refreshCluster() in reaction to
ConfigSaved event or calling refreshConfig().
* lib/model: Gofmt.
* lib/db: Warn instead of info log for failed removal.
* gui: Fix pending notifications not loading on GUI start.
* lib/db: Use short device ID in log message.
* lib/db: Return list of expired folder IDs after deleting them.
* lib/model: Refactor Pending...Changed events.
* lib/model: Adjust format of removed pending folders enumeration.
Use an array of objects with device / folder ID properties, matching
the other places where it's used.
* lib/db: Drop invalid entries in RemovePendingFoldersBeforeTime().
* lib/model: Gofmt.
My local gofmt did not complain here, strangely...
* gui: Handle PendingDevicesChanged event.
Even though it currently only holds one device at a time, wrap the
contents in an array under the "added" property name.
* lib/model: Fix null values in PendingFoldersChanged removed member.
* gui: Handle PendingFoldersChanged event.
* lib/model: Simplify construction of expiredPendingList.
* lib/model: Reduce code duplication in cleanPending().
Use goto and a label for the common parts of calling the DB removal
function and building the event data part.
* lib/events, gui: Mark ...Rejected events deprecated.
Extend comments explaining the conditions when the replacement event
types are emitted.
* lib/model: Wrap removed devices in array of objects as well.
* lib/db: Use iter.Value() instead of needless db.Get(iter.Key())
* lib/db: Add comment explaining RemovePendingFoldersBeforeTime().
* lib/model: Rename fields folderID and deviceID in event data.
* lib/db: Only list actually expired IDs as removed.
Skip entries where Delete() failed as well as invalid entries that got
removed automatically.
* lib/model: Gofmt
2021-01-25 10:58:10 +00:00
removedPendingFolders = append ( removedPendingFolders , map [ string ] string {
"folderID" : folderID ,
"deviceID" : deviceID . String ( ) ,
} )
2020-12-17 18:54:31 +00:00
}
}
lib/model: Forget pending folders no longer announced in ClusterConfig (fixes #5187) (#7205)
* lib/db: Add ExpirePendingFolders().
Use-case is to drop any no-longer-pending folders for a specific
device when parsing its ClusterConfig message where previously offered
folders are not mentioned any more.
The timestamp in ObservedFolder is stored with only second precision,
so round to seconds here as well. This allows calling the function
within the same second of adding or updating entries.
* lib/model: Weed out pending folders when receiving ClusterConfig.
Filter the entries by timestamp, which must be newer than or equal to
the reception time of the ClusterConfig. For just mentioned ones,
this assumption will hold as AddOrUpdatePendingFolder() updates the
timestamp.
* lib/model, gui: Notify when one or more pending folders expired.
Introduce new event type FolderOfferCancelled and use it to trigger a
complete refreshCluster() cycle. Listing individual entries would be
much more code and probably just as much work to answer the API
request.
* lib/model: Add comment and rename ExpirePendingFolders().
* lib/events: Rename FolderOfferCancelled to ClusterPendingChanged.
* lib/model: Reuse ClusterPendingChanged event for cleanPending()
Changing the config does not necessarily mean that the
/resut/cluster/pending endpoints need to be refreshed, but only if
something was actually removed. Detect this and indicate it through
the ClusterPendingChanged event, which is already hooked up to requery
respective endpoints within the GUI.
No more need for a separate refreshCluster() in reaction to
ConfigSaved event or calling refreshConfig().
* lib/model: Gofmt.
* lib/db: Warn instead of info log for failed removal.
* gui: Fix pending notifications not loading on GUI start.
* lib/db: Use short device ID in log message.
* lib/db: Return list of expired folder IDs after deleting them.
* lib/model: Refactor Pending...Changed events.
* lib/model: Adjust format of removed pending folders enumeration.
Use an array of objects with device / folder ID properties, matching
the other places where it's used.
* lib/db: Drop invalid entries in RemovePendingFoldersBeforeTime().
* lib/model: Gofmt.
My local gofmt did not complain here, strangely...
* gui: Handle PendingDevicesChanged event.
Even though it currently only holds one device at a time, wrap the
contents in an array under the "added" property name.
* lib/model: Fix null values in PendingFoldersChanged removed member.
* gui: Handle PendingFoldersChanged event.
* lib/model: Simplify construction of expiredPendingList.
* lib/model: Reduce code duplication in cleanPending().
Use goto and a label for the common parts of calling the DB removal
function and building the event data part.
* lib/events, gui: Mark ...Rejected events deprecated.
Extend comments explaining the conditions when the replacement event
types are emitted.
* lib/model: Wrap removed devices in array of objects as well.
* lib/db: Use iter.Value() instead of needless db.Get(iter.Key())
* lib/db: Add comment explaining RemovePendingFoldersBeforeTime().
* lib/model: Rename fields folderID and deviceID in event data.
* lib/db: Only list actually expired IDs as removed.
Skip entries where Delete() failed as well as invalid entries that got
removed automatically.
* lib/model: Gofmt
2021-01-25 10:58:10 +00:00
if len ( removedPendingFolders ) > 0 {
m . evLogger . Log ( events . PendingFoldersChanged , map [ string ] interface { } {
"removed" : removedPendingFolders ,
} )
}
2020-12-17 18:54:31 +00:00
lib/model: Forget pending folders no longer announced in ClusterConfig (fixes #5187) (#7205)
* lib/db: Add ExpirePendingFolders().
Use-case is to drop any no-longer-pending folders for a specific
device when parsing its ClusterConfig message where previously offered
folders are not mentioned any more.
The timestamp in ObservedFolder is stored with only second precision,
so round to seconds here as well. This allows calling the function
within the same second of adding or updating entries.
* lib/model: Weed out pending folders when receiving ClusterConfig.
Filter the entries by timestamp, which must be newer than or equal to
the reception time of the ClusterConfig. For just mentioned ones,
this assumption will hold as AddOrUpdatePendingFolder() updates the
timestamp.
* lib/model, gui: Notify when one or more pending folders expired.
Introduce new event type FolderOfferCancelled and use it to trigger a
complete refreshCluster() cycle. Listing individual entries would be
much more code and probably just as much work to answer the API
request.
* lib/model: Add comment and rename ExpirePendingFolders().
* lib/events: Rename FolderOfferCancelled to ClusterPendingChanged.
* lib/model: Reuse ClusterPendingChanged event for cleanPending()
Changing the config does not necessarily mean that the
/resut/cluster/pending endpoints need to be refreshed, but only if
something was actually removed. Detect this and indicate it through
the ClusterPendingChanged event, which is already hooked up to requery
respective endpoints within the GUI.
No more need for a separate refreshCluster() in reaction to
ConfigSaved event or calling refreshConfig().
* lib/model: Gofmt.
* lib/db: Warn instead of info log for failed removal.
* gui: Fix pending notifications not loading on GUI start.
* lib/db: Use short device ID in log message.
* lib/db: Return list of expired folder IDs after deleting them.
* lib/model: Refactor Pending...Changed events.
* lib/model: Adjust format of removed pending folders enumeration.
Use an array of objects with device / folder ID properties, matching
the other places where it's used.
* lib/db: Drop invalid entries in RemovePendingFoldersBeforeTime().
* lib/model: Gofmt.
My local gofmt did not complain here, strangely...
* gui: Handle PendingDevicesChanged event.
Even though it currently only holds one device at a time, wrap the
contents in an array under the "added" property name.
* lib/model: Fix null values in PendingFoldersChanged removed member.
* gui: Handle PendingFoldersChanged event.
* lib/model: Simplify construction of expiredPendingList.
* lib/model: Reduce code duplication in cleanPending().
Use goto and a label for the common parts of calling the DB removal
function and building the event data part.
* lib/events, gui: Mark ...Rejected events deprecated.
Extend comments explaining the conditions when the replacement event
types are emitted.
* lib/model: Wrap removed devices in array of objects as well.
* lib/db: Use iter.Value() instead of needless db.Get(iter.Key())
* lib/db: Add comment explaining RemovePendingFoldersBeforeTime().
* lib/model: Rename fields folderID and deviceID in event data.
* lib/db: Only list actually expired IDs as removed.
Skip entries where Delete() failed as well as invalid entries that got
removed automatically.
* lib/model: Gofmt
2021-01-25 10:58:10 +00:00
var removedPendingDevices [ ] map [ string ] string
2020-12-17 18:54:31 +00:00
pendingDevices , err := m . db . PendingDevices ( )
if err != nil {
2021-06-09 11:35:17 +00:00
msg := "Could not iterate through pending device entries for cleanup"
l . Warnf ( "%v: %v" , msg , err )
m . evLogger . Log ( events . Failure , msg )
2020-12-17 18:54:31 +00:00
return
}
for deviceID := range pendingDevices {
if _ , ok := ignoredDevices [ deviceID ] ; ok {
l . Debugf ( "Discarding now ignored pending device %v" , deviceID )
lib/model: Forget pending folders no longer announced in ClusterConfig (fixes #5187) (#7205)
* lib/db: Add ExpirePendingFolders().
Use-case is to drop any no-longer-pending folders for a specific
device when parsing its ClusterConfig message where previously offered
folders are not mentioned any more.
The timestamp in ObservedFolder is stored with only second precision,
so round to seconds here as well. This allows calling the function
within the same second of adding or updating entries.
* lib/model: Weed out pending folders when receiving ClusterConfig.
Filter the entries by timestamp, which must be newer than or equal to
the reception time of the ClusterConfig. For just mentioned ones,
this assumption will hold as AddOrUpdatePendingFolder() updates the
timestamp.
* lib/model, gui: Notify when one or more pending folders expired.
Introduce new event type FolderOfferCancelled and use it to trigger a
complete refreshCluster() cycle. Listing individual entries would be
much more code and probably just as much work to answer the API
request.
* lib/model: Add comment and rename ExpirePendingFolders().
* lib/events: Rename FolderOfferCancelled to ClusterPendingChanged.
* lib/model: Reuse ClusterPendingChanged event for cleanPending()
Changing the config does not necessarily mean that the
/resut/cluster/pending endpoints need to be refreshed, but only if
something was actually removed. Detect this and indicate it through
the ClusterPendingChanged event, which is already hooked up to requery
respective endpoints within the GUI.
No more need for a separate refreshCluster() in reaction to
ConfigSaved event or calling refreshConfig().
* lib/model: Gofmt.
* lib/db: Warn instead of info log for failed removal.
* gui: Fix pending notifications not loading on GUI start.
* lib/db: Use short device ID in log message.
* lib/db: Return list of expired folder IDs after deleting them.
* lib/model: Refactor Pending...Changed events.
* lib/model: Adjust format of removed pending folders enumeration.
Use an array of objects with device / folder ID properties, matching
the other places where it's used.
* lib/db: Drop invalid entries in RemovePendingFoldersBeforeTime().
* lib/model: Gofmt.
My local gofmt did not complain here, strangely...
* gui: Handle PendingDevicesChanged event.
Even though it currently only holds one device at a time, wrap the
contents in an array under the "added" property name.
* lib/model: Fix null values in PendingFoldersChanged removed member.
* gui: Handle PendingFoldersChanged event.
* lib/model: Simplify construction of expiredPendingList.
* lib/model: Reduce code duplication in cleanPending().
Use goto and a label for the common parts of calling the DB removal
function and building the event data part.
* lib/events, gui: Mark ...Rejected events deprecated.
Extend comments explaining the conditions when the replacement event
types are emitted.
* lib/model: Wrap removed devices in array of objects as well.
* lib/db: Use iter.Value() instead of needless db.Get(iter.Key())
* lib/db: Add comment explaining RemovePendingFoldersBeforeTime().
* lib/model: Rename fields folderID and deviceID in event data.
* lib/db: Only list actually expired IDs as removed.
Skip entries where Delete() failed as well as invalid entries that got
removed automatically.
* lib/model: Gofmt
2021-01-25 10:58:10 +00:00
goto removeDevice
2020-12-17 18:54:31 +00:00
}
if _ , ok := existingDevices [ deviceID ] ; ok {
l . Debugf ( "Discarding now added pending device %v" , deviceID )
lib/model: Forget pending folders no longer announced in ClusterConfig (fixes #5187) (#7205)
* lib/db: Add ExpirePendingFolders().
Use-case is to drop any no-longer-pending folders for a specific
device when parsing its ClusterConfig message where previously offered
folders are not mentioned any more.
The timestamp in ObservedFolder is stored with only second precision,
so round to seconds here as well. This allows calling the function
within the same second of adding or updating entries.
* lib/model: Weed out pending folders when receiving ClusterConfig.
Filter the entries by timestamp, which must be newer than or equal to
the reception time of the ClusterConfig. For just mentioned ones,
this assumption will hold as AddOrUpdatePendingFolder() updates the
timestamp.
* lib/model, gui: Notify when one or more pending folders expired.
Introduce new event type FolderOfferCancelled and use it to trigger a
complete refreshCluster() cycle. Listing individual entries would be
much more code and probably just as much work to answer the API
request.
* lib/model: Add comment and rename ExpirePendingFolders().
* lib/events: Rename FolderOfferCancelled to ClusterPendingChanged.
* lib/model: Reuse ClusterPendingChanged event for cleanPending()
Changing the config does not necessarily mean that the
/resut/cluster/pending endpoints need to be refreshed, but only if
something was actually removed. Detect this and indicate it through
the ClusterPendingChanged event, which is already hooked up to requery
respective endpoints within the GUI.
No more need for a separate refreshCluster() in reaction to
ConfigSaved event or calling refreshConfig().
* lib/model: Gofmt.
* lib/db: Warn instead of info log for failed removal.
* gui: Fix pending notifications not loading on GUI start.
* lib/db: Use short device ID in log message.
* lib/db: Return list of expired folder IDs after deleting them.
* lib/model: Refactor Pending...Changed events.
* lib/model: Adjust format of removed pending folders enumeration.
Use an array of objects with device / folder ID properties, matching
the other places where it's used.
* lib/db: Drop invalid entries in RemovePendingFoldersBeforeTime().
* lib/model: Gofmt.
My local gofmt did not complain here, strangely...
* gui: Handle PendingDevicesChanged event.
Even though it currently only holds one device at a time, wrap the
contents in an array under the "added" property name.
* lib/model: Fix null values in PendingFoldersChanged removed member.
* gui: Handle PendingFoldersChanged event.
* lib/model: Simplify construction of expiredPendingList.
* lib/model: Reduce code duplication in cleanPending().
Use goto and a label for the common parts of calling the DB removal
function and building the event data part.
* lib/events, gui: Mark ...Rejected events deprecated.
Extend comments explaining the conditions when the replacement event
types are emitted.
* lib/model: Wrap removed devices in array of objects as well.
* lib/db: Use iter.Value() instead of needless db.Get(iter.Key())
* lib/db: Add comment explaining RemovePendingFoldersBeforeTime().
* lib/model: Rename fields folderID and deviceID in event data.
* lib/db: Only list actually expired IDs as removed.
Skip entries where Delete() failed as well as invalid entries that got
removed automatically.
* lib/model: Gofmt
2021-01-25 10:58:10 +00:00
goto removeDevice
2020-12-17 18:54:31 +00:00
}
lib/model: Forget pending folders no longer announced in ClusterConfig (fixes #5187) (#7205)
* lib/db: Add ExpirePendingFolders().
Use-case is to drop any no-longer-pending folders for a specific
device when parsing its ClusterConfig message where previously offered
folders are not mentioned any more.
The timestamp in ObservedFolder is stored with only second precision,
so round to seconds here as well. This allows calling the function
within the same second of adding or updating entries.
* lib/model: Weed out pending folders when receiving ClusterConfig.
Filter the entries by timestamp, which must be newer than or equal to
the reception time of the ClusterConfig. For just mentioned ones,
this assumption will hold as AddOrUpdatePendingFolder() updates the
timestamp.
* lib/model, gui: Notify when one or more pending folders expired.
Introduce new event type FolderOfferCancelled and use it to trigger a
complete refreshCluster() cycle. Listing individual entries would be
much more code and probably just as much work to answer the API
request.
* lib/model: Add comment and rename ExpirePendingFolders().
* lib/events: Rename FolderOfferCancelled to ClusterPendingChanged.
* lib/model: Reuse ClusterPendingChanged event for cleanPending()
Changing the config does not necessarily mean that the
/resut/cluster/pending endpoints need to be refreshed, but only if
something was actually removed. Detect this and indicate it through
the ClusterPendingChanged event, which is already hooked up to requery
respective endpoints within the GUI.
No more need for a separate refreshCluster() in reaction to
ConfigSaved event or calling refreshConfig().
* lib/model: Gofmt.
* lib/db: Warn instead of info log for failed removal.
* gui: Fix pending notifications not loading on GUI start.
* lib/db: Use short device ID in log message.
* lib/db: Return list of expired folder IDs after deleting them.
* lib/model: Refactor Pending...Changed events.
* lib/model: Adjust format of removed pending folders enumeration.
Use an array of objects with device / folder ID properties, matching
the other places where it's used.
* lib/db: Drop invalid entries in RemovePendingFoldersBeforeTime().
* lib/model: Gofmt.
My local gofmt did not complain here, strangely...
* gui: Handle PendingDevicesChanged event.
Even though it currently only holds one device at a time, wrap the
contents in an array under the "added" property name.
* lib/model: Fix null values in PendingFoldersChanged removed member.
* gui: Handle PendingFoldersChanged event.
* lib/model: Simplify construction of expiredPendingList.
* lib/model: Reduce code duplication in cleanPending().
Use goto and a label for the common parts of calling the DB removal
function and building the event data part.
* lib/events, gui: Mark ...Rejected events deprecated.
Extend comments explaining the conditions when the replacement event
types are emitted.
* lib/model: Wrap removed devices in array of objects as well.
* lib/db: Use iter.Value() instead of needless db.Get(iter.Key())
* lib/db: Add comment explaining RemovePendingFoldersBeforeTime().
* lib/model: Rename fields folderID and deviceID in event data.
* lib/db: Only list actually expired IDs as removed.
Skip entries where Delete() failed as well as invalid entries that got
removed automatically.
* lib/model: Gofmt
2021-01-25 10:58:10 +00:00
continue
removeDevice :
2021-06-07 08:29:24 +00:00
if err := m . db . RemovePendingDevice ( deviceID ) ; err != nil {
2021-06-09 11:35:17 +00:00
msg := "Failed to remove pending device entry"
l . Warnf ( "%v: %v" , msg , err )
m . evLogger . Log ( events . Failure , msg )
2021-06-07 08:29:24 +00:00
continue
}
lib/model: Forget pending folders no longer announced in ClusterConfig (fixes #5187) (#7205)
* lib/db: Add ExpirePendingFolders().
Use-case is to drop any no-longer-pending folders for a specific
device when parsing its ClusterConfig message where previously offered
folders are not mentioned any more.
The timestamp in ObservedFolder is stored with only second precision,
so round to seconds here as well. This allows calling the function
within the same second of adding or updating entries.
* lib/model: Weed out pending folders when receiving ClusterConfig.
Filter the entries by timestamp, which must be newer than or equal to
the reception time of the ClusterConfig. For just mentioned ones,
this assumption will hold as AddOrUpdatePendingFolder() updates the
timestamp.
* lib/model, gui: Notify when one or more pending folders expired.
Introduce new event type FolderOfferCancelled and use it to trigger a
complete refreshCluster() cycle. Listing individual entries would be
much more code and probably just as much work to answer the API
request.
* lib/model: Add comment and rename ExpirePendingFolders().
* lib/events: Rename FolderOfferCancelled to ClusterPendingChanged.
* lib/model: Reuse ClusterPendingChanged event for cleanPending()
Changing the config does not necessarily mean that the
/resut/cluster/pending endpoints need to be refreshed, but only if
something was actually removed. Detect this and indicate it through
the ClusterPendingChanged event, which is already hooked up to requery
respective endpoints within the GUI.
No more need for a separate refreshCluster() in reaction to
ConfigSaved event or calling refreshConfig().
* lib/model: Gofmt.
* lib/db: Warn instead of info log for failed removal.
* gui: Fix pending notifications not loading on GUI start.
* lib/db: Use short device ID in log message.
* lib/db: Return list of expired folder IDs after deleting them.
* lib/model: Refactor Pending...Changed events.
* lib/model: Adjust format of removed pending folders enumeration.
Use an array of objects with device / folder ID properties, matching
the other places where it's used.
* lib/db: Drop invalid entries in RemovePendingFoldersBeforeTime().
* lib/model: Gofmt.
My local gofmt did not complain here, strangely...
* gui: Handle PendingDevicesChanged event.
Even though it currently only holds one device at a time, wrap the
contents in an array under the "added" property name.
* lib/model: Fix null values in PendingFoldersChanged removed member.
* gui: Handle PendingFoldersChanged event.
* lib/model: Simplify construction of expiredPendingList.
* lib/model: Reduce code duplication in cleanPending().
Use goto and a label for the common parts of calling the DB removal
function and building the event data part.
* lib/events, gui: Mark ...Rejected events deprecated.
Extend comments explaining the conditions when the replacement event
types are emitted.
* lib/model: Wrap removed devices in array of objects as well.
* lib/db: Use iter.Value() instead of needless db.Get(iter.Key())
* lib/db: Add comment explaining RemovePendingFoldersBeforeTime().
* lib/model: Rename fields folderID and deviceID in event data.
* lib/db: Only list actually expired IDs as removed.
Skip entries where Delete() failed as well as invalid entries that got
removed automatically.
* lib/model: Gofmt
2021-01-25 10:58:10 +00:00
removedPendingDevices = append ( removedPendingDevices , map [ string ] string {
"deviceID" : deviceID . String ( ) ,
} )
}
if len ( removedPendingDevices ) > 0 {
m . evLogger . Log ( events . PendingDevicesChanged , map [ string ] interface { } {
"removed" : removedPendingDevices ,
} )
2020-12-17 18:54:31 +00:00
}
}
2017-12-15 20:01:56 +00:00
// checkFolderRunningLocked returns nil if the folder is up and running and a
// descriptive error if not.
// Need to hold (read) lock on m.fmut when calling this.
2019-02-26 08:09:25 +00:00
func ( m * model ) checkFolderRunningLocked ( folder string ) error {
2023-09-02 14:42:46 +00:00
_ , ok := m . folderRunners . Get ( folder )
2017-12-15 20:01:56 +00:00
if ok {
return nil
}
if cfg , ok := m . cfg . Folder ( folder ) ; ! ok {
2021-03-07 12:43:22 +00:00
return ErrFolderMissing
2017-12-15 20:01:56 +00:00
} else if cfg . Paused {
2018-01-14 17:01:06 +00:00
return ErrFolderPaused
2017-12-15 20:01:56 +00:00
}
2021-03-07 12:43:22 +00:00
return ErrFolderNotRunning
2017-12-15 20:01:56 +00:00
}
2020-12-17 18:54:31 +00:00
// PendingDevices lists unknown devices that tried to connect.
func ( m * model ) PendingDevices ( ) ( map [ protocol . DeviceID ] db . ObservedDevice , error ) {
return m . db . PendingDevices ( )
}
// PendingFolders lists folders that we don't yet share with the offering devices. It
// returns the entries grouped by folder and filters for a given device unless the
// argument is specified as EmptyDeviceID.
func ( m * model ) PendingFolders ( device protocol . DeviceID ) ( map [ string ] db . PendingFolder , error ) {
return m . db . PendingFoldersForDevice ( device )
}
2021-06-07 08:29:24 +00:00
// DismissPendingDevices removes the record of a specific pending device.
func ( m * model ) DismissPendingDevice ( device protocol . DeviceID ) error {
l . Debugf ( "Discarding pending device %v" , device )
err := m . db . RemovePendingDevice ( device )
if err != nil {
return err
}
removedPendingDevices := [ ] map [ string ] string {
{ "deviceID" : device . String ( ) } ,
}
m . evLogger . Log ( events . PendingDevicesChanged , map [ string ] interface { } {
"removed" : removedPendingDevices ,
} )
return nil
}
// DismissPendingFolders removes records of pending folders. Either a specific folder /
// device combination, or all matching a specific folder ID if the device argument is
// specified as EmptyDeviceID.
func ( m * model ) DismissPendingFolder ( device protocol . DeviceID , folder string ) error {
var removedPendingFolders [ ] map [ string ] string
if device == protocol . EmptyDeviceID {
l . Debugf ( "Discarding pending removed folder %s from all devices" , folder )
err := m . db . RemovePendingFolder ( folder )
if err != nil {
return err
}
removedPendingFolders = [ ] map [ string ] string {
{ "folderID" : folder } ,
}
} else {
l . Debugf ( "Discarding pending folder %s from device %v" , folder , device )
err := m . db . RemovePendingFolderForDevice ( folder , device )
if err != nil {
return err
}
removedPendingFolders = [ ] map [ string ] string {
{
"folderID" : folder ,
"deviceID" : device . String ( ) ,
} ,
}
}
if len ( removedPendingFolders ) > 0 {
m . evLogger . Log ( events . PendingFoldersChanged , map [ string ] interface { } {
"removed" : removedPendingFolders ,
} )
}
return nil
}
2015-07-22 07:02:55 +00:00
// mapFolders returns a map of folder ID to folder configuration for the given
// slice of folder configurations.
func mapFolders ( folders [ ] config . FolderConfiguration ) map [ string ] config . FolderConfiguration {
m := make ( map [ string ] config . FolderConfiguration , len ( folders ) )
for _ , cfg := range folders {
m [ cfg . ID ] = cfg
}
return m
}
// mapDevices returns a map of device ID to nothing for the given slice of
// device IDs.
func mapDevices ( devices [ ] protocol . DeviceID ) map [ protocol . DeviceID ] struct { } {
m := make ( map [ protocol . DeviceID ] struct { } , len ( devices ) )
for _ , dev := range devices {
m [ dev ] = struct { } { }
}
return m
}
2020-12-17 18:54:31 +00:00
func observedDeviceSet ( devices [ ] config . ObservedDevice ) deviceIDSet {
res := make ( deviceIDSet , len ( devices ) )
for _ , dev := range devices {
res [ dev . ID ] = struct { } { }
}
return res
}
2021-02-05 15:07:21 +00:00
func readOffsetIntoBuf ( fs fs . Filesystem , file string , offset int64 , buf [ ] byte ) ( int , error ) {
2017-08-19 14:36:56 +00:00
fd , err := fs . Open ( file )
2016-04-15 10:59:41 +00:00
if err != nil {
l . Debugln ( "readOffsetIntoBuf.Open" , file , err )
2021-02-05 15:07:21 +00:00
return 0 , err
2016-04-15 10:59:41 +00:00
}
defer fd . Close ( )
2021-02-05 15:07:21 +00:00
n , err := fd . ReadAt ( buf , offset )
2016-04-15 10:59:41 +00:00
if err != nil {
l . Debugln ( "readOffsetIntoBuf.ReadAt" , file , err )
}
2021-02-05 15:07:21 +00:00
return n , err
2016-04-15 10:59:41 +00:00
}
2016-11-07 16:40:48 +00:00
// folderDeviceSet is a set of (folder, deviceID) pairs
type folderDeviceSet map [ string ] map [ protocol . DeviceID ] struct { }
// set adds the (dev, folder) pair to the set
func ( s folderDeviceSet ) set ( dev protocol . DeviceID , folder string ) {
devs , ok := s [ folder ]
if ! ok {
devs = make ( map [ protocol . DeviceID ] struct { } )
s [ folder ] = devs
}
devs [ dev ] = struct { } { }
}
// has returns true if the (dev, folder) pair is in the set
func ( s folderDeviceSet ) has ( dev protocol . DeviceID , folder string ) bool {
_ , ok := s [ folder ] [ dev ]
return ok
}
// hasDevice returns true if the device is set on any folder
func ( s folderDeviceSet ) hasDevice ( dev protocol . DeviceID ) bool {
for _ , devices := range s {
if _ , ok := devices [ dev ] ; ok {
return true
}
}
return false
}
2018-08-25 08:32:35 +00:00
2018-10-05 08:26:25 +00:00
// syncMutexMap is a type safe wrapper for a sync.Map that holds mutexes
type syncMutexMap struct {
inner stdsync . Map
}
func ( m * syncMutexMap ) Get ( key string ) sync . Mutex {
v , _ := m . inner . LoadOrStore ( key , sync . NewMutex ( ) )
return v . ( sync . Mutex )
}
2019-01-05 17:10:02 +00:00
2020-11-20 14:53:13 +00:00
type deviceIDSet map [ protocol . DeviceID ] struct { }
func ( s deviceIDSet ) add ( ids [ ] protocol . DeviceID ) {
for _ , id := range ids {
if _ , ok := s [ id ] ; ! ok {
s [ id ] = struct { } { }
2020-10-02 09:49:51 +00:00
}
}
2020-11-20 14:53:13 +00:00
}
func ( s deviceIDSet ) AsSlice ( ) [ ] protocol . DeviceID {
ids := make ( [ ] protocol . DeviceID , 0 , len ( s ) )
for id := range s {
ids = append ( ids , id )
}
return ids
2020-10-02 09:49:51 +00:00
}
2020-11-09 14:33:32 +00:00
func encryptionTokenPath ( cfg config . FolderConfiguration ) string {
2021-02-12 07:38:43 +00:00
return filepath . Join ( cfg . MarkerName , config . EncryptionTokenName )
2020-11-09 14:33:32 +00:00
}
type storedEncryptionToken struct {
FolderID string
Token [ ] byte
}
func readEncryptionToken ( cfg config . FolderConfiguration ) ( [ ] byte , error ) {
2022-04-10 18:55:05 +00:00
fd , err := cfg . Filesystem ( nil ) . Open ( encryptionTokenPath ( cfg ) )
2020-11-09 14:33:32 +00:00
if err != nil {
return nil , err
}
defer fd . Close ( )
var stored storedEncryptionToken
if err := json . NewDecoder ( fd ) . Decode ( & stored ) ; err != nil {
return nil , err
}
return stored . Token , nil
}
func writeEncryptionToken ( token [ ] byte , cfg config . FolderConfiguration ) error {
tokenName := encryptionTokenPath ( cfg )
2023-03-12 19:06:59 +00:00
fd , err := cfg . Filesystem ( nil ) . OpenFile ( tokenName , fs . OptReadWrite | fs . OptCreate , 0 o666 )
2020-11-09 14:33:32 +00:00
if err != nil {
return err
}
defer fd . Close ( )
return json . NewEncoder ( fd ) . Encode ( storedEncryptionToken {
FolderID : cfg . ID ,
Token : token ,
} )
}
2021-02-04 20:10:41 +00:00
func newFolderConfiguration ( w config . Wrapper , id , label string , fsType fs . FilesystemType , path string ) config . FolderConfiguration {
fcfg := w . DefaultFolder ( )
fcfg . ID = id
fcfg . Label = label
fcfg . FilesystemType = fsType
fcfg . Path = path
return fcfg
}
2021-02-12 21:51:29 +00:00
type updatedPendingFolder struct {
FolderID string ` json:"folderID" `
FolderLabel string ` json:"folderLabel" `
DeviceID protocol . DeviceID ` json:"deviceID" `
ReceiveEncrypted bool ` json:"receiveEncrypted" `
2021-06-17 11:53:02 +00:00
RemoteEncrypted bool ` json:"remoteEncrypted" `
2021-02-12 21:51:29 +00:00
}
2021-04-29 17:21:07 +00:00
// redactPathError checks if the error is actually a os.PathError, and if yes
// returns a redactedError with the path removed.
func redactPathError ( err error ) ( error , bool ) {
perr , ok := err . ( * os . PathError )
if ! ok {
return nil , false
}
return & redactedError {
error : err ,
redacted : fmt . Errorf ( "%v: %w" , perr . Op , perr . Err ) ,
} , true
}
type redactedError struct {
error
redacted error
}
2023-09-06 10:52:01 +00:00
func without [ E comparable , S ~ [ ] E ] ( s S , e E ) S {
for i , x := range s {
if x == e {
return append ( s [ : i ] , s [ i + 1 : ] ... )
}
}
return s
}