mirror of
https://github.com/octoleo/syncthing.git
synced 2024-09-19 21:29:01 +00:00
Make parallel hasher configurable, remove finisher setting (fixes #1199)
This commit is contained in:
parent
5d173168cc
commit
5827a686b8
@ -60,9 +60,9 @@ type FolderConfiguration struct {
|
|||||||
IgnorePerms bool `xml:"ignorePerms,attr"`
|
IgnorePerms bool `xml:"ignorePerms,attr"`
|
||||||
Versioning VersioningConfiguration `xml:"versioning"`
|
Versioning VersioningConfiguration `xml:"versioning"`
|
||||||
LenientMtimes bool `xml:"lenientMtimes"`
|
LenientMtimes bool `xml:"lenientMtimes"`
|
||||||
Copiers int `xml:"copiers" default:"1"` // This defines how many files are handled concurrently.
|
Copiers int `xml:"copiers" default:"1"` // This defines how many files are handled concurrently.
|
||||||
Pullers int `xml:"pullers" default:"16"` // Defines how many blocks are fetched at the same time, possibly between separate copier routines.
|
Pullers int `xml:"pullers" default:"16"` // Defines how many blocks are fetched at the same time, possibly between separate copier routines.
|
||||||
Finishers int `xml:"finishers" default:"1"` // Most of the time, should be equal to the number of copiers. These are CPU bound due to hashing.
|
Hashers int `xml:"hashers" default:"0"` // Less than one sets the value to the number of cores. These are CPU bound due to hashing.
|
||||||
|
|
||||||
Invalid string `xml:"-"` // Set at runtime when there is an error, not saved
|
Invalid string `xml:"-"` // Set at runtime when there is an error, not saved
|
||||||
|
|
||||||
@ -358,9 +358,6 @@ func (cfg *Configuration) prepare(myID protocol.DeviceID) {
|
|||||||
if cfg.Folders[i].Pullers == 0 {
|
if cfg.Folders[i].Pullers == 0 {
|
||||||
cfg.Folders[i].Pullers = 16
|
cfg.Folders[i].Pullers = 16
|
||||||
}
|
}
|
||||||
if cfg.Folders[i].Finishers == 0 {
|
|
||||||
cfg.Folders[i].Finishers = 1
|
|
||||||
}
|
|
||||||
sort.Sort(FolderDeviceConfigurationList(cfg.Folders[i].Devices))
|
sort.Sort(FolderDeviceConfigurationList(cfg.Folders[i].Devices))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -89,7 +89,7 @@ func TestDeviceConfig(t *testing.T) {
|
|||||||
RescanIntervalS: 600,
|
RescanIntervalS: 600,
|
||||||
Copiers: 1,
|
Copiers: 1,
|
||||||
Pullers: 16,
|
Pullers: 16,
|
||||||
Finishers: 1,
|
Hashers: 0,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
expectedDevices := []DeviceConfiguration{
|
expectedDevices := []DeviceConfiguration{
|
||||||
|
@ -190,7 +190,6 @@ func (m *Model) StartFolderRW(folder string) {
|
|||||||
progressEmitter: m.progressEmitter,
|
progressEmitter: m.progressEmitter,
|
||||||
copiers: cfg.Copiers,
|
copiers: cfg.Copiers,
|
||||||
pullers: cfg.Pullers,
|
pullers: cfg.Pullers,
|
||||||
finishers: cfg.Finishers,
|
|
||||||
queue: newJobQueue(),
|
queue: newJobQueue(),
|
||||||
}
|
}
|
||||||
m.folderRunners[folder] = p
|
m.folderRunners[folder] = p
|
||||||
@ -1138,6 +1137,7 @@ func (m *Model) ScanFolderSub(folder, sub string) error {
|
|||||||
TempLifetime: time.Duration(m.cfg.Options().KeepTemporariesH) * time.Hour,
|
TempLifetime: time.Duration(m.cfg.Options().KeepTemporariesH) * time.Hour,
|
||||||
CurrentFiler: cFiler{m, folder},
|
CurrentFiler: cFiler{m, folder},
|
||||||
IgnorePerms: folderCfg.IgnorePerms,
|
IgnorePerms: folderCfg.IgnorePerms,
|
||||||
|
Hashers: folderCfg.Hashers,
|
||||||
}
|
}
|
||||||
|
|
||||||
m.setState(folder, FolderScanning)
|
m.setState(folder, FolderScanning)
|
||||||
|
@ -77,7 +77,6 @@ type Puller struct {
|
|||||||
progressEmitter *ProgressEmitter
|
progressEmitter *ProgressEmitter
|
||||||
copiers int
|
copiers int
|
||||||
pullers int
|
pullers int
|
||||||
finishers int
|
|
||||||
queue *jobQueue
|
queue *jobQueue
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -258,7 +257,7 @@ func (p *Puller) pullerIteration(ignores *ignore.Matcher) int {
|
|||||||
var doneWg sync.WaitGroup
|
var doneWg sync.WaitGroup
|
||||||
|
|
||||||
if debug {
|
if debug {
|
||||||
l.Debugln(p, "c", p.copiers, "p", p.pullers, "f", p.finishers)
|
l.Debugln(p, "c", p.copiers, "p", p.pullers)
|
||||||
}
|
}
|
||||||
|
|
||||||
for i := 0; i < p.copiers; i++ {
|
for i := 0; i < p.copiers; i++ {
|
||||||
@ -279,14 +278,12 @@ func (p *Puller) pullerIteration(ignores *ignore.Matcher) int {
|
|||||||
}()
|
}()
|
||||||
}
|
}
|
||||||
|
|
||||||
for i := 0; i < p.finishers; i++ {
|
doneWg.Add(1)
|
||||||
doneWg.Add(1)
|
// finisherRoutine finishes when finisherChan is closed
|
||||||
// finisherRoutine finishes when finisherChan is closed
|
go func() {
|
||||||
go func() {
|
p.finisherRoutine(finisherChan)
|
||||||
p.finisherRoutine(finisherChan)
|
doneWg.Done()
|
||||||
doneWg.Done()
|
}()
|
||||||
}()
|
|
||||||
}
|
|
||||||
|
|
||||||
p.model.fmut.RLock()
|
p.model.fmut.RLock()
|
||||||
folderFiles := p.model.folderFiles[p.folder]
|
folderFiles := p.model.folderFiles[p.folder]
|
||||||
|
@ -49,6 +49,8 @@ type Walker struct {
|
|||||||
// detected. Scanned files will get zero permission bits and the
|
// detected. Scanned files will get zero permission bits and the
|
||||||
// NoPermissionBits flag set.
|
// NoPermissionBits flag set.
|
||||||
IgnorePerms bool
|
IgnorePerms bool
|
||||||
|
// Number of routines to use for hashing
|
||||||
|
Hashers int
|
||||||
}
|
}
|
||||||
|
|
||||||
type TempNamer interface {
|
type TempNamer interface {
|
||||||
@ -75,9 +77,14 @@ func (w *Walker) Walk() (chan protocol.FileInfo, error) {
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
workers := w.Hashers
|
||||||
|
if workers < 1 {
|
||||||
|
workers = runtime.NumCPU()
|
||||||
|
}
|
||||||
|
|
||||||
files := make(chan protocol.FileInfo)
|
files := make(chan protocol.FileInfo)
|
||||||
hashedFiles := make(chan protocol.FileInfo)
|
hashedFiles := make(chan protocol.FileInfo)
|
||||||
newParallelHasher(w.Dir, w.BlockSize, runtime.NumCPU(), hashedFiles, files)
|
newParallelHasher(w.Dir, w.BlockSize, workers, hashedFiles, files)
|
||||||
|
|
||||||
go func() {
|
go func() {
|
||||||
hashFiles := w.walkAndHashFiles(files)
|
hashFiles := w.walkAndHashFiles(files)
|
||||||
|
Loading…
Reference in New Issue
Block a user