From cebe21a3af14df1d84c119b6d0e66effe619e14a Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Fri, 17 Apr 2015 15:19:40 +0900 Subject: [PATCH] Don't buffer large files a long time while scanning --- internal/model/model.go | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/internal/model/model.go b/internal/model/model.go index f5b97b17b..499dc1f6b 100644 --- a/internal/model/model.go +++ b/internal/model/model.go @@ -1194,18 +1194,24 @@ nextSub: return err } - batchSize := 100 - batch := make([]protocol.FileInfo, 0, batchSize) + batchSizeFiles := 100 + batchSizeBlocks := 2048 // about 256 MB + + batch := make([]protocol.FileInfo, 0, batchSizeFiles) + blocksHandled := 0 + for f := range fchan { - if len(batch) == batchSize { + if len(batch) == batchSizeFiles || blocksHandled > batchSizeBlocks { if err := m.CheckFolderHealth(folder); err != nil { l.Infof("Stopping folder %s mid-scan due to folder error: %s", folder, err) return err } m.updateLocals(folder, batch) batch = batch[:0] + blocksHandled = 0 } batch = append(batch, f) + blocksHandled += len(f.Blocks) } if err := m.CheckFolderHealth(folder); err != nil { @@ -1240,7 +1246,7 @@ nextSub: return true } - if len(batch) == batchSize { + if len(batch) == batchSizeFiles { m.updateLocals(folder, batch) batch = batch[:0] }