From 9bb928bb38070997b7a5deda75c02a3f2df44ec0 Mon Sep 17 00:00:00 2001
From: Jakob Borg <jakob@nym.se>
Date: Tue, 10 Feb 2015 20:12:17 +0100
Subject: [PATCH] Reduce memory usage by writing smaller batches

---
 internal/db/leveldb.go | 33 +++++++++++++++++++++++++++++++++
 1 file changed, 33 insertions(+)

diff --git a/internal/db/leveldb.go b/internal/db/leveldb.go
index 99b83ba0b..775b273ec 100644
--- a/internal/db/leveldb.go
+++ b/internal/db/leveldb.go
@@ -102,6 +102,9 @@ type dbWriter interface {
 	Delete([]byte)
 }
 
+// Flush batches to disk when they contain this many records.
+const batchFlushSize = 64
+
 // deviceKey returns a byte slice encoding the following information:
 //	   keyTypeDevice (1 byte)
 //	   folder (64 bytes)
@@ -275,6 +278,21 @@ func ldbGenericReplace(db *leveldb.DB, folder, device []byte, fs []protocol.File
 			}
 			moreDb = dbi.Next()
 		}
+
+		// Write out and reuse the batch every few records, to avoid the batch
+		// growing too large and thus allocating unnecessarily much memory.
+		if batch.Len() > batchFlushSize {
+			if debugDB {
+				l.Debugf("db.Write %p", batch)
+			}
+
+			err = db.Write(batch, nil)
+			if err != nil {
+				panic(err)
+			}
+
+			batch.Reset()
+		}
 	}
 
 	if debugDB {
@@ -393,6 +411,21 @@ func ldbUpdate(db *leveldb.DB, folder, device []byte, fs []protocol.FileInfo) in
 				ldbUpdateGlobal(snap, batch, folder, device, name, f.Version)
 			}
 		}
+
+		// Write out and reuse the batch every few records, to avoid the batch
+		// growing too large and thus allocating unnecessarily much memory.
+		if batch.Len() > batchFlushSize {
+			if debugDB {
+				l.Debugf("db.Write %p", batch)
+			}
+
+			err = db.Write(batch, nil)
+			if err != nil {
+				panic(err)
+			}
+
+			batch.Reset()
+		}
 	}
 
 	if debugDB {