Merge pull request #1345 from calmh/smaller-batches

Reduce memory usage by writing smaller batches
This commit is contained in:
Audrius Butkevicius 2015-02-10 19:53:53 +00:00
commit 2edaf22590

View File

@ -102,6 +102,9 @@ type dbWriter interface {
Delete([]byte)
}
// Flush batches to disk when they contain this many records.
const batchFlushSize = 64
// deviceKey returns a byte slice encoding the following information:
// keyTypeDevice (1 byte)
// folder (64 bytes)
@ -275,6 +278,21 @@ func ldbGenericReplace(db *leveldb.DB, folder, device []byte, fs []protocol.File
}
moreDb = dbi.Next()
}
// Write out and reuse the batch every few records, to avoid the batch
// growing too large and thus allocating unnecessarily much memory.
if batch.Len() > batchFlushSize {
if debugDB {
l.Debugf("db.Write %p", batch)
}
err = db.Write(batch, nil)
if err != nil {
panic(err)
}
batch.Reset()
}
}
if debugDB {
@ -393,6 +411,21 @@ func ldbUpdate(db *leveldb.DB, folder, device []byte, fs []protocol.FileInfo) in
ldbUpdateGlobal(snap, batch, folder, device, name, f.Version)
}
}
// Write out and reuse the batch every few records, to avoid the batch
// growing too large and thus allocating unnecessarily much memory.
if batch.Len() > batchFlushSize {
if debugDB {
l.Debugf("db.Write %p", batch)
}
err = db.Write(batch, nil)
if err != nil {
panic(err)
}
batch.Reset()
}
}
if debugDB {