mirror of
https://github.com/octoleo/syncthing.git
synced 2024-11-09 14:50:56 +00:00
Also keep GlobalSize in RAM
This commit is contained in:
Jakob Borg
parent
d4f81e8791
commit
c268e4ad1b
@ -167,13 +167,9 @@ func globalKeyFolder(key []byte) []byte {
|
||||
return folder[:izero]
|
||||
}
|
||||
|
||||
func isLocalDevice(dev []byte) bool {
|
||||
return bytes.Equal(dev, protocol.LocalDeviceID[:])
|
||||
}
|
||||
|
||||
type deletionHandler func(db dbReader, batch dbWriter, folder, device, name []byte, dbi iterator.Iterator) int64
|
||||
|
||||
func ldbGenericReplace(db *leveldb.DB, folder, device []byte, fs []protocol.FileInfo, deleteFn deletionHandler) int64 {
|
||||
func ldbGenericReplace(db *leveldb.DB, folder, device []byte, fs []protocol.FileInfo, localSize, globalSize *sizeTracker, deleteFn deletionHandler) int64 {
|
||||
runtime.GC()
|
||||
|
||||
sort.Sort(fileList(fs)) // sort list on name, same as in the database
|
||||
@ -200,6 +196,7 @@ func ldbGenericReplace(db *leveldb.DB, folder, device []byte, fs []protocol.File
|
||||
fsi := 0
|
||||
var maxLocalVer int64
|
||||
|
||||
isLocalDevice := bytes.Equal(device, protocol.LocalDeviceID[:])
|
||||
for {
|
||||
var newName, oldName []byte
|
||||
moreFs := fsi < len(fs)
|
||||
@ -227,10 +224,13 @@ func ldbGenericReplace(db *leveldb.DB, folder, device []byte, fs []protocol.File
|
||||
if lv := ldbInsert(batch, folder, device, fs[fsi]); lv > maxLocalVer {
|
||||
maxLocalVer = lv
|
||||
}
|
||||
if isLocalDevice {
|
||||
localSize.addFile(fs[fsi])
|
||||
}
|
||||
if fs[fsi].IsInvalid() {
|
||||
ldbRemoveFromGlobal(snap, batch, folder, device, newName)
|
||||
ldbRemoveFromGlobal(snap, batch, folder, device, newName, globalSize)
|
||||
} else {
|
||||
ldbUpdateGlobal(snap, batch, folder, device, fs[fsi])
|
||||
ldbUpdateGlobal(snap, batch, folder, device, fs[fsi], globalSize)
|
||||
}
|
||||
fsi++
|
||||
|
||||
@ -246,10 +246,14 @@ func ldbGenericReplace(db *leveldb.DB, folder, device []byte, fs []protocol.File
|
||||
if lv := ldbInsert(batch, folder, device, fs[fsi]); lv > maxLocalVer {
|
||||
maxLocalVer = lv
|
||||
}
|
||||
if isLocalDevice {
|
||||
localSize.removeFile(ef)
|
||||
localSize.addFile(fs[fsi])
|
||||
}
|
||||
if fs[fsi].IsInvalid() {
|
||||
ldbRemoveFromGlobal(snap, batch, folder, device, newName)
|
||||
ldbRemoveFromGlobal(snap, batch, folder, device, newName, globalSize)
|
||||
} else {
|
||||
ldbUpdateGlobal(snap, batch, folder, device, fs[fsi])
|
||||
ldbUpdateGlobal(snap, batch, folder, device, fs[fsi], globalSize)
|
||||
}
|
||||
} else {
|
||||
l.Debugln("generic replace; equal - ignore")
|
||||
@ -289,19 +293,19 @@ func ldbGenericReplace(db *leveldb.DB, folder, device []byte, fs []protocol.File
|
||||
return maxLocalVer
|
||||
}
|
||||
|
||||
func ldbReplace(db *leveldb.DB, folder, device []byte, fs []protocol.FileInfo) int64 {
|
||||
func ldbReplace(db *leveldb.DB, folder, device []byte, fs []protocol.FileInfo, localSize, globalSize *sizeTracker) int64 {
|
||||
// TODO: Return the remaining maxLocalVer?
|
||||
return ldbGenericReplace(db, folder, device, fs, func(db dbReader, batch dbWriter, folder, device, name []byte, dbi iterator.Iterator) int64 {
|
||||
return ldbGenericReplace(db, folder, device, fs, localSize, globalSize, func(db dbReader, batch dbWriter, folder, device, name []byte, dbi iterator.Iterator) int64 {
|
||||
// Database has a file that we are missing. Remove it.
|
||||
l.Debugf("delete; folder=%q device=%v name=%q", folder, protocol.DeviceIDFromBytes(device), name)
|
||||
ldbRemoveFromGlobal(db, batch, folder, device, name)
|
||||
ldbRemoveFromGlobal(db, batch, folder, device, name, globalSize)
|
||||
l.Debugf("batch.Delete %p %x", batch, dbi.Key())
|
||||
batch.Delete(dbi.Key())
|
||||
return 0
|
||||
})
|
||||
}
|
||||
|
||||
func ldbUpdate(db *leveldb.DB, folder, device []byte, fs []protocol.FileInfo, size *sizeTracker) int64 {
|
||||
func ldbUpdate(db *leveldb.DB, folder, device []byte, fs []protocol.FileInfo, localSize, globalSize *sizeTracker) int64 {
|
||||
runtime.GC()
|
||||
|
||||
batch := new(leveldb.Batch)
|
||||
@ -318,24 +322,24 @@ func ldbUpdate(db *leveldb.DB, folder, device []byte, fs []protocol.FileInfo, si
|
||||
|
||||
var maxLocalVer int64
|
||||
var fk []byte
|
||||
localDev := isLocalDevice(device)
|
||||
isLocalDevice := bytes.Equal(device, protocol.LocalDeviceID[:])
|
||||
for _, f := range fs {
|
||||
name := []byte(f.Name)
|
||||
fk = deviceKeyInto(fk[:cap(fk)], folder, device, name)
|
||||
l.Debugf("snap.Get %p %x", snap, fk)
|
||||
bs, err := snap.Get(fk, nil)
|
||||
if err == leveldb.ErrNotFound {
|
||||
if localDev {
|
||||
size.addFile(f)
|
||||
if isLocalDevice {
|
||||
localSize.addFile(f)
|
||||
}
|
||||
|
||||
if lv := ldbInsert(batch, folder, device, f); lv > maxLocalVer {
|
||||
maxLocalVer = lv
|
||||
}
|
||||
if f.IsInvalid() {
|
||||
ldbRemoveFromGlobal(snap, batch, folder, device, name)
|
||||
ldbRemoveFromGlobal(snap, batch, folder, device, name, globalSize)
|
||||
} else {
|
||||
ldbUpdateGlobal(snap, batch, folder, device, f)
|
||||
ldbUpdateGlobal(snap, batch, folder, device, f, globalSize)
|
||||
}
|
||||
continue
|
||||
}
|
||||
@ -348,18 +352,18 @@ func ldbUpdate(db *leveldb.DB, folder, device []byte, fs []protocol.FileInfo, si
|
||||
// Flags might change without the version being bumped when we set the
|
||||
// invalid flag on an existing file.
|
||||
if !ef.Version.Equal(f.Version) || ef.Flags != f.Flags {
|
||||
if localDev {
|
||||
size.removeFile(ef)
|
||||
size.addFile(f)
|
||||
if isLocalDevice {
|
||||
localSize.removeFile(ef)
|
||||
localSize.addFile(f)
|
||||
}
|
||||
|
||||
if lv := ldbInsert(batch, folder, device, f); lv > maxLocalVer {
|
||||
maxLocalVer = lv
|
||||
}
|
||||
if f.IsInvalid() {
|
||||
ldbRemoveFromGlobal(snap, batch, folder, device, name)
|
||||
ldbRemoveFromGlobal(snap, batch, folder, device, name, globalSize)
|
||||
} else {
|
||||
ldbUpdateGlobal(snap, batch, folder, device, f)
|
||||
ldbUpdateGlobal(snap, batch, folder, device, f, globalSize)
|
||||
}
|
||||
}
|
||||
|
||||
@ -404,7 +408,7 @@ func ldbInsert(batch dbWriter, folder, device []byte, file protocol.FileInfo) in
|
||||
// ldbUpdateGlobal adds this device+version to the version list for the given
|
||||
// file. If the device is already present in the list, the version is updated.
|
||||
// If the file does not have an entry in the global list, it is created.
|
||||
func ldbUpdateGlobal(db dbReader, batch dbWriter, folder, device []byte, file protocol.FileInfo) bool {
|
||||
func ldbUpdateGlobal(db dbReader, batch dbWriter, folder, device []byte, file protocol.FileInfo, globalSize *sizeTracker) bool {
|
||||
l.Debugf("update global; folder=%q device=%v file=%q version=%d", folder, protocol.DeviceIDFromBytes(device), file.Name, file.Version)
|
||||
name := []byte(file.Name)
|
||||
gk := globalKey(folder, name)
|
||||
@ -414,7 +418,8 @@ func ldbUpdateGlobal(db dbReader, batch dbWriter, folder, device []byte, file pr
|
||||
}
|
||||
|
||||
var fl versionList
|
||||
|
||||
var oldFile protocol.FileInfo
|
||||
var hasOldFile bool
|
||||
// Remove the device from the current version list
|
||||
if svl != nil {
|
||||
err = fl.UnmarshalXDR(svl)
|
||||
@ -422,6 +427,12 @@ func ldbUpdateGlobal(db dbReader, batch dbWriter, folder, device []byte, file pr
|
||||
panic(err)
|
||||
}
|
||||
|
||||
if len(fl.versions) > 0 {
|
||||
// Keep the current neweset file around so we can subtract it from
|
||||
// the globalSize if we replace it.
|
||||
oldFile, hasOldFile = ldbGet(db, folder, fl.versions[0].device, name)
|
||||
}
|
||||
|
||||
for i := range fl.versions {
|
||||
if bytes.Compare(fl.versions[i].device, device) == 0 {
|
||||
if fl.versions[i].version.Equal(file.Version) {
|
||||
@ -439,6 +450,7 @@ func ldbUpdateGlobal(db dbReader, batch dbWriter, folder, device []byte, file pr
|
||||
version: file.Version,
|
||||
}
|
||||
|
||||
insertedAt := -1
|
||||
// Find a position in the list to insert this file. The file at the front
|
||||
// of the list is the newer, the "global".
|
||||
for i := range fl.versions {
|
||||
@ -447,6 +459,7 @@ func ldbUpdateGlobal(db dbReader, batch dbWriter, folder, device []byte, file pr
|
||||
// The version at this point in the list is equal to or lesser
|
||||
// ("older") than us. We insert ourselves in front of it.
|
||||
fl.versions = insertVersion(fl.versions, i, nv)
|
||||
insertedAt = i
|
||||
goto done
|
||||
|
||||
case protocol.ConcurrentLesser, protocol.ConcurrentGreater:
|
||||
@ -462,6 +475,7 @@ func ldbUpdateGlobal(db dbReader, batch dbWriter, folder, device []byte, file pr
|
||||
}
|
||||
if file.WinsConflict(of) {
|
||||
fl.versions = insertVersion(fl.versions, i, nv)
|
||||
insertedAt = i
|
||||
goto done
|
||||
}
|
||||
}
|
||||
@ -469,8 +483,20 @@ func ldbUpdateGlobal(db dbReader, batch dbWriter, folder, device []byte, file pr
|
||||
|
||||
// We didn't find a position for an insert above, so append to the end.
|
||||
fl.versions = append(fl.versions, nv)
|
||||
insertedAt = len(fl.versions) - 1
|
||||
|
||||
done:
|
||||
if insertedAt == 0 {
|
||||
// We just inserted a new newest version. Fixup the global size
|
||||
// calculation.
|
||||
if !file.Version.Equal(oldFile.Version) {
|
||||
globalSize.addFile(file)
|
||||
if hasOldFile {
|
||||
globalSize.removeFile(oldFile)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
l.Debugf("batch.Put %p %x", batch, gk)
|
||||
l.Debugf("new global after update: %v", fl)
|
||||
batch.Put(gk, fl.MustMarshalXDR())
|
||||
@ -488,7 +514,7 @@ func insertVersion(vl []fileVersion, i int, v fileVersion) []fileVersion {
|
||||
// ldbRemoveFromGlobal removes the device from the global version list for the
|
||||
// given file. If the version list is empty after this, the file entry is
|
||||
// removed entirely.
|
||||
func ldbRemoveFromGlobal(db dbReader, batch dbWriter, folder, device, file []byte) {
|
||||
func ldbRemoveFromGlobal(db dbReader, batch dbWriter, folder, device, file []byte, globalSize *sizeTracker) {
|
||||
l.Debugf("remove from global; folder=%q device=%v file=%q", folder, protocol.DeviceIDFromBytes(device), file)
|
||||
|
||||
gk := globalKey(folder, file)
|
||||
@ -505,8 +531,17 @@ func ldbRemoveFromGlobal(db dbReader, batch dbWriter, folder, device, file []byt
|
||||
panic(err)
|
||||
}
|
||||
|
||||
removed := false
|
||||
for i := range fl.versions {
|
||||
if bytes.Compare(fl.versions[i].device, device) == 0 {
|
||||
if i == 0 && globalSize != nil {
|
||||
f, ok := ldbGet(db, folder, device, file)
|
||||
if !ok {
|
||||
panic("removing nonexistent file")
|
||||
}
|
||||
globalSize.removeFile(f)
|
||||
removed = true
|
||||
}
|
||||
fl.versions = append(fl.versions[:i], fl.versions[i+1:]...)
|
||||
break
|
||||
}
|
||||
@ -519,6 +554,13 @@ func ldbRemoveFromGlobal(db dbReader, batch dbWriter, folder, device, file []byt
|
||||
l.Debugf("batch.Put %p %x", batch, gk)
|
||||
l.Debugf("new global after remove: %v", fl)
|
||||
batch.Put(gk, fl.MustMarshalXDR())
|
||||
if removed {
|
||||
f, ok := ldbGet(db, folder, fl.versions[0].device, file)
|
||||
if !ok {
|
||||
panic("new global is nonexistent file")
|
||||
}
|
||||
globalSize.addFile(f)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -579,7 +621,7 @@ func ldbWithAllFolderTruncated(db *leveldb.DB, folder []byte, fn func(device []b
|
||||
case "", ".", "..", "/": // A few obviously invalid filenames
|
||||
l.Infof("Dropping invalid filename %q from database", f.Name)
|
||||
batch := new(leveldb.Batch)
|
||||
ldbRemoveFromGlobal(db, batch, folder, device, nil)
|
||||
ldbRemoveFromGlobal(db, batch, folder, device, nil, nil)
|
||||
batch.Delete(dbi.Key())
|
||||
db.Write(batch, nil)
|
||||
continue
|
||||
@ -914,7 +956,7 @@ func unmarshalTrunc(bs []byte, truncate bool) (FileIntf, error) {
|
||||
return tf, err
|
||||
}
|
||||
|
||||
func ldbCheckGlobals(db *leveldb.DB, folder []byte) {
|
||||
func ldbCheckGlobals(db *leveldb.DB, folder []byte, globalSize *sizeTracker) {
|
||||
defer runtime.GC()
|
||||
|
||||
snap, err := db.GetSnapshot()
|
||||
@ -951,8 +993,9 @@ func ldbCheckGlobals(db *leveldb.DB, folder []byte) {
|
||||
|
||||
name := globalKeyName(gk)
|
||||
var newVL versionList
|
||||
for _, version := range vl.versions {
|
||||
for i, version := range vl.versions {
|
||||
fk = deviceKeyInto(fk[:cap(fk)], folder, version.device, name)
|
||||
|
||||
l.Debugf("snap.Get %p %x", snap, fk)
|
||||
_, err := snap.Get(fk, nil)
|
||||
if err == leveldb.ErrNotFound {
|
||||
@ -962,6 +1005,14 @@ func ldbCheckGlobals(db *leveldb.DB, folder []byte) {
|
||||
panic(err)
|
||||
}
|
||||
newVL.versions = append(newVL.versions, version)
|
||||
|
||||
if i == 0 {
|
||||
fi, ok := ldbGet(snap, folder, version.device, name)
|
||||
if !ok {
|
||||
panic("nonexistent global master file")
|
||||
}
|
||||
globalSize.addFile(fi)
|
||||
}
|
||||
}
|
||||
|
||||
if len(newVL.versions) != len(vl.versions) {
|
||||
|
||||
@ -27,7 +27,8 @@ type FileSet struct {
|
||||
folder string
|
||||
db *leveldb.DB
|
||||
blockmap *BlockMap
|
||||
sizeTracker
|
||||
localSize sizeTracker
|
||||
globalSize sizeTracker
|
||||
}
|
||||
|
||||
// FileIntf is the set of methods implemented by both protocol.FileInfo and
|
||||
@ -57,6 +58,7 @@ func (s *sizeTracker) addFile(f FileIntf) {
|
||||
if f.IsInvalid() {
|
||||
return
|
||||
}
|
||||
|
||||
s.mut.Lock()
|
||||
if f.IsDeleted() {
|
||||
s.deleted++
|
||||
@ -71,6 +73,7 @@ func (s *sizeTracker) removeFile(f FileIntf) {
|
||||
if f.IsInvalid() {
|
||||
return
|
||||
}
|
||||
|
||||
s.mut.Lock()
|
||||
if f.IsDeleted() {
|
||||
s.deleted--
|
||||
@ -78,6 +81,9 @@ func (s *sizeTracker) removeFile(f FileIntf) {
|
||||
s.files--
|
||||
}
|
||||
s.bytes -= f.Size()
|
||||
if s.deleted < 0 || s.files < 0 {
|
||||
panic("bug: removed more than added")
|
||||
}
|
||||
s.mut.Unlock()
|
||||
}
|
||||
|
||||
@ -96,7 +102,7 @@ func NewFileSet(folder string, db *leveldb.DB) *FileSet {
|
||||
mutex: sync.NewMutex(),
|
||||
}
|
||||
|
||||
ldbCheckGlobals(db, []byte(folder))
|
||||
ldbCheckGlobals(db, []byte(folder), &s.globalSize)
|
||||
|
||||
var deviceID protocol.DeviceID
|
||||
ldbWithAllFolderTruncated(db, []byte(folder), func(device []byte, f FileInfoTruncated) bool {
|
||||
@ -105,7 +111,7 @@ func NewFileSet(folder string, db *leveldb.DB) *FileSet {
|
||||
s.localVersion[deviceID] = f.LocalVersion
|
||||
}
|
||||
if deviceID == protocol.LocalDeviceID {
|
||||
s.addFile(f)
|
||||
s.localSize.addFile(f)
|
||||
}
|
||||
return true
|
||||
})
|
||||
@ -120,7 +126,7 @@ func (s *FileSet) Replace(device protocol.DeviceID, fs []protocol.FileInfo) {
|
||||
normalizeFilenames(fs)
|
||||
s.mutex.Lock()
|
||||
defer s.mutex.Unlock()
|
||||
s.localVersion[device] = ldbReplace(s.db, []byte(s.folder), device[:], fs)
|
||||
s.localVersion[device] = ldbReplace(s.db, []byte(s.folder), device[:], fs, &s.localSize, &s.globalSize)
|
||||
if len(fs) == 0 {
|
||||
// Reset the local version if all files were removed.
|
||||
s.localVersion[device] = 0
|
||||
@ -149,7 +155,7 @@ func (s *FileSet) Update(device protocol.DeviceID, fs []protocol.FileInfo) {
|
||||
s.blockmap.Discard(discards)
|
||||
s.blockmap.Update(updates)
|
||||
}
|
||||
if lv := ldbUpdate(s.db, []byte(s.folder), device[:], fs, &s.sizeTracker); lv > s.localVersion[device] {
|
||||
if lv := ldbUpdate(s.db, []byte(s.folder), device[:], fs, &s.localSize, &s.globalSize); lv > s.localVersion[device] {
|
||||
s.localVersion[device] = lv
|
||||
}
|
||||
}
|
||||
@ -225,6 +231,14 @@ func (s *FileSet) LocalVersion(device protocol.DeviceID) int64 {
|
||||
return s.localVersion[device]
|
||||
}
|
||||
|
||||
func (s *FileSet) LocalSize() (files, deleted int, bytes int64) {
|
||||
return s.localSize.Size()
|
||||
}
|
||||
|
||||
func (s *FileSet) GlobalSize() (files, deleted int, bytes int64) {
|
||||
return s.globalSize.Size()
|
||||
}
|
||||
|
||||
// ListFolders returns the folder IDs seen in the database.
|
||||
func ListFolders(db *leveldb.DB) []string {
|
||||
return ldbListFolders(db)
|
||||
|
||||
@ -329,8 +329,6 @@ func (m *Model) FolderStatistics() map[string]stats.FolderStatistics {
|
||||
// Completion returns the completion status, in percent, for the given device
|
||||
// and folder.
|
||||
func (m *Model) Completion(device protocol.DeviceID, folder string) float64 {
|
||||
var tot int64
|
||||
|
||||
m.fmut.RLock()
|
||||
rf, ok := m.folderFiles[folder]
|
||||
m.fmut.RUnlock()
|
||||
@ -338,29 +336,22 @@ func (m *Model) Completion(device protocol.DeviceID, folder string) float64 {
|
||||
return 0 // Folder doesn't exist, so we hardly have any of it
|
||||
}
|
||||
|
||||
rf.WithGlobalTruncated(func(f db.FileIntf) bool {
|
||||
if !f.IsDeleted() {
|
||||
tot += f.Size()
|
||||
}
|
||||
return true
|
||||
})
|
||||
|
||||
_, _, tot := rf.GlobalSize()
|
||||
if tot == 0 {
|
||||
return 100 // Folder is empty, so we have all of it
|
||||
}
|
||||
|
||||
var need int64
|
||||
rf.WithNeedTruncated(device, func(f db.FileIntf) bool {
|
||||
if !f.IsDeleted() {
|
||||
need += f.Size()
|
||||
}
|
||||
need += f.Size()
|
||||
return true
|
||||
})
|
||||
|
||||
res := 100 * (1 - float64(need)/float64(tot))
|
||||
l.Debugf("%v Completion(%s, %q): %f (%d / %d)", m, device, folder, res, need, tot)
|
||||
needRatio := float64(need) / float64(tot)
|
||||
completionPct := 100 * (1 - needRatio)
|
||||
l.Debugf("%v Completion(%s, %q): %f (%d / %d = %f)", m, device, folder, completionPct, need, tot, needRatio)
|
||||
|
||||
return res
|
||||
return completionPct
|
||||
}
|
||||
|
||||
func sizeOf(fs []protocol.FileInfo) (files, deleted int, bytes int64) {
|
||||
@ -389,13 +380,7 @@ func (m *Model) GlobalSize(folder string) (nfiles, deleted int, bytes int64) {
|
||||
m.fmut.RLock()
|
||||
defer m.fmut.RUnlock()
|
||||
if rf, ok := m.folderFiles[folder]; ok {
|
||||
rf.WithGlobalTruncated(func(f db.FileIntf) bool {
|
||||
fs, de, by := sizeOfFile(f)
|
||||
nfiles += fs
|
||||
deleted += de
|
||||
bytes += by
|
||||
return true
|
||||
})
|
||||
nfiles, deleted, bytes = rf.GlobalSize()
|
||||
}
|
||||
return
|
||||
}
|
||||
@ -406,7 +391,7 @@ func (m *Model) LocalSize(folder string) (nfiles, deleted int, bytes int64) {
|
||||
m.fmut.RLock()
|
||||
defer m.fmut.RUnlock()
|
||||
if rf, ok := m.folderFiles[folder]; ok {
|
||||
nfiles, deleted, bytes = rf.Size()
|
||||
nfiles, deleted, bytes = rf.LocalSize()
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user