From 5baa432906f5eaf6784c3207f5bb775e22c8497c Mon Sep 17 00:00:00 2001 From: Simon Frei Date: Sat, 2 Jun 2018 15:08:32 +0200 Subject: [PATCH] lib/db: Add index to track locally needed files (#4958) To optimize WithNeed, which is called for the local device whenever an index update is received. No tracking for remote devices to conserve db space, as WithNeed is only queried for completion. --- lib/db/leveldb.go | 91 +++++++++- lib/db/leveldb_dbinstance.go | 160 ++++++++++------ lib/db/leveldb_test.go | 96 ++++++++++ lib/db/leveldb_transactions.go | 181 ++++++++----------- lib/db/set.go | 1 + lib/db/structs.go | 4 + lib/db/testdata/v0.14.45-update0to3.db.jsons | 22 +++ lib/db/util_test.go | 82 +++++---- lib/model/requests_test.go | 2 +- lib/protocol/bep_extensions.go | 4 + 10 files changed, 440 insertions(+), 203 deletions(-) create mode 100644 lib/db/testdata/v0.14.45-update0to3.db.jsons diff --git a/lib/db/leveldb.go b/lib/db/leveldb.go index 732d6146a..2b82b23d4 100644 --- a/lib/db/leveldb.go +++ b/lib/db/leveldb.go @@ -13,7 +13,7 @@ import ( "github.com/syncthing/syncthing/lib/protocol" ) -const dbVersion = 2 +const dbVersion = 3 const ( KeyTypeDevice = iota @@ -28,13 +28,14 @@ const ( KeyTypeFolderMeta KeyTypeMiscData KeyTypeSequence + KeyTypeNeed ) -func (l VersionList) String() string { +func (vl VersionList) String() string { var b bytes.Buffer var id protocol.DeviceID b.WriteString("{") - for i, v := range l.Versions { + for i, v := range vl.Versions { if i > 0 { b.WriteString(", ") } @@ -45,18 +46,90 @@ func (l VersionList) String() string { return b.String() } +// update brings the VersionList up to date with file. It returns the updated +// VersionList, a potentially removed old FileVersion and its index, as well as +// the index where the new FileVersion was inserted. +func (vl VersionList) update(folder, device []byte, file protocol.FileInfo, db *Instance) (_ VersionList, removedFV FileVersion, removedAt int, insertedAt int) { + removedAt, insertedAt = -1, -1 + for i, v := range vl.Versions { + if bytes.Equal(v.Device, device) { + removedAt = i + removedFV = v + vl.Versions = append(vl.Versions[:i], vl.Versions[i+1:]...) + break + } + } + + nv := FileVersion{ + Device: device, + Version: file.Version, + Invalid: file.Invalid, + } + for i, v := range vl.Versions { + switch v.Version.Compare(file.Version) { + case protocol.Equal: + if nv.Invalid { + continue + } + fallthrough + + case protocol.Lesser: + // The version at this point in the list is equal to or lesser + // ("older") than us. We insert ourselves in front of it. + vl = vl.insertAt(i, nv) + return vl, removedFV, removedAt, i + + case protocol.ConcurrentLesser, protocol.ConcurrentGreater: + // The version at this point is in conflict with us. We must pull + // the actual file metadata to determine who wins. If we win, we + // insert ourselves in front of the loser here. (The "Lesser" and + // "Greater" in the condition above is just based on the device + // IDs in the version vector, which is not the only thing we use + // to determine the winner.) + // + // A surprise missing file entry here is counted as a win for us. + if of, ok := db.getFile(db.deviceKey(folder, v.Device, []byte(file.Name))); !ok || file.WinsConflict(of) { + vl = vl.insertAt(i, nv) + return vl, removedFV, removedAt, i + } + } + } + + // We didn't find a position for an insert above, so append to the end. + vl.Versions = append(vl.Versions, nv) + + return vl, removedFV, removedAt, len(vl.Versions) - 1 +} + +func (vl VersionList) insertAt(i int, v FileVersion) VersionList { + vl.Versions = append(vl.Versions, FileVersion{}) + copy(vl.Versions[i+1:], vl.Versions[i:]) + vl.Versions[i] = v + return vl +} + +func (vl VersionList) Get(device []byte) (FileVersion, bool) { + for _, v := range vl.Versions { + if bytes.Equal(v.Device, device) { + return v, true + } + } + + return FileVersion{}, false +} + type fileList []protocol.FileInfo -func (l fileList) Len() int { - return len(l) +func (fl fileList) Len() int { + return len(fl) } -func (l fileList) Swap(a, b int) { - l[a], l[b] = l[b], l[a] +func (fl fileList) Swap(a, b int) { + fl[a], fl[b] = fl[b], fl[a] } -func (l fileList) Less(a, b int) bool { - return l[a].Name < l[b].Name +func (fl fileList) Less(a, b int) bool { + return fl[a].Name < fl[b].Name } // Flush batches to disk when they contain this many records. diff --git a/lib/db/leveldb_dbinstance.go b/lib/db/leveldb_dbinstance.go index cd8f02cf8..19dadbcf5 100644 --- a/lib/db/leveldb_dbinstance.go +++ b/lib/db/leveldb_dbinstance.go @@ -103,6 +103,9 @@ func (db *Instance) UpdateSchema() { if prevVersion <= 1 { db.updateSchema1to2() } + if prevVersion <= 2 { + db.updateSchema2to3() + } l.Infof("Finished updating database schema version from %v to %v", prevVersion, dbVersion) miscDB.PutInt64("dbVersion", dbVersion) @@ -310,26 +313,32 @@ func (db *Instance) getFileTrunc(key []byte, trunc bool) (FileIntf, bool) { } func (db *Instance) getGlobal(folder, file []byte, truncate bool) (FileIntf, bool) { - k := db.globalKey(folder, file) - t := db.newReadOnlyTransaction() defer t.close() - bs, err := t.Get(k, nil) + _, _, f, ok := db.getGlobalInto(t, nil, nil, folder, file, truncate) + return f, ok +} + +func (db *Instance) getGlobalInto(t readOnlyTransaction, gk, dk, folder, file []byte, truncate bool) ([]byte, []byte, FileIntf, bool) { + gk = db.globalKeyInto(gk, folder, file) + + bs, err := t.Get(gk, nil) if err != nil { - return nil, false + return gk, dk, nil, false } vl, ok := unmarshalVersionList(bs) if !ok { - return nil, false + return gk, dk, nil, false } - if fi, ok := db.getFileTrunc(db.deviceKey(folder, vl.Versions[0].Device, file), truncate); ok { - return fi, true + dk = db.deviceKeyInto(dk, folder, vl.Versions[0].Device, file) + if fi, ok := db.getFileTrunc(dk, truncate); ok { + return gk, dk, fi, true } - return nil, false + return gk, dk, nil, false } func (db *Instance) withGlobal(folder, prefix []byte, truncate bool, fn Iterator) { @@ -409,6 +418,11 @@ func (db *Instance) availability(folder, file []byte) []protocol.DeviceID { } func (db *Instance) withNeed(folder, device []byte, truncate bool, fn Iterator) { + if bytes.Equal(device, protocol.LocalDeviceID[:]) { + db.withNeedLocal(folder, truncate, fn) + return + } + t := db.newReadOnlyTransaction() defer t.close() @@ -422,22 +436,11 @@ func (db *Instance) withNeed(folder, device []byte, truncate bool, fn Iterator) continue } - have := false // If we have the file, any version - need := false // If we have a lower version of the file - var haveFileVersion FileVersion - for _, v := range vl.Versions { - if bytes.Equal(v.Device, device) { - have = true - haveFileVersion = v - // XXX: This marks Concurrent (i.e. conflicting) changes as - // needs. Maybe we should do that, but it needs special - // handling in the puller. - need = !v.Version.GreaterEqual(vl.Versions[0].Version) - break - } - } - - if have && !need { + haveFV, have := vl.Get(device) + // XXX: This marks Concurrent (i.e. conflicting) changes as + // needs. Maybe we should do that, but it needs special + // handling in the puller. + if have && haveFV.Version.GreaterEqual(vl.Versions[0].Version) { continue } @@ -474,7 +477,7 @@ func (db *Instance) withNeed(folder, device []byte, truncate bool, fn Iterator) break } - l.Debugf("need folder=%q device=%v name=%q need=%v have=%v invalid=%v haveV=%v globalV=%v globalDev=%v", folder, protocol.DeviceIDFromBytes(device), name, need, have, haveFileVersion.Invalid, haveFileVersion.Version, needVersion, needDevice) + l.Debugf("need folder=%q device=%v name=%q have=%v invalid=%v haveV=%v globalV=%v globalDev=%v", folder, protocol.DeviceIDFromBytes(device), name, have, haveFV.Invalid, haveFV.Version, needVersion, needDevice) if !fn(gf) { return @@ -486,6 +489,28 @@ func (db *Instance) withNeed(folder, device []byte, truncate bool, fn Iterator) } } +func (db *Instance) withNeedLocal(folder []byte, truncate bool, fn Iterator) { + t := db.newReadOnlyTransaction() + defer t.close() + + dbi := t.NewIterator(util.BytesPrefix(db.needKey(folder, nil)[:keyPrefixLen+keyFolderLen]), nil) + defer dbi.Release() + + var dk []byte + var gk []byte + var f FileIntf + var ok bool + for dbi.Next() { + gk, dk, f, ok = db.getGlobalInto(t, gk, dk, folder, db.globalKeyName(dbi.Key()), truncate) + if !ok { + continue + } + if !fn(f) { + return + } + } +} + func (db *Instance) ListFolders() []string { t := db.newReadOnlyTransaction() defer t.close() @@ -511,36 +536,21 @@ func (db *Instance) ListFolders() []string { } func (db *Instance) dropFolder(folder []byte) { - t := db.newReadOnlyTransaction() + t := db.newReadWriteTransaction() defer t.close() - // Remove all items related to the given folder from the device->file bucket - dbi := t.NewIterator(util.BytesPrefix([]byte{KeyTypeDevice}), nil) - for dbi.Next() { - itemFolder := db.deviceKeyFolder(dbi.Key()) - if bytes.Equal(folder, itemFolder) { - db.Delete(dbi.Key(), nil) - } + for _, key := range [][]byte{ + // Remove all items related to the given folder from the device->file bucket + db.deviceKey(folder, nil, nil)[:keyPrefixLen+keyFolderLen], + // Remove all sequences related to the folder + db.sequenceKey([]byte(folder), 0)[:keyPrefixLen+keyFolderLen], + // Remove all items related to the given folder from the global bucket + db.globalKey(folder, nil)[:keyPrefixLen+keyFolderLen], + // Remove all needs related to the folder + db.needKey(folder, nil)[:keyPrefixLen+keyFolderLen], + } { + t.deleteKeyPrefix(key) } - dbi.Release() - - // Remove all sequences related to the folder - sequenceKey := db.sequenceKey([]byte(folder), 0) - dbi = t.NewIterator(util.BytesPrefix(sequenceKey[:keyPrefixLen+keyFolderLen]), nil) - for dbi.Next() { - db.Delete(dbi.Key(), nil) - } - dbi.Release() - - // Remove all items related to the given folder from the global bucket - dbi = t.NewIterator(util.BytesPrefix([]byte{KeyTypeGlobal}), nil) - for dbi.Next() { - itemFolder, ok := db.globalKeyFolder(dbi.Key()) - if ok && bytes.Equal(folder, itemFolder) { - db.Delete(dbi.Key(), nil) - } - } - dbi.Release() } func (db *Instance) dropDeviceFolder(device, folder []byte, meta *metadataTracker) { @@ -680,13 +690,14 @@ func (db *Instance) updateSchema0to1() { l.Infof("Updated symlink type for %d index entries and added %d invalid files to global list", symlinkConv, ignAdded) } +// updateSchema1to2 introduces a sequenceKey->deviceKey bucket for local items +// to allow iteration in sequence order (simplifies sending indexes). func (db *Instance) updateSchema1to2() { t := db.newReadWriteTransaction() defer t.close() var sk []byte var dk []byte - for _, folderStr := range db.ListFolders() { folder := []byte(folderStr) db.withHave(folder, protocol.LocalDeviceID[:], nil, true, func(f FileIntf) bool { @@ -699,6 +710,34 @@ func (db *Instance) updateSchema1to2() { } } +// updateSchema2to3 introduces a needKey->nil bucket for locally needed files. +func (db *Instance) updateSchema2to3() { + t := db.newReadWriteTransaction() + defer t.close() + + var nk []byte + var dk []byte + for _, folderStr := range db.ListFolders() { + folder := []byte(folderStr) + db.withGlobal(folder, nil, true, func(f FileIntf) bool { + name := []byte(f.FileName()) + dk = db.deviceKeyInto(dk, folder, protocol.LocalDeviceID[:], name) + var v protocol.Vector + haveFile, ok := db.getFileTrunc(dk, true) + if ok { + v = haveFile.FileVersion() + } + if !need(f, ok, v) { + return true + } + nk = t.db.needKeyInto(nk, folder, []byte(f.FileName())) + t.Put(nk, nil) + t.checkFlush() + return true + }) + } +} + // deviceKey returns a byte slice encoding the following information: // keyTypeDevice (1 byte) // folder (4 bytes) @@ -755,7 +794,7 @@ func (db *Instance) globalKeyInto(gk, folder, file []byte) []byte { gk[0] = KeyTypeGlobal binary.BigEndian.PutUint32(gk[keyPrefixLen:], db.folderIdx.ID(folder)) copy(gk[keyPrefixLen+keyFolderLen:], file) - return gk[:reqLen] + return gk } // globalKeyName returns the filename from the key @@ -768,6 +807,17 @@ func (db *Instance) globalKeyFolder(key []byte) ([]byte, bool) { return db.folderIdx.Val(binary.BigEndian.Uint32(key[keyPrefixLen:])) } +// needKey is a globalKey with a different prefix +func (db *Instance) needKey(folder, file []byte) []byte { + return db.needKeyInto(nil, folder, file) +} + +func (db *Instance) needKeyInto(k, folder, file []byte) []byte { + k = db.globalKeyInto(k, folder, file) + k[0] = KeyTypeNeed + return k +} + // sequenceKey returns a byte slice encoding the following information: // KeyTypeSequence (1 byte) // folder (4 bytes) @@ -782,7 +832,7 @@ func (db *Instance) sequenceKeyInto(k []byte, folder []byte, seq int64) []byte { k[0] = KeyTypeSequence binary.BigEndian.PutUint32(k[keyPrefixLen:], db.folderIdx.ID(folder)) binary.BigEndian.PutUint64(k[keyPrefixLen+keyFolderLen:], uint64(seq)) - return k[:reqLen] + return k } // sequenceKeySequence returns the sequence number from the key diff --git a/lib/db/leveldb_test.go b/lib/db/leveldb_test.go index 3b953b03e..e724271ca 100644 --- a/lib/db/leveldb_test.go +++ b/lib/db/leveldb_test.go @@ -224,3 +224,99 @@ func TestInvalidFiles(t *testing.T) { t.Error("quux should not be invalid") } } + +const myID = 1 + +var ( + remoteDevice0, remoteDevice1 protocol.DeviceID + update0to3Folder = "UpdateSchema0to3" + invalid = "invalid" + slashPrefixed = "/notgood" + haveUpdate0to3 map[protocol.DeviceID]fileList +) + +func init() { + remoteDevice0, _ = protocol.DeviceIDFromString("AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR") + remoteDevice1, _ = protocol.DeviceIDFromString("I6KAH76-66SLLLB-5PFXSOA-UFJCDZC-YAOMLEK-CP2GB32-BV5RQST-3PSROAU") + haveUpdate0to3 = map[protocol.DeviceID]fileList{ + protocol.LocalDeviceID: { + protocol.FileInfo{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(1)}, + protocol.FileInfo{Name: slashPrefixed, Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(1)}, + }, + remoteDevice0: { + protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}, Blocks: genBlocks(2)}, + protocol.FileInfo{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(5), Invalid: true}, + protocol.FileInfo{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1003}}}, Blocks: genBlocks(7)}, + }, + remoteDevice1: { + protocol.FileInfo{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(7)}, + protocol.FileInfo{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1003}}}, Blocks: genBlocks(5), Invalid: true}, + protocol.FileInfo{Name: invalid, Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1004}}}, Blocks: genBlocks(5), Invalid: true}, + }, + } +} + +func TestUpdate0to3(t *testing.T) { + ldb, err := openJSONS("testdata/v0.14.45-update0to3.db.jsons") + + if err != nil { + t.Fatal(err) + } + db := newDBInstance(ldb, "") + + folder := []byte(update0to3Folder) + + db.updateSchema0to1() + + if _, ok := db.getFile(db.deviceKey(folder, protocol.LocalDeviceID[:], []byte(slashPrefixed))); ok { + t.Error("File prefixed by '/' was not removed during transition to schema 1") + } + + if _, err := db.Get(db.globalKey(folder, []byte(invalid)), nil); err != nil { + t.Error("Invalid file wasn't added to global list") + } + + db.updateSchema1to2() + + found := false + db.withHaveSequence(folder, 0, func(fi FileIntf) bool { + f := fi.(protocol.FileInfo) + l.Infoln(f) + if found { + t.Error("Unexpected additional file via sequence", f.FileName()) + return true + } + if e := haveUpdate0to3[protocol.LocalDeviceID][0]; f.IsEquivalent(e, true, true) { + found = true + } else { + t.Errorf("Wrong file via sequence, got %v, expected %v", f, e) + } + return true + }) + if !found { + t.Error("Local file wasn't added to sequence bucket", err) + } + + db.updateSchema2to3() + + need := map[string]protocol.FileInfo{ + haveUpdate0to3[remoteDevice0][0].Name: haveUpdate0to3[remoteDevice0][0], + haveUpdate0to3[remoteDevice1][0].Name: haveUpdate0to3[remoteDevice1][0], + haveUpdate0to3[remoteDevice0][2].Name: haveUpdate0to3[remoteDevice0][2], + } + db.withNeed(folder, protocol.LocalDeviceID[:], false, func(fi FileIntf) bool { + e, ok := need[fi.FileName()] + if !ok { + t.Error("Got unexpected needed file:", fi.FileName()) + } + f := fi.(protocol.FileInfo) + delete(need, f.Name) + if !f.IsEquivalent(e, true, true) { + t.Errorf("Wrong needed file, got %v, expected %v", f, e) + } + return true + }) + for n := range need { + t.Errorf(`Missing needed file "%v"`, n) + } +} diff --git a/lib/db/leveldb_transactions.go b/lib/db/leveldb_transactions.go index 48f076383..f9c949175 100644 --- a/lib/db/leveldb_transactions.go +++ b/lib/db/leveldb_transactions.go @@ -12,6 +12,7 @@ import ( "github.com/syncthing/syncthing/lib/protocol" "github.com/syndtr/goleveldb/leveldb" + "github.com/syndtr/goleveldb/leveldb/util" ) // A readOnlyTransaction represents a database snapshot. @@ -85,115 +86,87 @@ func (t readWriteTransaction) insertFile(fk, folder, device []byte, file protoco // If the file does not have an entry in the global list, it is created. func (t readWriteTransaction) updateGlobal(gk, folder, device []byte, file protocol.FileInfo, meta *metadataTracker) bool { l.Debugf("update global; folder=%q device=%v file=%q version=%v invalid=%v", folder, protocol.DeviceIDFromBytes(device), file.Name, file.Version, file.Invalid) - name := []byte(file.Name) - svl, _ := t.Get(gk, nil) // skip error, we check len(svl) != 0 later var fl VersionList - var oldFile protocol.FileInfo - var hasOldFile bool - // Remove the device from the current version list - if len(svl) != 0 { - fl.Unmarshal(svl) // skip error, range handles success case - for i := range fl.Versions { - if bytes.Equal(fl.Versions[i].Device, device) { - if fl.Versions[i].Version.Equal(file.Version) && fl.Versions[i].Invalid == file.Invalid { - // No need to do anything - return false - } - - if i == 0 { - // Keep the current newest file around so we can subtract it from - // the metadata if we replace it. - oldFile, hasOldFile = t.getFile(folder, fl.Versions[0].Device, name) - } - - fl.Versions = append(fl.Versions[:i], fl.Versions[i+1:]...) - break - } - } + if svl, err := t.Get(gk, nil); err == nil { + fl.Unmarshal(svl) // Ignore error, continue with empty fl } - - nv := FileVersion{ - Device: device, - Version: file.Version, - Invalid: file.Invalid, - } - - insertedAt := -1 - // Find a position in the list to insert this file. The file at the front - // of the list is the newer, the "global". -insert: - for i := range fl.Versions { - switch fl.Versions[i].Version.Compare(file.Version) { - case protocol.Equal: - if nv.Invalid { - continue insert - } - fallthrough - - case protocol.Lesser: - // The version at this point in the list is equal to or lesser - // ("older") than us. We insert ourselves in front of it. - fl.Versions = insertVersion(fl.Versions, i, nv) - insertedAt = i - break insert - - case protocol.ConcurrentLesser, protocol.ConcurrentGreater: - // The version at this point is in conflict with us. We must pull - // the actual file metadata to determine who wins. If we win, we - // insert ourselves in front of the loser here. (The "Lesser" and - // "Greater" in the condition above is just based on the device - // IDs in the version vector, which is not the only thing we use - // to determine the winner.) - // - // A surprise missing file entry here is counted as a win for us. - if of, ok := t.getFile(folder, fl.Versions[i].Device, name); !ok || file.WinsConflict(of) { - fl.Versions = insertVersion(fl.Versions, i, nv) - insertedAt = i - break insert - } - } - } - + fl, removedFV, removedAt, insertedAt := fl.update(folder, device, file, t.db) if insertedAt == -1 { - // We didn't find a position for an insert above, so append to the end. - fl.Versions = append(fl.Versions, nv) - insertedAt = len(fl.Versions) - 1 - } - // Fixup the global size calculation. - if hasOldFile { - // We removed the previous newest version - meta.removeFile(globalDeviceID, oldFile) - if insertedAt == 0 { - // inserted a new newest version - meta.addFile(globalDeviceID, file) - } else { - // The previous second version is now the first - if newGlobal, ok := t.getFile(folder, fl.Versions[0].Device, name); ok { - // A failure to get the file here is surprising and our - // global size data will be incorrect until a restart... - meta.addFile(globalDeviceID, newGlobal) - } - } - } else if insertedAt == 0 { - // We just inserted a new newest version. - meta.addFile(globalDeviceID, file) - if len(fl.Versions) > 1 { - // The previous newest version is now at index 1, grab it from there. - if oldFile, ok := t.getFile(folder, fl.Versions[1].Device, name); ok { - // A failure to get the file here is surprising and our - // global size data will be incorrect until a restart... - meta.removeFile(globalDeviceID, oldFile) - } - } + l.Debugln("update global; same version, global unchanged") + return false } - l.Debugf("new global after update: %v", fl) + if removedAt != 0 && insertedAt != 0 { + l.Debugf(`new global for "%v" after update: %v`, file.Name, fl) + t.Put(gk, mustMarshal(&fl)) + return true + } + + name := []byte(file.Name) + + // Remove the old global from the global size counter + var oldGlobalFV FileVersion + if removedAt == 0 { + oldGlobalFV = removedFV + } else if len(fl.Versions) > 1 { + // The previous newest version is now at index 1 + oldGlobalFV = fl.Versions[1] + } + if oldFile, ok := t.getFile(folder, oldGlobalFV.Device, name); ok { + // A failure to get the file here is surprising and our + // global size data will be incorrect until a restart... + meta.removeFile(globalDeviceID, oldFile) + } + + // Add the new global to the global size counter + var newGlobal protocol.FileInfo + if insertedAt == 0 { + // Inserted a new newest version + newGlobal = file + } else if new, ok := t.getFile(folder, fl.Versions[0].Device, name); ok { + // The previous second version is now the first + newGlobal = new + } else { + panic("This file must exist in the db") + } + meta.addFile(globalDeviceID, newGlobal) + + // Fixup the list of files we need. + nk := t.db.needKey(folder, name) + hasNeeded, _ := t.db.Has(nk, nil) + if localFV, haveLocalFV := fl.Get(protocol.LocalDeviceID[:]); need(newGlobal, haveLocalFV, localFV.Version) { + if !hasNeeded { + l.Debugf("local need insert; folder=%q, name=%q", folder, name) + t.Put(nk, nil) + } + } else if hasNeeded { + l.Debugf("local need delete; folder=%q, name=%q", folder, name) + t.Delete(nk) + } + + l.Debugf(`new global for "%v" after update: %v`, file.Name, fl) t.Put(gk, mustMarshal(&fl)) return true } +func need(global FileIntf, haveLocal bool, localVersion protocol.Vector) bool { + // We never need an invalid file. + if global.IsInvalid() { + return false + } + // We don't need a deleted file if we don't have it. + if global.IsDeleted() && !haveLocal { + return false + } + // We don't need the global file if we already have the same version. + if haveLocal && localVersion.Equal(global.FileVersion()) { + return false + } + return true +} + // removeFromGlobal removes the device from the global version list for the // given file. If the version list is empty after this, the file entry is // removed entirely. @@ -246,11 +219,13 @@ func (t readWriteTransaction) removeFromGlobal(gk, folder, device, file []byte, } } -func insertVersion(vl []FileVersion, i int, v FileVersion) []FileVersion { - t := append(vl, FileVersion{}) - copy(t[i+1:], t[i:]) - t[i] = v - return t +func (t readWriteTransaction) deleteKeyPrefix(prefix []byte) { + dbi := t.NewIterator(util.BytesPrefix(prefix), nil) + for dbi.Next() { + t.Delete(dbi.Key()) + t.checkFlush() + } + dbi.Release() } type marshaller interface { diff --git a/lib/db/set.go b/lib/db/set.go index b6e55707f..cdf155ed5 100644 --- a/lib/db/set.go +++ b/lib/db/set.go @@ -44,6 +44,7 @@ type FileIntf interface { HasPermissionBits() bool SequenceNo() int64 BlockSize() int + FileVersion() protocol.Vector } // The Iterator is called with either a protocol.FileInfo or a diff --git a/lib/db/structs.go b/lib/db/structs.go index 60ff35823..d0c4061d7 100644 --- a/lib/db/structs.go +++ b/lib/db/structs.go @@ -75,6 +75,10 @@ func (f FileInfoTruncated) SequenceNo() int64 { return f.Sequence } +func (f FileInfoTruncated) FileVersion() protocol.Vector { + return f.Version +} + func (f FileInfoTruncated) ConvertToInvalidFileInfo(invalidatedBy protocol.ShortID) protocol.FileInfo { return protocol.FileInfo{ Name: f.Name, diff --git a/lib/db/testdata/v0.14.45-update0to3.db.jsons b/lib/db/testdata/v0.14.45-update0to3.db.jsons new file mode 100644 index 000000000..87ef52417 --- /dev/null +++ b/lib/db/testdata/v0.14.45-update0to3.db.jsons @@ -0,0 +1,22 @@ +{"k":"AAAAAAAAAAABL25vdGdvb2Q=","v":"Cggvbm90Z29vZEoHCgUIARDoB1ACggEiGiAAAQIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHw=="} +{"k":"AAAAAAAAAAABYQ==","v":"CgFhSgcKBQgBEOgHUAGCASIaIAABAgMEBQYHCAkKCwwNDg8QERITFBUWFxgZGhscHR4f"} +{"k":"AAAAAAAAAAACYg==","v":"CgFiSgcKBQgBEOkHggEiGiAAAQIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eH4IBJBABGiABAgMEBQYHCAkKCwwNDg8QERITFBUWFxgZGhscHR4fIA=="} +{"k":"AAAAAAAAAAACYw==","v":"CgFjOAFKBwoFCAEQ6geCASIaIAABAgMEBQYHCAkKCwwNDg8QERITFBUWFxgZGhscHR4fggEkEAEaIAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gggEkEAIaIAIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHyAhggEkEAMaIAMEBQYHCAkKCwwNDg8QERITFBUWFxgZGhscHR4fICEiggEkEAQaIAQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gISIj"} +{"k":"AAAAAAAAAAACZA==","v":"CgFkSgcKBQgBEOsHggEiGiAAAQIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eH4IBJBABGiABAgMEBQYHCAkKCwwNDg8QERITFBUWFxgZGhscHR4fIIIBJBACGiACAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gIYIBJBADGiADBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHyAhIoIBJBAEGiAEBQYHCAkKCwwNDg8QERITFBUWFxgZGhscHR4fICEiI4IBJBAFGiAFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gISIjJIIBJBAGGiAGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHyAhIiMkJQ=="} +{"k":"AAAAAAAAAAADYw==","v":"CgFjSgcKBQgBEOoHggEiGiAAAQIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eH4IBJBABGiABAgMEBQYHCAkKCwwNDg8QERITFBUWFxgZGhscHR4fIIIBJBACGiACAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gIYIBJBADGiADBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHyAhIoIBJBAEGiAEBQYHCAkKCwwNDg8QERITFBUWFxgZGhscHR4fICEiI4IBJBAFGiAFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gISIjJIIBJBAGGiAGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHyAhIiMkJQ=="} +{"k":"AAAAAAAAAAADZA==","v":"CgFkOAFKBwoFCAEQ6weCASIaIAABAgMEBQYHCAkKCwwNDg8QERITFBUWFxgZGhscHR4fggEkEAEaIAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gggEkEAIaIAIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHyAhggEkEAMaIAMEBQYHCAkKCwwNDg8QERITFBUWFxgZGhscHR4fICEiggEkEAQaIAQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gISIj"} +{"k":"AAAAAAAAAAADaW52YWxpZA==","v":"CgdpbnZhbGlkOAFKBwoFCAEQ7AeCASIaIAABAgMEBQYHCAkKCwwNDg8QERITFBUWFxgZGhscHR4fggEkEAEaIAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gggEkEAIaIAIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHyAhggEkEAMaIAMEBQYHCAkKCwwNDg8QERITFBUWFxgZGhscHR4fICEiggEkEAQaIAQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gISIj"} +{"k":"AQAAAAAvbm90Z29vZA==","v":"CisKBwoFCAEQ6AcSIP//////////////////////////////////////////"} +{"k":"AQAAAABh","v":"CisKBwoFCAEQ6AcSIP//////////////////////////////////////////"} +{"k":"AQAAAABi","v":"CisKBwoFCAEQ6QcSIAIj5b8/Vx850vCTKUE+HcWcQZUIgmhv//rEL3j3A/At"} +{"k":"AQAAAABj","v":"CisKBwoFCAEQ6gcSIEeUA//e9Ja19eW8nAoVIh5wBzFkUJ+jB2GvYwlPb5RcCi0KBwoFCAEQ6gcSIAIj5b8/Vx850vCTKUE+HcWcQZUIgmhv//rEL3j3A/AtGAE="} +{"k":"AQAAAABk","v":"CisKBwoFCAEQ6wcSIAIj5b8/Vx850vCTKUE+HcWcQZUIgmhv//rEL3j3A/AtCi0KBwoFCAEQ6wcSIEeUA//e9Ja19eW8nAoVIh5wBzFkUJ+jB2GvYwlPb5RcGAE="} +{"k":"AQAAAABpbnZhbGlk","v":"Ci0KBwoFCAEQ7AcSIEeUA//e9Ja19eW8nAoVIh5wBzFkUJ+jB2GvYwlPb5RcGAE="} +{"k":"AgAAAAAAAQIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHy9ub3Rnb29k","v":"AAAAAA=="} +{"k":"AgAAAAAAAQIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eH2E=","v":"AAAAAA=="} +{"k":"BgAAAAAAAAAA","v":"VXBkYXRlU2NoZW1hMHRvMw=="} +{"k":"BwAAAAAAAAAA","v":""} +{"k":"BwAAAAEAAAAA","v":"//////////////////////////////////////////8="} +{"k":"BwAAAAIAAAAA","v":"AiPlvz9XHznS8JMpQT4dxZxBlQiCaG//+sQvePcD8C0="} +{"k":"BwAAAAMAAAAA","v":"R5QD/970lrX15bycChUiHnAHMWRQn6MHYa9jCU9vlFw="} +{"k":"CQAAAAA=","v":"CicIAjACigEg//////////////////////////////////////////8KJwgFMAKKASD4+Pj4+Pj4+Pj4+Pj4+Pj4+Pj4+Pj4+Pj4+Pj4+Pj4+AolCAKKASACI+W/P1cfOdLwkylBPh3FnEGVCIJob//6xC949wPwLQolCAGKASBHlAP/3vSWtfXlvJwKFSIecAcxZFCfowdhr2MJT2+UXBCyn6iaw4HGlxU="} diff --git a/lib/db/util_test.go b/lib/db/util_test.go index 8eb525ace..a41de3457 100644 --- a/lib/db/util_test.go +++ b/lib/db/util_test.go @@ -11,8 +11,6 @@ import ( "io" "os" - "github.com/syncthing/syncthing/lib/fs" - "github.com/syncthing/syncthing/lib/protocol" "github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb/storage" "github.com/syndtr/goleveldb/leveldb/util" @@ -58,37 +56,51 @@ func openJSONS(file string) (*leveldb.DB, error) { return db, nil } -func generateIgnoredFilesDB() { - // This generates a database with files with invalid flags, local and - // remote, in the format used in 0.14.48. +// The following commented tests were used to generate jsons files to stdout for +// future tests and are kept here for reference (reuse). - db := OpenMemory() - fs := NewFileSet("test", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), db) - fs.Update(protocol.LocalDeviceID, []protocol.FileInfo{ - { // invalid (ignored) file - Name: "foo", - Type: protocol.FileInfoTypeFile, - Invalid: true, - Version: protocol.Vector{Counters: []protocol.Counter{{ID: 1, Value: 1000}}}, - }, - { // regular file - Name: "bar", - Type: protocol.FileInfoTypeFile, - Version: protocol.Vector{Counters: []protocol.Counter{{ID: 1, Value: 1001}}}, - }, - }) - fs.Update(protocol.DeviceID{42}, []protocol.FileInfo{ - { // invalid file - Name: "baz", - Type: protocol.FileInfoTypeFile, - Invalid: true, - Version: protocol.Vector{Counters: []protocol.Counter{{ID: 42, Value: 1000}}}, - }, - { // regular file - Name: "quux", - Type: protocol.FileInfoTypeFile, - Version: protocol.Vector{Counters: []protocol.Counter{{ID: 42, Value: 1002}}}, - }, - }) - writeJSONS(os.Stdout, db.DB) -} +// TestGenerateIgnoredFilesDB generates a database with files with invalid flags, +// local and remote, in the format used in 0.14.48. +// func TestGenerateIgnoredFilesDB(t *testing.T) { +// db := OpenMemory() +// fs := NewFileSet("test", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), db) +// fs.Update(protocol.LocalDeviceID, []protocol.FileInfo{ +// { // invalid (ignored) file +// Name: "foo", +// Type: protocol.FileInfoTypeFile, +// Invalid: true, +// Version: protocol.Vector{Counters: []protocol.Counter{{ID: 1, Value: 1000}}}, +// }, +// { // regular file +// Name: "bar", +// Type: protocol.FileInfoTypeFile, +// Version: protocol.Vector{Counters: []protocol.Counter{{ID: 1, Value: 1001}}}, +// }, +// }) +// fs.Update(protocol.DeviceID{42}, []protocol.FileInfo{ +// { // invalid file +// Name: "baz", +// Type: protocol.FileInfoTypeFile, +// Invalid: true, +// Version: protocol.Vector{Counters: []protocol.Counter{{ID: 42, Value: 1000}}}, +// }, +// { // regular file +// Name: "quux", +// Type: protocol.FileInfoTypeFile, +// Version: protocol.Vector{Counters: []protocol.Counter{{ID: 42, Value: 1002}}}, +// }, +// }) +// writeJSONS(os.Stdout, db.DB) +// } + +// TestGenerateUpdate0to3DB generates a database with files with invalid flags, prefixed +// by a slash and other files to test database migration from version 0 to 3, in the +// format used in 0.14.45. +// func TestGenerateUpdate0to3DB(t *testing.T) { +// db := OpenMemory() +// fs := NewFileSet(update0to3Folder, fs.NewFilesystem(fs.FilesystemTypeBasic, "."), db) +// for devID, files := range haveUpdate0to3 { +// fs.Update(devID, files) +// } +// writeJSONS(os.Stdout, db.DB) +// } diff --git a/lib/model/requests_test.go b/lib/model/requests_test.go index 4c1514daf..19261dd5c 100644 --- a/lib/model/requests_test.go +++ b/lib/model/requests_test.go @@ -346,7 +346,7 @@ func pullInvalidIgnored(t *testing.T, ft config.FolderType) { expected := map[string]struct{}{invIgn: {}, ign: {}, ignExisting: {}} for _, f := range fs { if _, ok := expected[f.Name]; !ok { - t.Fatalf("Unexpected file %v was added to index", f.Name) + t.Errorf("Unexpected file %v was added to index", f.Name) } if !f.Invalid { t.Errorf("File %v wasn't marked as invalid", f.Name) diff --git a/lib/protocol/bep_extensions.go b/lib/protocol/bep_extensions.go index 5dde3ea9c..cfc4b988d 100644 --- a/lib/protocol/bep_extensions.go +++ b/lib/protocol/bep_extensions.go @@ -95,6 +95,10 @@ func (f FileInfo) SequenceNo() int64 { return f.Sequence } +func (f FileInfo) FileVersion() Vector { + return f.Version +} + // WinsConflict returns true if "f" is the one to choose when it is in // conflict with "other". func (f FileInfo) WinsConflict(other FileInfo) bool {