From 1471c15b29dfd4b0dd1f990a20d04e4e4f619fbd Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Sat, 10 Mar 2018 11:42:01 +0100 Subject: [PATCH] cmd/syncthing, lib/db: Be nicer about dropping deltas on upgrade (#4798) When dropping delta index IDs due to upgrade, only drop our local one. Previously, when dropping all of them, we would trigger a full send in both directions on first connect after upgrade. Then the other side would upgrade, doing the same thing. Net effect is full index data gets sent twice in both directions. With this change we just drop our local ID, meaning we will send our full index on first connect after upgrade. When the other side upgrades, they will do the same. This is a bit less cruel. --- cmd/syncthing/main.go | 7 +-- lib/db/leveldb_dbinstance.go | 40 ++++++++++++++-- lib/db/leveldb_test.go | 90 ++++++++++++++++++++++++++++++++++++ 3 files changed, 130 insertions(+), 7 deletions(-) diff --git a/cmd/syncthing/main.go b/cmd/syncthing/main.go index 1b71dd5f3..191558ea7 100644 --- a/cmd/syncthing/main.go +++ b/cmd/syncthing/main.go @@ -726,7 +726,8 @@ func syncthingMain(runtimeOptions RuntimeOptions) { if runtimeOptions.resetDeltaIdxs { l.Infoln("Reinitializing delta index IDs") - ldb.DropDeltaIndexIDs() + ldb.DropLocalDeltaIndexIDs() + ldb.DropRemoteDeltaIndexIDs() } protectedFiles := []string{ @@ -762,8 +763,8 @@ func syncthingMain(runtimeOptions RuntimeOptions) { } // Drop delta indexes in case we've changed random stuff we - // shouldn't have. - ldb.DropDeltaIndexIDs() + // shouldn't have. We will resend our index on next connect. + ldb.DropLocalDeltaIndexIDs() // Remember the new version. miscDB.PutString("prevVersion", Version) diff --git a/lib/db/leveldb_dbinstance.go b/lib/db/leveldb_dbinstance.go index 5c56b8c75..cd3973cfb 100644 --- a/lib/db/leveldb_dbinstance.go +++ b/lib/db/leveldb_dbinstance.go @@ -745,6 +745,15 @@ func (db *Instance) indexIDKey(device, folder []byte) []byte { return k } +func (db *Instance) indexIDDevice(key []byte) []byte { + device, ok := db.deviceIdx.Val(binary.BigEndian.Uint32(key[keyPrefixLen:])) + if !ok { + // uuh ... + return nil + } + return device +} + func (db *Instance) mtimesKey(folder []byte) []byte { prefix := make([]byte, 5) // key type + 4 bytes folder idx number prefix[0] = KeyTypeVirtualMtime @@ -759,10 +768,33 @@ func (db *Instance) folderMetaKey(folder []byte) []byte { return prefix } -// DropDeltaIndexIDs removes all index IDs from the database. This will -// cause a full index transmission on the next connection. -func (db *Instance) DropDeltaIndexIDs() { - db.dropPrefix([]byte{KeyTypeIndexID}) +// DropLocalDeltaIndexIDs removes all index IDs for the local device ID from +// the database. This will cause a full index transmission on the next +// connection. +func (db *Instance) DropLocalDeltaIndexIDs() { + db.dropDeltaIndexIDs(true) +} + +// DropRemoteDeltaIndexIDs removes all index IDs for the other devices than +// the local one from the database. This will cause them to send us a full +// index on the next connection. +func (db *Instance) DropRemoteDeltaIndexIDs() { + db.dropDeltaIndexIDs(false) +} + +func (db *Instance) dropDeltaIndexIDs(local bool) { + t := db.newReadWriteTransaction() + defer t.close() + + dbi := t.NewIterator(util.BytesPrefix([]byte{KeyTypeIndexID}), nil) + defer dbi.Release() + + for dbi.Next() { + device := db.indexIDDevice(dbi.Key()) + if bytes.Equal(device, protocol.LocalDeviceID[:]) == local { + t.Delete(dbi.Key()) + } + } } func (db *Instance) dropMtimes(folder []byte) { diff --git a/lib/db/leveldb_test.go b/lib/db/leveldb_test.go index 035680f2b..5dd58e0a7 100644 --- a/lib/db/leveldb_test.go +++ b/lib/db/leveldb_test.go @@ -9,6 +9,8 @@ package db import ( "bytes" "testing" + + "github.com/syncthing/syncthing/lib/protocol" ) func TestDeviceKey(t *testing.T) { @@ -62,3 +64,91 @@ func TestGlobalKey(t *testing.T) { t.Error("should not have been found") } } + +func TestDropIndexIDs(t *testing.T) { + db := OpenMemory() + + d1 := []byte("device67890123456789012345678901") + d2 := []byte("device12345678901234567890123456") + + // Set some index IDs + + db.setIndexID(protocol.LocalDeviceID[:], []byte("foo"), 1) + db.setIndexID(protocol.LocalDeviceID[:], []byte("bar"), 2) + db.setIndexID(d1, []byte("foo"), 3) + db.setIndexID(d1, []byte("bar"), 4) + db.setIndexID(d2, []byte("foo"), 5) + db.setIndexID(d2, []byte("bar"), 6) + + // Verify them + + if db.getIndexID(protocol.LocalDeviceID[:], []byte("foo")) != 1 { + t.Fatal("fail local 1") + } + if db.getIndexID(protocol.LocalDeviceID[:], []byte("bar")) != 2 { + t.Fatal("fail local 2") + } + if db.getIndexID(d1, []byte("foo")) != 3 { + t.Fatal("fail remote 1") + } + if db.getIndexID(d1, []byte("bar")) != 4 { + t.Fatal("fail remote 2") + } + if db.getIndexID(d2, []byte("foo")) != 5 { + t.Fatal("fail remote 3") + } + if db.getIndexID(d2, []byte("bar")) != 6 { + t.Fatal("fail remote 4") + } + + // Drop the local ones, verify only they got dropped + + db.DropLocalDeltaIndexIDs() + + if db.getIndexID(protocol.LocalDeviceID[:], []byte("foo")) != 0 { + t.Fatal("fail local 1") + } + if db.getIndexID(protocol.LocalDeviceID[:], []byte("bar")) != 0 { + t.Fatal("fail local 2") + } + if db.getIndexID(d1, []byte("foo")) != 3 { + t.Fatal("fail remote 1") + } + if db.getIndexID(d1, []byte("bar")) != 4 { + t.Fatal("fail remote 2") + } + if db.getIndexID(d2, []byte("foo")) != 5 { + t.Fatal("fail remote 3") + } + if db.getIndexID(d2, []byte("bar")) != 6 { + t.Fatal("fail remote 4") + } + + // Set local ones again + + db.setIndexID(protocol.LocalDeviceID[:], []byte("foo"), 1) + db.setIndexID(protocol.LocalDeviceID[:], []byte("bar"), 2) + + // Drop the remote ones, verify only they got dropped + + db.DropRemoteDeltaIndexIDs() + + if db.getIndexID(protocol.LocalDeviceID[:], []byte("foo")) != 1 { + t.Fatal("fail local 1") + } + if db.getIndexID(protocol.LocalDeviceID[:], []byte("bar")) != 2 { + t.Fatal("fail local 2") + } + if db.getIndexID(d1, []byte("foo")) != 0 { + t.Fatal("fail remote 1") + } + if db.getIndexID(d1, []byte("bar")) != 0 { + t.Fatal("fail remote 2") + } + if db.getIndexID(d2, []byte("foo")) != 0 { + t.Fatal("fail remote 3") + } + if db.getIndexID(d2, []byte("bar")) != 0 { + t.Fatal("fail remote 4") + } +}