lib/db: Catch unignored/conflicting files as needed (fixes #5053) (#5054)

lib/db/leveldb_dbinstance_updateschema.go

@@ -20,10 +20,11 @@ import (
 //   2: v0.14.48
 //   3: v0.14.49
 //   4: v0.14.49
-//   5: v0.14.50
+//   5: v0.14.49
+//   6: v0.14.50
 const (
-	dbVersion             = 5
-	dbMinSyncthingVersion = "v0.14.49"
+	dbVersion             = 6
+	dbMinSyncthingVersion = "v0.14.50"
 )
 
 type databaseDowngradeError struct {
@@ -62,12 +63,12 @@ func (db *Instance) updateSchema() error {
 	if prevVersion < 3 {
 		db.updateSchema2to3()
 	}
-	// This update fixes a problem that only exists in dbVersion 3.
-	if prevVersion == 3 {
-		db.updateSchema3to4()
+	// This update fixes problems existing in versions 3 and 4
+	if prevVersion == 3 || prevVersion == 4 {
+		db.updateSchemaTo5()
 	}
-	if prevVersion < 5 {
-		db.updateSchema4to5()
+	if prevVersion < 6 {
+		db.updateSchema5to6()
 	}
 
 	miscDB.PutInt64("dbVersion", dbVersion)
@@ -192,10 +193,11 @@ func (db *Instance) updateSchema2to3() {
 	}
 }
 
-// updateSchema3to4 resets the need bucket due a bug existing in dbVersion 3 /
-// v0.14.49-rc.1
+// updateSchemaTo5 resets the need bucket due to bugs existing in the v0.14.49
+// release candidates (dbVersion 3 and 4)
 // https://github.com/syncthing/syncthing/issues/5007
-func (db *Instance) updateSchema3to4() {
+// https://github.com/syncthing/syncthing/issues/5053
+func (db *Instance) updateSchemaTo5() {
 	t := db.newReadWriteTransaction()
 	var nk []byte
 	for _, folderStr := range db.ListFolders() {
@@ -207,7 +209,7 @@ func (db *Instance) updateSchema3to4() {
 	db.updateSchema2to3()
 }
 
-func (db *Instance) updateSchema4to5() {
+func (db *Instance) updateSchema5to6() {
 	// For every local file with the Invalid bit set, clear the Invalid bit and
 	// set LocalFlags = FlagLocalIgnored.
 

lib/db/leveldb_transactions.go

@@ -99,11 +99,31 @@ func (t readWriteTransaction) updateGlobal(gk, folder, device []byte, file proto
 
 	name := []byte(file.Name)
 
-	if removedAt != 0 && insertedAt != 0 {
-		if bytes.Equal(device, protocol.LocalDeviceID[:]) && file.Version.Equal(fl.Versions[0].Version) {
-			l.Debugf("local need delete; folder=%q, name=%q", folder, name)
-			t.Delete(t.db.needKey(folder, name))
+	var newGlobal protocol.FileInfo
+	if insertedAt == 0 {
+		// Inserted a new newest version
+		newGlobal = file
+	} else if new, ok := t.getFile(folder, fl.Versions[0].Device, name); ok {
+		// The previous second version is now the first
+		newGlobal = new
+	} else {
+		panic("This file must exist in the db")
+	}
+
+	// Fixup the list of files we need.
+	nk := t.db.needKey(folder, name)
+	hasNeeded, _ := t.db.Has(nk, nil)
+	if localFV, haveLocalFV := fl.Get(protocol.LocalDeviceID[:]); need(newGlobal, haveLocalFV, localFV.Version) {
+		if !hasNeeded {
+			l.Debugf("local need insert; folder=%q, name=%q", folder, name)
+			t.Put(nk, nil)
 		}
+	} else if hasNeeded {
+		l.Debugf("local need delete; folder=%q, name=%q", folder, name)
+		t.Delete(nk)
+	}
+
+	if removedAt != 0 && insertedAt != 0 {
 		l.Debugf(`new global for "%v" after update: %v`, file.Name, fl)
 		t.Put(gk, mustMarshal(&fl))
 		return true
@@ -124,31 +144,8 @@ func (t readWriteTransaction) updateGlobal(gk, folder, device []byte, file proto
 	}
 
 	// Add the new global to the global size counter
-	var newGlobal protocol.FileInfo
-	if insertedAt == 0 {
-		// Inserted a new newest version
-		newGlobal = file
-	} else if new, ok := t.getFile(folder, fl.Versions[0].Device, name); ok {
-		// The previous second version is now the first
-		newGlobal = new
-	} else {
-		panic("This file must exist in the db")
-	}
 	meta.addFile(globalDeviceID, newGlobal)
 
-	// Fixup the list of files we need.
-	nk := t.db.needKey(folder, name)
-	hasNeeded, _ := t.db.Has(nk, nil)
-	if localFV, haveLocalFV := fl.Get(protocol.LocalDeviceID[:]); need(newGlobal, haveLocalFV, localFV.Version) {
-		if !hasNeeded {
-			l.Debugf("local need insert; folder=%q, name=%q", folder, name)
-			t.Put(nk, nil)
-		}
-	} else if hasNeeded {
-		l.Debugf("local need delete; folder=%q, name=%q", folder, name)
-		t.Delete(nk)
-	}
-
 	l.Debugf(`new global for "%v" after update: %v`, file.Name, fl)
 	t.Put(gk, mustMarshal(&fl))
 

lib/db/set_test.go

@@ -1065,6 +1065,36 @@ func TestNeedDeleted(t *testing.T) {
 	}
 }
 
+func TestNeedAfterUnignore(t *testing.T) {
+	ldb := db.OpenMemory()
+
+	folder := "test"
+	file := "foo"
+	s := db.NewFileSet(folder, fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
+
+	remID := remoteDevice0.Short()
+
+	// Initial state: Devices in sync, locally ignored
+	local := protocol.FileInfo{Name: file, Version: protocol.Vector{Counters: []protocol.Counter{{ID: remID, Value: 1}, {ID: myID, Value: 1}}}, ModifiedS: 10}
+	local.SetIgnored(myID)
+	remote := protocol.FileInfo{Name: file, Version: protocol.Vector{Counters: []protocol.Counter{{ID: remID, Value: 1}, {ID: myID, Value: 1}}}, ModifiedS: 10}
+	s.Update(protocol.LocalDeviceID, fileList{local})
+	s.Update(remoteDevice0, fileList{remote})
+
+	// Unignore locally -> conflicting changes. Remote is newer, thus winning.
+	local.Version = local.Version.Update(myID)
+	local.Version = local.Version.DropOthers(myID)
+	local.LocalFlags = 0
+	local.ModifiedS = 0
+	s.Update(protocol.LocalDeviceID, fileList{local})
+
+	if need := needList(s, protocol.LocalDeviceID); len(need) != 1 {
+		t.Fatal("Expected one local need, got", need)
+	} else if !need[0].IsEquivalent(remote, false, false) {
+		t.Fatalf("Got %v, expected %v", need[0], remote)
+	}
+}
+
 func replace(fs *db.FileSet, device protocol.DeviceID, files []protocol.FileInfo) {
 	fs.Drop(device)
 	fs.Update(device, files)