lib/db: Handle indirection error repairing sequences (fixes #7026) (#7525)

lib/db/db_test.go

@@ -516,7 +516,7 @@ func TestCheckGlobals(t *testing.T) {
 	}
 
 	// Clean up global entry of the now missing file
-	if repaired, err := db.checkGlobals([]byte(fs.folder)); err != nil {
+	if repaired, err := db.checkGlobals(fs.folder); err != nil {
 		t.Fatal(err)
 	} else if repaired != 1 {
 		t.Error("Expected 1 repaired global item, got", repaired)

lib/db/lowlevel.go

@@ -441,13 +441,14 @@ func (db *Lowlevel) dropDeviceFolder(device, folder []byte, meta *metadataTracke
 	return t.Commit()
 }
 
-func (db *Lowlevel) checkGlobals(folder []byte) (int, error) {
+func (db *Lowlevel) checkGlobals(folderStr string) (int, error) {
 	t, err := db.newReadWriteTransaction()
 	if err != nil {
 		return 0, err
 	}
 	defer t.close()
 
+	folder := []byte(folderStr)
 	key, err := db.keyer.GenerateGlobalVersionKey(nil, folder, nil)
 	if err != nil {
 		return 0, err
@@ -509,7 +510,7 @@ func (db *Lowlevel) checkGlobals(folder []byte) (int, error) {
 		return 0, err
 	}
 
-	l.Debugf("global db check completed for %q", folder)
+	l.Debugf("global db check completed for %v", folder)
 	return fixed, t.Commit()
 }
 
@@ -834,8 +835,10 @@ func (b *bloomFilter) hash(id []byte) uint64 {
 
 // checkRepair checks folder metadata and sequences for miscellaneous errors.
 func (db *Lowlevel) checkRepair() error {
+	db.gcMut.RLock()
+	defer db.gcMut.RUnlock()
 	for _, folder := range db.ListFolders() {
-		if _, err := db.getMetaAndCheck(folder); err != nil {
+		if _, err := db.getMetaAndCheckGCLocked(folder); err != nil {
 			return err
 		}
 	}
@@ -858,6 +861,14 @@ func (db *Lowlevel) getMetaAndCheckGCLocked(folder string) (*metadataTracker, er
 		l.Infof("Repaired %d local need entries for folder %v in database", fixed, folder)
 	}
 
+	fixed, err = db.checkGlobals(folder)
+	if err != nil {
+		return nil, fmt.Errorf("checking globals: %w", err)
+	}
+	if fixed != 0 {
+		l.Infof("Repaired %d global entries for folder %v in database", fixed, folder)
+	}
+
 	meta, err := db.recalcMeta(folder)
 	if err != nil {
 		return nil, fmt.Errorf("recalculating metadata: %w", err)
@@ -869,6 +880,10 @@ func (db *Lowlevel) getMetaAndCheckGCLocked(folder string) (*metadataTracker, er
 	}
 	if fixed != 0 {
 		l.Infof("Repaired %d sequence entries for folder %v in database", fixed, folder)
+		meta, err = db.recalcMeta(folder)
+		if err != nil {
+			return nil, fmt.Errorf("recalculating metadata: %w", err)
+		}
 	}
 
 	return meta, nil
@@ -906,11 +921,6 @@ func (db *Lowlevel) recalcMeta(folderStr string) (*metadataTracker, error) {
 	folder := []byte(folderStr)
 
 	meta := newMetadataTracker(db.keyer, db.evLogger)
-	if fixed, err := db.checkGlobals(folder); err != nil {
-		return nil, fmt.Errorf("checking globals: %w", err)
-	} else if fixed > 0 {
-		l.Infof("Repaired %d global entries for folder %v in database", fixed, folderStr)
-	}
 
 	t, err := db.newReadWriteTransaction(meta.CommitHook(folder))
 	if err != nil {
@@ -1022,6 +1032,34 @@ func (db *Lowlevel) repairSequenceGCLocked(folderStr string, meta *metadataTrack
 	for it.Next() {
 		intf, err := t.unmarshalTrunc(it.Value(), false)
 		if err != nil {
+			// Delete local items with invalid indirected blocks/versions.
+			// They will be rescanned.
+			var ierr *blocksIndirectionError
+			if ok := errors.As(err, &ierr); ok && backend.IsNotFound(err) {
+				intf, err = t.unmarshalTrunc(it.Value(), true)
+				if err != nil {
+					return 0, err
+				}
+				name := []byte(intf.FileName())
+				gk, err := t.keyer.GenerateGlobalVersionKey(nil, folder, name)
+				if err != nil {
+					return 0, err
+				}
+				_, err = t.removeFromGlobal(gk, nil, folder, protocol.LocalDeviceID[:], name, nil)
+				if err != nil {
+					return 0, err
+				}
+				sk, err = db.keyer.GenerateSequenceKey(sk, folder, intf.SequenceNo())
+				if err != nil {
+					return 0, err
+				}
+				if err := t.Delete(sk); err != nil {
+					return 0, err
+				}
+				if err := t.Delete(it.Key()); err != nil {
+					return 0, err
+				}
+			}
 			return 0, err
 		}
 		fi := intf.(protocol.FileInfo)

lib/db/schemaupdater.go

@@ -100,7 +100,6 @@ func (db *schemaUpdater) updateSchema() error {
 		{11, 11, "v1.6.0", db.updateSchemaTo11},
 		{13, 13, "v1.7.0", db.updateSchemaTo13},
 		{14, 14, "v1.9.0", db.updateSchemaTo14},
-		{14, 15, "v1.9.0", db.migration15},
 		{14, 16, "v1.9.0", db.checkRepairMigration},
 	}
 
@@ -774,15 +773,6 @@ func (db *schemaUpdater) updateSchemaTo14(_ int) error {
 	return nil
 }
 
-func (db *schemaUpdater) migration15(_ int) error {
-	for _, folder := range db.ListFolders() {
-		if _, err := db.recalcMeta(folder); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
 func (db *schemaUpdater) checkRepairMigration(_ int) error {
 	for _, folder := range db.ListFolders() {
 		_, err := db.getMetaAndCheckGCLocked(folder)

lib/db/transactions.go

@@ -103,6 +103,18 @@ func (t readOnlyTransaction) unmarshalTrunc(bs []byte, trunc bool) (protocol.Fil
 	return fi, nil
 }
 
+type blocksIndirectionError struct {
+	err error
+}
+
+func (e *blocksIndirectionError) Error() string {
+	return fmt.Sprintf("filling Blocks: %v", e.err)
+}
+
+func (e *blocksIndirectionError) Unwrap() error {
+	return e.err
+}
+
 // fillFileInfo follows the (possible) indirection of blocks and version
 // vector and fills it out.
 func (t readOnlyTransaction) fillFileInfo(fi *protocol.FileInfo) error {
@@ -113,7 +125,7 @@ func (t readOnlyTransaction) fillFileInfo(fi *protocol.FileInfo) error {
 		key = t.keyer.GenerateBlockListKey(key, fi.BlocksHash)
 		bs, err := t.Get(key)
 		if err != nil {
-			return fmt.Errorf("filling Blocks: %w", err)
+			return &blocksIndirectionError{err}
 		}
 		var bl BlockList
 		if err := bl.Unmarshal(bs); err != nil {