lib/db: Schema update to repair sequence index (ref #6304) (#6350)

lib/db/schemaupdater.go

@@ -7,9 +7,11 @@
 package db
 
 import (
+	"bytes"
 	"fmt"
 	"strings"
 
+	"github.com/syncthing/syncthing/lib/db/backend"
 	"github.com/syncthing/syncthing/lib/protocol"
 )
 
@@ -23,11 +25,17 @@ import (
 //   6: v0.14.50
 //   7: v0.14.53
 //   8: v1.4.0
+//   9: v1.4.0
 const (
-	dbVersion             = 8
+	dbVersion             = 9
 	dbMinSyncthingVersion = "v1.4.0"
 )
 
+var (
+	errFolderIdxMissing = fmt.Errorf("folder db index missing")
+	errDeviceIdxMissing = fmt.Errorf("device db index missing")
+)
+
 type databaseDowngradeError struct {
 	minSyncthingVersion string
 }
@@ -80,7 +88,7 @@ func (db *schemaUpdater) updateSchema() error {
 		{5, db.updateSchemaTo5},
 		{6, db.updateSchema5to6},
 		{7, db.updateSchema6to7},
-		{8, db.updateSchema7to8},
+		{9, db.updateSchemato9},
 	}
 
 	for _, m := range migrations {
@@ -421,8 +429,9 @@ func (db *schemaUpdater) updateSchema6to7(_ int) error {
 	return t.Commit()
 }
 
-func (db *schemaUpdater) updateSchema7to8(_ int) error {
+func (db *schemaUpdater) updateSchemato9(prev int) error {
 	// Loads and rewrites all files with blocks, to deduplicate block lists.
+	// Checks for missing or incorrect sequence entries and rewrites those.
 
 	t, err := db.newReadWriteTransaction()
 	if err != nil {
@@ -430,15 +439,59 @@ func (db *schemaUpdater) updateSchema7to8(_ int) error {
 	}
 	defer t.close()
 
+	var sk []byte
 	it, err := t.NewPrefixIterator([]byte{KeyTypeDevice})
 	if err != nil {
 		return err
 	}
+	metas := make(map[string]*metadataTracker)
 	for it.Next() {
 		var fi protocol.FileInfo
 		if err := fi.Unmarshal(it.Value()); err != nil {
 			return err
 		}
+		device, ok := t.keyer.DeviceFromDeviceFileKey(it.Key())
+		if !ok {
+			return errDeviceIdxMissing
+		}
+		if bytes.Equal(device, protocol.LocalDeviceID[:]) {
+			folder, ok := t.keyer.FolderFromDeviceFileKey(it.Key())
+			if !ok {
+				return errFolderIdxMissing
+			}
+			if sk, err = t.keyer.GenerateSequenceKey(sk, folder, fi.Sequence); err != nil {
+				return err
+			}
+			switch dk, err := t.Get(sk); {
+			case err != nil:
+				if !backend.IsNotFound(err) {
+					return err
+				}
+				fallthrough
+			case !bytes.Equal(it.Key(), dk):
+				folderStr := string(folder)
+				meta, ok := metas[folderStr]
+				if !ok {
+					meta = loadMetadataTracker(db.Lowlevel, folderStr)
+					metas[folderStr] = meta
+				}
+				fi.Sequence = meta.nextLocalSeq()
+				if sk, err = t.keyer.GenerateSequenceKey(sk, folder, fi.Sequence); err != nil {
+					return err
+				}
+				if err := t.Put(sk, it.Key()); err != nil {
+					return err
+				}
+				if err := t.putFile(it.Key(), fi); err != nil {
+					return err
+				}
+				continue
+			}
+		}
+		if prev == 8 {
+			// The transition to 8 already did the changes below.
+			continue
+		}
 		if fi.Blocks == nil {
 			continue
 		}
@@ -451,6 +504,12 @@ func (db *schemaUpdater) updateSchema7to8(_ int) error {
 		return err
 	}
 
+	for folder, meta := range metas {
+		if err := meta.toDB(t, []byte(folder)); err != nil {
+			return err
+		}
+	}
+
 	db.recordTime(blockGCTimeKey)
 
 	return t.Commit()

lib/db/set.go

@@ -71,66 +71,68 @@ func init() {
 }
 
 func NewFileSet(folder string, fs fs.Filesystem, db *Lowlevel) *FileSet {
-	var s = &FileSet{
+	return &FileSet{
 		folder:      folder,
 		fs:          fs,
 		db:          db,
-		meta:        newMetadataTracker(),
+		meta:        loadMetadataTracker(db, folder),
 		updateMutex: sync.NewMutex(),
 	}
+}
 
-	recalc := func() *FileSet {
-		if err := s.recalcMeta(); backend.IsClosed(err) {
+func loadMetadataTracker(db *Lowlevel, folder string) *metadataTracker {
+	meta := newMetadataTracker()
+
+	recalc := func() *metadataTracker {
+		if err := recalcMeta(meta, db, folder); backend.IsClosed(err) {
 			return nil
 		} else if err != nil {
 			panic(err)
 		}
-		return s
+		return meta
 	}
 
-	if err := s.meta.fromDB(db, []byte(folder)); err != nil {
+	if err := meta.fromDB(db, []byte(folder)); err != nil {
 		l.Infof("No stored folder metadata for %q; recalculating", folder)
 		return recalc()
 	}
 
-	if metaOK := s.verifyLocalSequence(); !metaOK {
+	if metaOK := verifyLocalSequence(meta, db, folder); !metaOK {
 		l.Infof("Stored folder metadata for %q is out of date after crash; recalculating", folder)
 		return recalc()
 	}
 
-	if age := time.Since(s.meta.Created()); age > databaseRecheckInterval {
+	if age := time.Since(meta.Created()); age > databaseRecheckInterval {
 		l.Infof("Stored folder metadata for %q is %v old; recalculating", folder, age)
 		return recalc()
 	}
 
-	return s
+	return meta
 }
 
-func (s *FileSet) recalcMeta() error {
-	s.meta = newMetadataTracker()
-
-	if err := s.db.checkGlobals([]byte(s.folder), s.meta); err != nil {
+func recalcMeta(meta *metadataTracker, db *Lowlevel, folder string) error {
+	if err := db.checkGlobals([]byte(folder), meta); err != nil {
 		return err
 	}
 
-	t, err := s.db.newReadWriteTransaction()
+	t, err := db.newReadWriteTransaction()
 	if err != nil {
 		return err
 	}
 	defer t.close()
 
 	var deviceID protocol.DeviceID
-	err = t.withAllFolderTruncated([]byte(s.folder), func(device []byte, f FileInfoTruncated) bool {
+	err = t.withAllFolderTruncated([]byte(folder), func(device []byte, f FileInfoTruncated) bool {
 		copy(deviceID[:], device)
-		s.meta.addFile(deviceID, f)
+		meta.addFile(deviceID, f)
 		return true
 	})
 	if err != nil {
 		return err
 	}
 
-	s.meta.SetCreated()
-	if err := s.meta.toDB(t, []byte(s.folder)); err != nil {
+	meta.SetCreated()
+	if err := meta.toDB(t, []byte(folder)); err != nil {
 		return err
 	}
 	return t.Commit()
@@ -138,7 +140,7 @@ func (s *FileSet) recalcMeta() error {
 
 // Verify the local sequence number from actual sequence entries. Returns
 // true if it was all good, or false if a fixup was necessary.
-func (s *FileSet) verifyLocalSequence() bool {
+func verifyLocalSequence(meta *metadataTracker, db *Lowlevel, folder string) bool {
 	// Walk the sequence index from the current (supposedly) highest
 	// sequence number and raise the alarm if we get anything. This recovers
 	// from the occasion where we have written sequence entries to disk but
@@ -149,15 +151,20 @@ func (s *FileSet) verifyLocalSequence() bool {
 	// number than we've actually seen and receive some duplicate updates
 	// and then be in sync again.
 
-	curSeq := s.meta.Sequence(protocol.LocalDeviceID)
+	curSeq := meta.Sequence(protocol.LocalDeviceID)
 
-	snap := s.Snapshot()
+	t, err := db.newReadOnlyTransaction()
+	if err != nil {
+		panic(err)
+	}
 	ok := true
-	snap.WithHaveSequence(curSeq+1, func(fi FileIntf) bool {
+	if err := t.withHaveSequence([]byte(folder), curSeq+1, func(fi FileIntf) bool {
 		ok = false // we got something, which we should not have
 		return false
-	})
-	snap.Release()
+	}); err != nil && !backend.IsClosed(err) {
+		panic(err)
+	}
+	t.close()
 
 	return ok
 }