diff --git a/lib/db/schemaupdater.go b/lib/db/schemaupdater.go index 77a117ddb..29349fe9a 100644 --- a/lib/db/schemaupdater.go +++ b/lib/db/schemaupdater.go @@ -7,9 +7,11 @@ package db import ( + "bytes" "fmt" "strings" + "github.com/syncthing/syncthing/lib/db/backend" "github.com/syncthing/syncthing/lib/protocol" ) @@ -23,11 +25,17 @@ import ( // 6: v0.14.50 // 7: v0.14.53 // 8: v1.4.0 +// 9: v1.4.0 const ( - dbVersion = 8 + dbVersion = 9 dbMinSyncthingVersion = "v1.4.0" ) +var ( + errFolderIdxMissing = fmt.Errorf("folder db index missing") + errDeviceIdxMissing = fmt.Errorf("device db index missing") +) + type databaseDowngradeError struct { minSyncthingVersion string } @@ -80,7 +88,7 @@ func (db *schemaUpdater) updateSchema() error { {5, db.updateSchemaTo5}, {6, db.updateSchema5to6}, {7, db.updateSchema6to7}, - {8, db.updateSchema7to8}, + {9, db.updateSchemato9}, } for _, m := range migrations { @@ -421,8 +429,9 @@ func (db *schemaUpdater) updateSchema6to7(_ int) error { return t.Commit() } -func (db *schemaUpdater) updateSchema7to8(_ int) error { +func (db *schemaUpdater) updateSchemato9(prev int) error { // Loads and rewrites all files with blocks, to deduplicate block lists. + // Checks for missing or incorrect sequence entries and rewrites those. t, err := db.newReadWriteTransaction() if err != nil { @@ -430,15 +439,59 @@ func (db *schemaUpdater) updateSchema7to8(_ int) error { } defer t.close() + var sk []byte it, err := t.NewPrefixIterator([]byte{KeyTypeDevice}) if err != nil { return err } + metas := make(map[string]*metadataTracker) for it.Next() { var fi protocol.FileInfo if err := fi.Unmarshal(it.Value()); err != nil { return err } + device, ok := t.keyer.DeviceFromDeviceFileKey(it.Key()) + if !ok { + return errDeviceIdxMissing + } + if bytes.Equal(device, protocol.LocalDeviceID[:]) { + folder, ok := t.keyer.FolderFromDeviceFileKey(it.Key()) + if !ok { + return errFolderIdxMissing + } + if sk, err = t.keyer.GenerateSequenceKey(sk, folder, fi.Sequence); err != nil { + return err + } + switch dk, err := t.Get(sk); { + case err != nil: + if !backend.IsNotFound(err) { + return err + } + fallthrough + case !bytes.Equal(it.Key(), dk): + folderStr := string(folder) + meta, ok := metas[folderStr] + if !ok { + meta = loadMetadataTracker(db.Lowlevel, folderStr) + metas[folderStr] = meta + } + fi.Sequence = meta.nextLocalSeq() + if sk, err = t.keyer.GenerateSequenceKey(sk, folder, fi.Sequence); err != nil { + return err + } + if err := t.Put(sk, it.Key()); err != nil { + return err + } + if err := t.putFile(it.Key(), fi); err != nil { + return err + } + continue + } + } + if prev == 8 { + // The transition to 8 already did the changes below. + continue + } if fi.Blocks == nil { continue } @@ -451,6 +504,12 @@ func (db *schemaUpdater) updateSchema7to8(_ int) error { return err } + for folder, meta := range metas { + if err := meta.toDB(t, []byte(folder)); err != nil { + return err + } + } + db.recordTime(blockGCTimeKey) return t.Commit() diff --git a/lib/db/set.go b/lib/db/set.go index 5a8d94cde..e84659d28 100644 --- a/lib/db/set.go +++ b/lib/db/set.go @@ -71,66 +71,68 @@ func init() { } func NewFileSet(folder string, fs fs.Filesystem, db *Lowlevel) *FileSet { - var s = &FileSet{ + return &FileSet{ folder: folder, fs: fs, db: db, - meta: newMetadataTracker(), + meta: loadMetadataTracker(db, folder), updateMutex: sync.NewMutex(), } +} - recalc := func() *FileSet { - if err := s.recalcMeta(); backend.IsClosed(err) { +func loadMetadataTracker(db *Lowlevel, folder string) *metadataTracker { + meta := newMetadataTracker() + + recalc := func() *metadataTracker { + if err := recalcMeta(meta, db, folder); backend.IsClosed(err) { return nil } else if err != nil { panic(err) } - return s + return meta } - if err := s.meta.fromDB(db, []byte(folder)); err != nil { + if err := meta.fromDB(db, []byte(folder)); err != nil { l.Infof("No stored folder metadata for %q; recalculating", folder) return recalc() } - if metaOK := s.verifyLocalSequence(); !metaOK { + if metaOK := verifyLocalSequence(meta, db, folder); !metaOK { l.Infof("Stored folder metadata for %q is out of date after crash; recalculating", folder) return recalc() } - if age := time.Since(s.meta.Created()); age > databaseRecheckInterval { + if age := time.Since(meta.Created()); age > databaseRecheckInterval { l.Infof("Stored folder metadata for %q is %v old; recalculating", folder, age) return recalc() } - return s + return meta } -func (s *FileSet) recalcMeta() error { - s.meta = newMetadataTracker() - - if err := s.db.checkGlobals([]byte(s.folder), s.meta); err != nil { +func recalcMeta(meta *metadataTracker, db *Lowlevel, folder string) error { + if err := db.checkGlobals([]byte(folder), meta); err != nil { return err } - t, err := s.db.newReadWriteTransaction() + t, err := db.newReadWriteTransaction() if err != nil { return err } defer t.close() var deviceID protocol.DeviceID - err = t.withAllFolderTruncated([]byte(s.folder), func(device []byte, f FileInfoTruncated) bool { + err = t.withAllFolderTruncated([]byte(folder), func(device []byte, f FileInfoTruncated) bool { copy(deviceID[:], device) - s.meta.addFile(deviceID, f) + meta.addFile(deviceID, f) return true }) if err != nil { return err } - s.meta.SetCreated() - if err := s.meta.toDB(t, []byte(s.folder)); err != nil { + meta.SetCreated() + if err := meta.toDB(t, []byte(folder)); err != nil { return err } return t.Commit() @@ -138,7 +140,7 @@ func (s *FileSet) recalcMeta() error { // Verify the local sequence number from actual sequence entries. Returns // true if it was all good, or false if a fixup was necessary. -func (s *FileSet) verifyLocalSequence() bool { +func verifyLocalSequence(meta *metadataTracker, db *Lowlevel, folder string) bool { // Walk the sequence index from the current (supposedly) highest // sequence number and raise the alarm if we get anything. This recovers // from the occasion where we have written sequence entries to disk but @@ -149,15 +151,20 @@ func (s *FileSet) verifyLocalSequence() bool { // number than we've actually seen and receive some duplicate updates // and then be in sync again. - curSeq := s.meta.Sequence(protocol.LocalDeviceID) + curSeq := meta.Sequence(protocol.LocalDeviceID) - snap := s.Snapshot() + t, err := db.newReadOnlyTransaction() + if err != nil { + panic(err) + } ok := true - snap.WithHaveSequence(curSeq+1, func(fi FileIntf) bool { + if err := t.withHaveSequence([]byte(folder), curSeq+1, func(fi FileIntf) bool { ok = false // we got something, which we should not have return false - }) - snap.Release() + }); err != nil && !backend.IsClosed(err) { + panic(err) + } + t.close() return ok }