mirror of
https://github.com/octoleo/syncthing.git
synced 2024-12-22 19:08:58 +00:00
lib/db: Move database schema migration into its own file (#4985)
This commit is contained in:
Simon Frei
Jakob Borg
parent
4781e1d86f
commit
30056cd1ae
@ -751,9 +751,6 @@ func syncthingMain(runtimeOptions RuntimeOptions) {
|
||||
miscDB.PutString("prevVersion", Version)
|
||||
}
|
||||
|
||||
// Potential database transitions
|
||||
ldb.UpdateSchema()
|
||||
|
||||
m := model.NewModel(cfg, myID, "syncthing", Version, ldb, protectedFiles)
|
||||
|
||||
if t := os.Getenv("STDEADLOCKTIMEOUT"); t != "" {
|
||||
|
||||
@ -13,8 +13,6 @@ import (
|
||||
"github.com/syncthing/syncthing/lib/protocol"
|
||||
)
|
||||
|
||||
const dbVersion = 3
|
||||
|
||||
const (
|
||||
KeyTypeDevice = iota
|
||||
KeyTypeGlobal
|
||||
|
||||
@ -83,34 +83,10 @@ func newDBInstance(db *leveldb.DB, location string) *Instance {
|
||||
}
|
||||
i.folderIdx = newSmallIndex(i, []byte{KeyTypeFolderIdx})
|
||||
i.deviceIdx = newSmallIndex(i, []byte{KeyTypeDeviceIdx})
|
||||
i.updateSchema()
|
||||
return i
|
||||
}
|
||||
|
||||
// UpdateSchema does transitions to the current db version if necessary
|
||||
func (db *Instance) UpdateSchema() {
|
||||
miscDB := NewNamespacedKV(db, string(KeyTypeMiscData))
|
||||
prevVersion, _ := miscDB.Int64("dbVersion")
|
||||
|
||||
if prevVersion >= dbVersion {
|
||||
return
|
||||
}
|
||||
|
||||
l.Infof("Updating database schema version from %v to %v...", prevVersion, dbVersion)
|
||||
|
||||
if prevVersion == 0 {
|
||||
db.updateSchema0to1()
|
||||
}
|
||||
if prevVersion <= 1 {
|
||||
db.updateSchema1to2()
|
||||
}
|
||||
if prevVersion <= 2 {
|
||||
db.updateSchema2to3()
|
||||
}
|
||||
|
||||
l.Infof("Finished updating database schema version from %v to %v", prevVersion, dbVersion)
|
||||
miscDB.PutInt64("dbVersion", dbVersion)
|
||||
}
|
||||
|
||||
// Committed returns the number of items committed to the database since startup
|
||||
func (db *Instance) Committed() int64 {
|
||||
return atomic.LoadInt64(&db.committed)
|
||||
@ -620,124 +596,6 @@ func (db *Instance) checkGlobals(folder []byte, meta *metadataTracker) {
|
||||
l.Debugf("db check completed for %q", folder)
|
||||
}
|
||||
|
||||
func (db *Instance) updateSchema0to1() {
|
||||
t := db.newReadWriteTransaction()
|
||||
defer t.close()
|
||||
|
||||
dbi := t.NewIterator(util.BytesPrefix([]byte{KeyTypeDevice}), nil)
|
||||
defer dbi.Release()
|
||||
|
||||
symlinkConv := 0
|
||||
changedFolders := make(map[string]struct{})
|
||||
ignAdded := 0
|
||||
meta := newMetadataTracker() // dummy metadata tracker
|
||||
var gk []byte
|
||||
|
||||
for dbi.Next() {
|
||||
folder := db.deviceKeyFolder(dbi.Key())
|
||||
device := db.deviceKeyDevice(dbi.Key())
|
||||
name := db.deviceKeyName(dbi.Key())
|
||||
|
||||
// Remove files with absolute path (see #4799)
|
||||
if strings.HasPrefix(string(name), "/") {
|
||||
if _, ok := changedFolders[string(folder)]; !ok {
|
||||
changedFolders[string(folder)] = struct{}{}
|
||||
}
|
||||
gk = db.globalKeyInto(gk, folder, name)
|
||||
t.removeFromGlobal(gk, folder, device, nil, nil)
|
||||
t.Delete(dbi.Key())
|
||||
t.checkFlush()
|
||||
continue
|
||||
}
|
||||
|
||||
// Change SYMLINK_FILE and SYMLINK_DIRECTORY types to the current SYMLINK
|
||||
// type (previously SYMLINK_UNKNOWN). It does this for all devices, both
|
||||
// local and remote, and does not reset delta indexes. It shouldn't really
|
||||
// matter what the symlink type is, but this cleans it up for a possible
|
||||
// future when SYMLINK_FILE and SYMLINK_DIRECTORY are no longer understood.
|
||||
var f protocol.FileInfo
|
||||
if err := f.Unmarshal(dbi.Value()); err != nil {
|
||||
// probably can't happen
|
||||
continue
|
||||
}
|
||||
if f.Type == protocol.FileInfoTypeDeprecatedSymlinkDirectory || f.Type == protocol.FileInfoTypeDeprecatedSymlinkFile {
|
||||
f.Type = protocol.FileInfoTypeSymlink
|
||||
bs, err := f.Marshal()
|
||||
if err != nil {
|
||||
panic("can't happen: " + err.Error())
|
||||
}
|
||||
t.Put(dbi.Key(), bs)
|
||||
t.checkFlush()
|
||||
symlinkConv++
|
||||
}
|
||||
|
||||
// Add invalid files to global list
|
||||
if f.Invalid {
|
||||
gk = db.globalKeyInto(gk, folder, name)
|
||||
if t.updateGlobal(gk, folder, device, f, meta) {
|
||||
if _, ok := changedFolders[string(folder)]; !ok {
|
||||
changedFolders[string(folder)] = struct{}{}
|
||||
}
|
||||
ignAdded++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for folder := range changedFolders {
|
||||
db.dropFolderMeta([]byte(folder))
|
||||
}
|
||||
|
||||
l.Infof("Updated symlink type for %d index entries and added %d invalid files to global list", symlinkConv, ignAdded)
|
||||
}
|
||||
|
||||
// updateSchema1to2 introduces a sequenceKey->deviceKey bucket for local items
|
||||
// to allow iteration in sequence order (simplifies sending indexes).
|
||||
func (db *Instance) updateSchema1to2() {
|
||||
t := db.newReadWriteTransaction()
|
||||
defer t.close()
|
||||
|
||||
var sk []byte
|
||||
var dk []byte
|
||||
for _, folderStr := range db.ListFolders() {
|
||||
folder := []byte(folderStr)
|
||||
db.withHave(folder, protocol.LocalDeviceID[:], nil, true, func(f FileIntf) bool {
|
||||
sk = db.sequenceKeyInto(sk, folder, f.SequenceNo())
|
||||
dk = db.deviceKeyInto(dk, folder, protocol.LocalDeviceID[:], []byte(f.FileName()))
|
||||
t.Put(sk, dk)
|
||||
t.checkFlush()
|
||||
return true
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// updateSchema2to3 introduces a needKey->nil bucket for locally needed files.
|
||||
func (db *Instance) updateSchema2to3() {
|
||||
t := db.newReadWriteTransaction()
|
||||
defer t.close()
|
||||
|
||||
var nk []byte
|
||||
var dk []byte
|
||||
for _, folderStr := range db.ListFolders() {
|
||||
folder := []byte(folderStr)
|
||||
db.withGlobal(folder, nil, true, func(f FileIntf) bool {
|
||||
name := []byte(f.FileName())
|
||||
dk = db.deviceKeyInto(dk, folder, protocol.LocalDeviceID[:], name)
|
||||
var v protocol.Vector
|
||||
haveFile, ok := db.getFileTrunc(dk, true)
|
||||
if ok {
|
||||
v = haveFile.FileVersion()
|
||||
}
|
||||
if !need(f, ok, v) {
|
||||
return true
|
||||
}
|
||||
nk = t.db.needKeyInto(nk, folder, []byte(f.FileName()))
|
||||
t.Put(nk, nil)
|
||||
t.checkFlush()
|
||||
return true
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// deviceKey returns a byte slice encoding the following information:
|
||||
// keyTypeDevice (1 byte)
|
||||
// folder (4 bytes)
|
||||
|
||||
155
lib/db/leveldb_dbinstance_updateschema.go
Normal file
155
lib/db/leveldb_dbinstance_updateschema.go
Normal file
@ -0,0 +1,155 @@
|
||||
// Copyright (C) 2018 The Syncthing Authors.
|
||||
//
|
||||
// This Source Code Form is subject to the terms of the Mozilla Public
|
||||
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
// You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
package db
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/syncthing/syncthing/lib/protocol"
|
||||
"github.com/syndtr/goleveldb/leveldb/util"
|
||||
)
|
||||
|
||||
const dbVersion = 3
|
||||
|
||||
func (db *Instance) updateSchema() {
|
||||
miscDB := NewNamespacedKV(db, string(KeyTypeMiscData))
|
||||
prevVersion, _ := miscDB.Int64("dbVersion")
|
||||
|
||||
if prevVersion >= dbVersion {
|
||||
return
|
||||
}
|
||||
|
||||
l.Infof("Updating database schema version from %v to %v...", prevVersion, dbVersion)
|
||||
|
||||
if prevVersion < 1 {
|
||||
db.updateSchema0to1()
|
||||
}
|
||||
if prevVersion < 2 {
|
||||
db.updateSchema1to2()
|
||||
}
|
||||
if prevVersion < 3 {
|
||||
db.updateSchema2to3()
|
||||
}
|
||||
|
||||
miscDB.PutInt64("dbVersion", dbVersion)
|
||||
}
|
||||
|
||||
func (db *Instance) updateSchema0to1() {
|
||||
t := db.newReadWriteTransaction()
|
||||
defer t.close()
|
||||
|
||||
dbi := t.NewIterator(util.BytesPrefix([]byte{KeyTypeDevice}), nil)
|
||||
defer dbi.Release()
|
||||
|
||||
symlinkConv := 0
|
||||
changedFolders := make(map[string]struct{})
|
||||
ignAdded := 0
|
||||
meta := newMetadataTracker() // dummy metadata tracker
|
||||
var gk []byte
|
||||
|
||||
for dbi.Next() {
|
||||
folder := db.deviceKeyFolder(dbi.Key())
|
||||
device := db.deviceKeyDevice(dbi.Key())
|
||||
name := db.deviceKeyName(dbi.Key())
|
||||
|
||||
// Remove files with absolute path (see #4799)
|
||||
if strings.HasPrefix(string(name), "/") {
|
||||
if _, ok := changedFolders[string(folder)]; !ok {
|
||||
changedFolders[string(folder)] = struct{}{}
|
||||
}
|
||||
gk = db.globalKeyInto(gk, folder, name)
|
||||
t.removeFromGlobal(gk, folder, device, nil, nil)
|
||||
t.Delete(dbi.Key())
|
||||
t.checkFlush()
|
||||
continue
|
||||
}
|
||||
|
||||
// Change SYMLINK_FILE and SYMLINK_DIRECTORY types to the current SYMLINK
|
||||
// type (previously SYMLINK_UNKNOWN). It does this for all devices, both
|
||||
// local and remote, and does not reset delta indexes. It shouldn't really
|
||||
// matter what the symlink type is, but this cleans it up for a possible
|
||||
// future when SYMLINK_FILE and SYMLINK_DIRECTORY are no longer understood.
|
||||
var f protocol.FileInfo
|
||||
if err := f.Unmarshal(dbi.Value()); err != nil {
|
||||
// probably can't happen
|
||||
continue
|
||||
}
|
||||
if f.Type == protocol.FileInfoTypeDeprecatedSymlinkDirectory || f.Type == protocol.FileInfoTypeDeprecatedSymlinkFile {
|
||||
f.Type = protocol.FileInfoTypeSymlink
|
||||
bs, err := f.Marshal()
|
||||
if err != nil {
|
||||
panic("can't happen: " + err.Error())
|
||||
}
|
||||
t.Put(dbi.Key(), bs)
|
||||
t.checkFlush()
|
||||
symlinkConv++
|
||||
}
|
||||
|
||||
// Add invalid files to global list
|
||||
if f.Invalid {
|
||||
gk = db.globalKeyInto(gk, folder, name)
|
||||
if t.updateGlobal(gk, folder, device, f, meta) {
|
||||
if _, ok := changedFolders[string(folder)]; !ok {
|
||||
changedFolders[string(folder)] = struct{}{}
|
||||
}
|
||||
ignAdded++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for folder := range changedFolders {
|
||||
db.dropFolderMeta([]byte(folder))
|
||||
}
|
||||
}
|
||||
|
||||
// updateSchema1to2 introduces a sequenceKey->deviceKey bucket for local items
|
||||
// to allow iteration in sequence order (simplifies sending indexes).
|
||||
func (db *Instance) updateSchema1to2() {
|
||||
t := db.newReadWriteTransaction()
|
||||
defer t.close()
|
||||
|
||||
var sk []byte
|
||||
var dk []byte
|
||||
for _, folderStr := range db.ListFolders() {
|
||||
folder := []byte(folderStr)
|
||||
db.withHave(folder, protocol.LocalDeviceID[:], nil, true, func(f FileIntf) bool {
|
||||
sk = db.sequenceKeyInto(sk, folder, f.SequenceNo())
|
||||
dk = db.deviceKeyInto(dk, folder, protocol.LocalDeviceID[:], []byte(f.FileName()))
|
||||
t.Put(sk, dk)
|
||||
t.checkFlush()
|
||||
return true
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// updateSchema2to3 introduces a needKey->nil bucket for locally needed files.
|
||||
func (db *Instance) updateSchema2to3() {
|
||||
t := db.newReadWriteTransaction()
|
||||
defer t.close()
|
||||
|
||||
var nk []byte
|
||||
var dk []byte
|
||||
for _, folderStr := range db.ListFolders() {
|
||||
folder := []byte(folderStr)
|
||||
db.withGlobal(folder, nil, true, func(f FileIntf) bool {
|
||||
name := []byte(f.FileName())
|
||||
dk = db.deviceKeyInto(dk, folder, protocol.LocalDeviceID[:], name)
|
||||
var v protocol.Vector
|
||||
haveFile, ok := db.getFileTrunc(dk, true)
|
||||
if ok {
|
||||
v = haveFile.FileVersion()
|
||||
}
|
||||
if !need(f, ok, v) {
|
||||
return true
|
||||
}
|
||||
nk = t.db.needKeyInto(nk, folder, []byte(f.FileName()))
|
||||
t.Put(nk, nil)
|
||||
t.checkFlush()
|
||||
return true
|
||||
})
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user