octoleo/syncthing
1
0
Fork 0
mirror of https://github.com/octoleo/syncthing.git synced 2024-12-22 19:08:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

all: Transactionalize db.FileSet (fixes #5952) (#6239)

Browse Source
This commit is contained in:
Simon Frei 2020-01-21 18:23:08 +01:00 committed by Jakob Borg
parent d62a0cf692
commit 08f0e125ef
23 changed files with 957 additions and 891 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
gui/default/syncthing/core/syncthingController.js
View File

@ -350,6 +350,9 @@ angular.module('syncthing.core')
var debouncedFuncs = {};
function refreshFolder(folder) {
if ($scope.folders[folder].paused) {
return;
}
var key = "refreshFolder" + folder;
if (!debouncedFuncs[key]) {
debouncedFuncs[key] = debounce(function () {
@ -780,10 +783,6 @@ angular.module('syncthing.core')
};
$scope.folderStatus = function (folderCfg) {
if (typeof $scope.model[folderCfg.id] === 'undefined') {
return 'unknown';
}
if (folderCfg.paused) {
return 'paused';
}
@ -791,7 +790,7 @@ angular.module('syncthing.core')
var folderInfo = $scope.model[folderCfg.id];
// after restart syncthing process state may be empty
if (!folderInfo.state) {
if (typeof folderInfo === 'undefined' || !folderInfo.state) {
return 'unknown';
}

25
lib/api/api.go
View File

@ -743,15 +743,18 @@ func (s *service) getDBRemoteNeed(w http.ResponseWriter, r *http.Request) {
page, perpage := getPagingParams(qs)
if files, err := s.model.RemoteNeedFolderFiles(deviceID, folder, page, perpage); err != nil {
snap, err := s.model.DBSnapshot(folder)
if err != nil {
http.Error(w, err.Error(), http.StatusNotFound)
} else {
sendJSON(w, map[string]interface{}{
"files": toJsonFileInfoSlice(files),
"page": page,
"perpage": perpage,
})
return
}
defer snap.Release()
files := snap.RemoteNeedFolderFiles(deviceID, page, perpage)
sendJSON(w, map[string]interface{}{
"files": toJsonFileInfoSlice(files),
"page": page,
"perpage": perpage,
})
}
func (s *service) getDBLocalChanged(w http.ResponseWriter, r *http.Request) {
@ -761,7 +764,13 @@ func (s *service) getDBLocalChanged(w http.ResponseWriter, r *http.Request) {
page, perpage := getPagingParams(qs)
files := s.model.LocalChangedFiles(folder, page, perpage)
snap, err := s.model.DBSnapshot(folder)
if err != nil {
http.Error(w, err.Error(), http.StatusNotFound)
return
}
defer snap.Release()
files := snap.LocalChangedFiles(page, perpage)
sendJSON(w, map[string]interface{}{
"files": toJsonFileInfoSlice(files),

4
lib/api/api_test.go
View File

@ -29,7 +29,6 @@ import (
"github.com/syncthing/syncthing/lib/events"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/locations"
"github.com/syncthing/syncthing/lib/model"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/sync"
"github.com/syncthing/syncthing/lib/tlsutil"
@ -528,8 +527,7 @@ func startHTTP(cfg *mockedConfig) (string, *suture.Supervisor, error) {
// Instantiate the API service
urService := ur.New(cfg, m, connections, false)
summaryService := model.NewFolderSummaryService(cfg, m, protocol.LocalDeviceID, events.NoopLogger)
svc := New(protocol.LocalDeviceID, cfg, assetDir, "syncthing", m, eventSub, diskEventSub, events.NoopLogger, discoverer, connections, urService, summaryService, errorLog, systemLog, cpu, nil, false).(*service)
svc := New(protocol.LocalDeviceID, cfg, assetDir, "syncthing", m, eventSub, diskEventSub, events.NoopLogger, discoverer, connections, urService, &mockedFolderSummaryService{}, errorLog, systemLog, cpu, nil, false).(*service)
defer os.Remove(token)
svc.started = addrChan

48
lib/api/mocked_model_test.go
View File

@ -36,12 +36,8 @@ func (m *mockedModel) NeedFolderFiles(folder string, page, perpage int) ([]db.Fi
return nil, nil, nil
}
func (m *mockedModel) RemoteNeedFolderFiles(device protocol.DeviceID, folder string, page, perpage int) ([]db.FileInfoTruncated, error) {
return nil, nil
}
func (m *mockedModel) NeedSize(folder string) db.Counts {
return db.Counts{}
func (m *mockedModel) FolderProgressBytesCompleted(_ string) int64 {
return 0
}
func (m *mockedModel) ConnectionStats() map[string]interface{} {
@ -112,26 +108,6 @@ func (m *mockedModel) Connection(deviceID protocol.DeviceID) (connections.Connec
return nil, false
}
func (m *mockedModel) GlobalSize(folder string) db.Counts {
return db.Counts{}
}
func (m *mockedModel) LocalSize(folder string) db.Counts {
return db.Counts{}
}
func (m *mockedModel) ReceiveOnlyChangedSize(folder string) db.Counts {
return db.Counts{}
}
func (m *mockedModel) CurrentSequence(folder string) (int64, bool) {
return 0, false
}
func (m *mockedModel) RemoteSequence(folder string) (int64, bool) {
return 0, false
}
func (m *mockedModel) State(folder string) (string, time.Time, error) {
return "", time.Time{}, nil
}
@ -148,10 +124,6 @@ func (m *mockedModel) WatchError(folder string) error {
return nil
}
func (m *mockedModel) LocalChangedFiles(folder string, page, perpage int) []db.FileInfoTruncated {
return nil
}
func (m *mockedModel) Serve() {}
func (m *mockedModel) Stop() {}
@ -188,3 +160,19 @@ func (m *mockedModel) GetHello(protocol.DeviceID) protocol.HelloIntf {
}
func (m *mockedModel) StartDeadlockDetector(timeout time.Duration) {}
func (m *mockedModel) DBSnapshot(_ string) (*db.Snapshot, error) {
return nil, nil
}
type mockedFolderSummaryService struct{}
func (m *mockedFolderSummaryService) Serve() {}
func (m *mockedFolderSummaryService) Stop() {}
func (m *mockedFolderSummaryService) Summary(folder string) (map[string]interface{}, error) {
return map[string]interface{}{"mocked": true}, nil
}
func (m *mockedFolderSummaryService) OnEventRequest() {}

28
lib/db/benchmark_test.go
View File

@ -145,10 +145,12 @@ func BenchmarkNeedHalf(b *testing.B) {
b.ResetTimer()
for i := 0; i < b.N; i++ {
count := 0
benchS.WithNeed(protocol.LocalDeviceID, func(fi db.FileIntf) bool {
snap := benchS.Snapshot()
snap.WithNeed(protocol.LocalDeviceID, func(fi db.FileIntf) bool {
count++
return true
})
snap.Release()
if count != len(secondHalf) {
b.Errorf("wrong length %d != %d", count, len(secondHalf))
}
@ -167,10 +169,12 @@ func BenchmarkNeedHalfRemote(b *testing.B) {
b.ResetTimer()
for i := 0; i < b.N; i++ {
count := 0
fset.WithNeed(remoteDevice0, func(fi db.FileIntf) bool {
snap := fset.Snapshot()
snap.WithNeed(remoteDevice0, func(fi db.FileIntf) bool {
count++
return true
})
snap.Release()
if count != len(secondHalf) {
b.Errorf("wrong length %d != %d", count, len(secondHalf))
}
@ -186,10 +190,12 @@ func BenchmarkHave(b *testing.B) {
b.ResetTimer()
for i := 0; i < b.N; i++ {
count := 0
benchS.WithHave(protocol.LocalDeviceID, func(fi db.FileIntf) bool {
snap := benchS.Snapshot()
snap.WithHave(protocol.LocalDeviceID, func(fi db.FileIntf) bool {
count++
return true
})
snap.Release()
if count != len(firstHalf) {
b.Errorf("wrong length %d != %d", count, len(firstHalf))
}
@ -205,10 +211,12 @@ func BenchmarkGlobal(b *testing.B) {
b.ResetTimer()
for i := 0; i < b.N; i++ {
count := 0
benchS.WithGlobal(func(fi db.FileIntf) bool {
snap := benchS.Snapshot()
snap.WithGlobal(func(fi db.FileIntf) bool {
count++
return true
})
snap.Release()
if count != len(files) {
b.Errorf("wrong length %d != %d", count, len(files))
}
@ -224,10 +232,12 @@ func BenchmarkNeedHalfTruncated(b *testing.B) {
b.ResetTimer()
for i := 0; i < b.N; i++ {
count := 0
benchS.WithNeedTruncated(protocol.LocalDeviceID, func(fi db.FileIntf) bool {
snap := benchS.Snapshot()
snap.WithNeedTruncated(protocol.LocalDeviceID, func(fi db.FileIntf) bool {
count++
return true
})
snap.Release()
if count != len(secondHalf) {
b.Errorf("wrong length %d != %d", count, len(secondHalf))
}
@ -243,10 +253,12 @@ func BenchmarkHaveTruncated(b *testing.B) {
b.ResetTimer()
for i := 0; i < b.N; i++ {
count := 0
benchS.WithHaveTruncated(protocol.LocalDeviceID, func(fi db.FileIntf) bool {
snap := benchS.Snapshot()
snap.WithHaveTruncated(protocol.LocalDeviceID, func(fi db.FileIntf) bool {
count++
return true
})
snap.Release()
if count != len(firstHalf) {
b.Errorf("wrong length %d != %d", count, len(firstHalf))
}
@ -262,10 +274,12 @@ func BenchmarkGlobalTruncated(b *testing.B) {
b.ResetTimer()
for i := 0; i < b.N; i++ {
count := 0
benchS.WithGlobalTruncated(func(fi db.FileIntf) bool {
snap := benchS.Snapshot()
snap.WithGlobalTruncated(func(fi db.FileIntf) bool {
count++
return true
})
snap.Release()
if count != len(files) {
b.Errorf("wrong length %d != %d", count, len(files))
}

28
lib/db/db_test.go
View File

@ -72,7 +72,9 @@ func TestIgnoredFiles(t *testing.T) {
// Local files should have the "ignored" bit in addition to just being
// generally invalid if we want to look at the simulation of that bit.
fi, ok := fs.Get(protocol.LocalDeviceID, "foo")
snap := fs.Snapshot()
defer snap.Release()
fi, ok := snap.Get(protocol.LocalDeviceID, "foo")
if !ok {
t.Fatal("foo should exist")
}
@ -83,7 +85,7 @@ func TestIgnoredFiles(t *testing.T) {
t.Error("foo should be ignored")
}
fi, ok = fs.Get(protocol.LocalDeviceID, "bar")
fi, ok = snap.Get(protocol.LocalDeviceID, "bar")
if !ok {
t.Fatal("bar should exist")
}
@ -97,7 +99,7 @@ func TestIgnoredFiles(t *testing.T) {
// Remote files have the invalid bit as usual, and the IsInvalid() method
// should pick this up too.
fi, ok = fs.Get(protocol.DeviceID{42}, "baz")
fi, ok = snap.Get(protocol.DeviceID{42}, "baz")
if !ok {
t.Fatal("baz should exist")
}
@ -108,7 +110,7 @@ func TestIgnoredFiles(t *testing.T) {
t.Error("baz should be invalid")
}
fi, ok = fs.Get(protocol.DeviceID{42}, "quux")
fi, ok = snap.Get(protocol.DeviceID{42}, "quux")
if !ok {
t.Fatal("quux should exist")
}
@ -167,7 +169,11 @@ func TestUpdate0to3(t *testing.T) {
t.Fatal(err)
}
if _, ok, err := db.getFileDirty(folder, protocol.LocalDeviceID[:], []byte(slashPrefixed)); err != nil {
trans, err := db.newReadOnlyTransaction()
if err != nil {
t.Fatal(err)
}
if _, ok, err := trans.getFile(folder, protocol.LocalDeviceID[:], []byte(slashPrefixed)); err != nil {
t.Fatal(err)
} else if ok {
t.Error("File prefixed by '/' was not removed during transition to schema 1")
@ -186,7 +192,11 @@ func TestUpdate0to3(t *testing.T) {
}
found := false
_ = db.withHaveSequence(folder, 0, func(fi FileIntf) bool {
trans, err = db.newReadOnlyTransaction()
if err != nil {
t.Fatal(err)
}
_ = trans.withHaveSequence(folder, 0, func(fi FileIntf) bool {
f := fi.(protocol.FileInfo)
l.Infoln(f)
if found {
@ -213,7 +223,11 @@ func TestUpdate0to3(t *testing.T) {
haveUpdate0to3[remoteDevice1][0].Name: haveUpdate0to3[remoteDevice1][0],
haveUpdate0to3[remoteDevice0][2].Name: haveUpdate0to3[remoteDevice0][2],
}
_ = db.withNeed(folder, protocol.LocalDeviceID[:], false, func(fi FileIntf) bool {
trans, err = db.newReadOnlyTransaction()
if err != nil {
t.Fatal(err)
}
_ = trans.withNeed(folder, protocol.LocalDeviceID[:], false, func(fi FileIntf) bool {
e, ok := need[fi.FileName()]
if !ok {
t.Error("Got unexpected needed file:", fi.FileName())

399
lib/db/lowlevel.go
View File

@ -194,405 +194,6 @@ func (db *Lowlevel) updateLocalFiles(folder []byte, fs []protocol.FileInfo, meta
return t.commit()
}
func (db *Lowlevel) withHave(folder, device, prefix []byte, truncate bool, fn Iterator) error {
t, err := db.newReadOnlyTransaction()
if err != nil {
return err
}
defer t.close()
if len(prefix) > 0 {
unslashedPrefix := prefix
if bytes.HasSuffix(prefix, []byte{'/'}) {
unslashedPrefix = unslashedPrefix[:len(unslashedPrefix)-1]
} else {
prefix = append(prefix, '/')
}
key, err := db.keyer.GenerateDeviceFileKey(nil, folder, device, unslashedPrefix)
if err != nil {
return err
}
if f, ok, err := t.getFileTrunc(key, true); err != nil {
return err
} else if ok && !fn(f) {
return nil
}
}
key, err := db.keyer.GenerateDeviceFileKey(nil, folder, device, prefix)
if err != nil {
return err
}
dbi, err := t.NewPrefixIterator(key)
if err != nil {
return err
}
defer dbi.Release()
for dbi.Next() {
name := db.keyer.NameFromDeviceFileKey(dbi.Key())
if len(prefix) > 0 && !bytes.HasPrefix(name, prefix) {
return nil
}
f, err := unmarshalTrunc(dbi.Value(), truncate)
if err != nil {
l.Debugln("unmarshal error:", err)
continue
}
if !fn(f) {
return nil
}
}
return dbi.Error()
}
func (db *Lowlevel) withHaveSequence(folder []byte, startSeq int64, fn Iterator) error {
t, err := db.newReadOnlyTransaction()
if err != nil {
return err
}
defer t.close()
first, err := db.keyer.GenerateSequenceKey(nil, folder, startSeq)
if err != nil {
return err
}
last, err := db.keyer.GenerateSequenceKey(nil, folder, maxInt64)
if err != nil {
return err
}
dbi, err := t.NewRangeIterator(first, last)
if err != nil {
return err
}
defer dbi.Release()
for dbi.Next() {
f, ok, err := t.getFileByKey(dbi.Value())
if err != nil {
return err
}
if !ok {
l.Debugln("missing file for sequence number", db.keyer.SequenceFromSequenceKey(dbi.Key()))
continue
}
if shouldDebug() {
if seq := db.keyer.SequenceFromSequenceKey(dbi.Key()); f.Sequence != seq {
l.Warnf("Sequence index corruption (folder %v, file %v): sequence %d != expected %d", string(folder), f.Name, f.Sequence, seq)
panic("sequence index corruption")
}
}
if !fn(f) {
return nil
}
}
return dbi.Error()
}
func (db *Lowlevel) withAllFolderTruncated(folder []byte, fn func(device []byte, f FileInfoTruncated) bool) error {
t, err := db.newReadWriteTransaction()
if err != nil {
return err
}
defer t.close()
key, err := db.keyer.GenerateDeviceFileKey(nil, folder, nil, nil)
if err != nil {
return err
}
dbi, err := t.NewPrefixIterator(key.WithoutNameAndDevice())
if err != nil {
return err
}
defer dbi.Release()
var gk, keyBuf []byte
for dbi.Next() {
device, ok := db.keyer.DeviceFromDeviceFileKey(dbi.Key())
if !ok {
// Not having the device in the index is bad. Clear it.
if err := t.Delete(dbi.Key()); err != nil {
return err
}
continue
}
var f FileInfoTruncated
// The iterator function may keep a reference to the unmarshalled
// struct, which in turn references the buffer it was unmarshalled
// from. dbi.Value() just returns an internal slice that it reuses, so
// we need to copy it.
err := f.Unmarshal(append([]byte{}, dbi.Value()...))
if err != nil {
return err
}
switch f.Name {
case "", ".", "..", "/": // A few obviously invalid filenames
l.Infof("Dropping invalid filename %q from database", f.Name)
name := []byte(f.Name)
gk, err = db.keyer.GenerateGlobalVersionKey(gk, folder, name)
if err != nil {
return err
}
keyBuf, err = t.removeFromGlobal(gk, keyBuf, folder, device, name, nil)
if err != nil {
return err
}
if err := t.Delete(dbi.Key()); err != nil {
return err
}
continue
}
if !fn(device, f) {
return nil
}
}
if err := dbi.Error(); err != nil {
return err
}
return t.commit()
}
func (db *Lowlevel) getFileDirty(folder, device, file []byte) (protocol.FileInfo, bool, error) {
t, err := db.newReadOnlyTransaction()
if err != nil {
return protocol.FileInfo{}, false, err
}
defer t.close()
return t.getFile(folder, device, file)
}
func (db *Lowlevel) getGlobalDirty(folder, file []byte, truncate bool) (FileIntf, bool, error) {
t, err := db.newReadOnlyTransaction()
if err != nil {
return nil, false, err
}
defer t.close()
_, f, ok, err := t.getGlobal(nil, folder, file, truncate)
return f, ok, err
}
func (db *Lowlevel) withGlobal(folder, prefix []byte, truncate bool, fn Iterator) error {
t, err := db.newReadOnlyTransaction()
if err != nil {
return err
}
defer t.close()
if len(prefix) > 0 {
unslashedPrefix := prefix
if bytes.HasSuffix(prefix, []byte{'/'}) {
unslashedPrefix = unslashedPrefix[:len(unslashedPrefix)-1]
} else {
prefix = append(prefix, '/')
}
if _, f, ok, err := t.getGlobal(nil, folder, unslashedPrefix, truncate); err != nil {
return err
} else if ok && !fn(f) {
return nil
}
}
key, err := db.keyer.GenerateGlobalVersionKey(nil, folder, prefix)
if err != nil {
return err
}
dbi, err := t.NewPrefixIterator(key)
if err != nil {
return err
}
defer dbi.Release()
var dk []byte
for dbi.Next() {
name := db.keyer.NameFromGlobalVersionKey(dbi.Key())
if len(prefix) > 0 && !bytes.HasPrefix(name, prefix) {
return nil
}
vl, ok := unmarshalVersionList(dbi.Value())
if !ok {
continue
}
dk, err = db.keyer.GenerateDeviceFileKey(dk, folder, vl.Versions[0].Device, name)
if err != nil {
return err
}
f, ok, err := t.getFileTrunc(dk, truncate)
if err != nil {
return err
}
if !ok {
continue
}
if !fn(f) {
return nil
}
}
if err != nil {
return err
}
return dbi.Error()
}
func (db *Lowlevel) availability(folder, file []byte) ([]protocol.DeviceID, error) {
k, err := db.keyer.GenerateGlobalVersionKey(nil, folder, file)
if err != nil {
return nil, err
}
bs, err := db.Get(k)
if backend.IsNotFound(err) {
return nil, nil
}
if err != nil {
return nil, err
}
vl, ok := unmarshalVersionList(bs)
if !ok {
return nil, nil
}
var devices []protocol.DeviceID
for _, v := range vl.Versions {
if !v.Version.Equal(vl.Versions[0].Version) {
break
}
if v.Invalid {
continue
}
n := protocol.DeviceIDFromBytes(v.Device)
devices = append(devices, n)
}
return devices, nil
}
func (db *Lowlevel) withNeed(folder, device []byte, truncate bool, fn Iterator) error {
if bytes.Equal(device, protocol.LocalDeviceID[:]) {
return db.withNeedLocal(folder, truncate, fn)
}
t, err := db.newReadOnlyTransaction()
if err != nil {
return err
}
defer t.close()
key, err := db.keyer.GenerateGlobalVersionKey(nil, folder, nil)
if err != nil {
return err
}
dbi, err := t.NewPrefixIterator(key.WithoutName())
if err != nil {
return err
}
defer dbi.Release()
var dk []byte
devID := protocol.DeviceIDFromBytes(device)
for dbi.Next() {
vl, ok := unmarshalVersionList(dbi.Value())
if !ok {
continue
}
haveFV, have := vl.Get(device)
// XXX: This marks Concurrent (i.e. conflicting) changes as
// needs. Maybe we should do that, but it needs special
// handling in the puller.
if have && haveFV.Version.GreaterEqual(vl.Versions[0].Version) {
continue
}
name := db.keyer.NameFromGlobalVersionKey(dbi.Key())
needVersion := vl.Versions[0].Version
needDevice := protocol.DeviceIDFromBytes(vl.Versions[0].Device)
for i := range vl.Versions {
if !vl.Versions[i].Version.Equal(needVersion) {
// We haven't found a valid copy of the file with the needed version.
break
}
if vl.Versions[i].Invalid {
// The file is marked invalid, don't use it.
continue
}
dk, err = db.keyer.GenerateDeviceFileKey(dk, folder, vl.Versions[i].Device, name)
if err != nil {
return err
}
gf, ok, err := t.getFileTrunc(dk, truncate)
if err != nil {
return err
}
if !ok {
continue
}
if gf.IsDeleted() && !have {
// We don't need deleted files that we don't have
break
}
l.Debugf("need folder=%q device=%v name=%q have=%v invalid=%v haveV=%v globalV=%v globalDev=%v", folder, devID, name, have, haveFV.Invalid, haveFV.Version, needVersion, needDevice)
if !fn(gf) {
return nil
}
// This file is handled, no need to look further in the version list
break
}
}
return dbi.Error()
}
func (db *Lowlevel) withNeedLocal(folder []byte, truncate bool, fn Iterator) error {
t, err := db.newReadOnlyTransaction()
if err != nil {
return err
}
defer t.close()
key, err := db.keyer.GenerateNeedFileKey(nil, folder, nil)
if err != nil {
return err
}
dbi, err := t.NewPrefixIterator(key.WithoutName())
if err != nil {
return err
}
defer dbi.Release()
var keyBuf []byte
var f FileIntf
var ok bool
for dbi.Next() {
keyBuf, f, ok, err = t.getGlobal(keyBuf, folder, db.keyer.NameFromGlobalVersionKey(dbi.Key()), truncate)
if err != nil {
return err
}
if !ok {
continue
}
if !fn(f) {
return nil
}
}
return dbi.Error()
}
func (db *Lowlevel) dropFolder(folder []byte) error {
t, err := db.newReadWriteTransaction()
if err != nil {

77
lib/db/meta.go
View File

@ -15,12 +15,16 @@ import (
"github.com/syncthing/syncthing/lib/sync"
)
// metadataTracker keeps metadata on a per device, per local flag basis.
type metadataTracker struct {
mut sync.RWMutex
type countsMap struct {
counts CountsSet
indexes map[metaKey]int // device ID + local flags -> index in counts
dirty bool
}
// metadataTracker keeps metadata on a per device, per local flag basis.
type metadataTracker struct {
countsMap
mut sync.RWMutex
dirty bool
}
type metaKey struct {
@ -30,8 +34,10 @@ type metaKey struct {
func newMetadataTracker() *metadataTracker {
return &metadataTracker{
mut: sync.NewRWMutex(),
indexes: make(map[metaKey]int),
mut: sync.NewRWMutex(),
countsMap: countsMap{
indexes: make(map[metaKey]int),
},
}
}
@ -49,7 +55,7 @@ func (m *metadataTracker) Unmarshal(bs []byte) error {
return nil
}
// Unmarshal returns the protobuf representation of the metadataTracker
// Marshal returns the protobuf representation of the metadataTracker
func (m *metadataTracker) Marshal() ([]byte, error) {
return m.counts.Marshal()
}
@ -260,16 +266,11 @@ func (m *metadataTracker) resetCounts(dev protocol.DeviceID) {
m.mut.Unlock()
}
// Counts returns the counts for the given device ID and flag. `flag` should
// be zero or have exactly one bit set.
func (m *metadataTracker) Counts(dev protocol.DeviceID, flag uint32) Counts {
func (m *countsMap) Counts(dev protocol.DeviceID, flag uint32) Counts {
if bits.OnesCount32(flag) > 1 {
panic("incorrect usage: set at most one bit in flag")
}
m.mut.RLock()
defer m.mut.RUnlock()
idx, ok := m.indexes[metaKey{dev, flag}]
if !ok {
return Counts{}
@ -278,6 +279,37 @@ func (m *metadataTracker) Counts(dev protocol.DeviceID, flag uint32) Counts {
return m.counts.Counts[idx]
}
// Counts returns the counts for the given device ID and flag. `flag` should
// be zero or have exactly one bit set.
func (m *metadataTracker) Counts(dev protocol.DeviceID, flag uint32) Counts {
m.mut.RLock()
defer m.mut.RUnlock()
return m.countsMap.Counts(dev, flag)
}
// Snapshot returns a copy of the metadata for reading.
func (m *metadataTracker) Snapshot() *countsMap {
m.mut.RLock()
defer m.mut.RUnlock()
c := &countsMap{
counts: CountsSet{
Counts: make([]Counts, len(m.counts.Counts)),
Created: m.counts.Created,
},
indexes: make(map[metaKey]int, len(m.indexes)),
}
for k, v := range m.indexes {
c.indexes[k] = v
}
for i := range m.counts.Counts {
c.counts.Counts[i] = m.counts.Counts[i]
}
return c
}
// nextLocalSeq allocates a new local sequence number
func (m *metadataTracker) nextLocalSeq() int64 {
m.mut.Lock()
@ -291,26 +323,25 @@ func (m *metadataTracker) nextLocalSeq() int64 {
// devices returns the list of devices tracked, excluding the local device
// (which we don't know the ID of)
func (m *metadataTracker) devices() []protocol.DeviceID {
devs := make(map[protocol.DeviceID]struct{}, len(m.counts.Counts))
m.mut.RLock()
defer m.mut.RUnlock()
return m.countsMap.devices()
}
func (m *countsMap) devices() []protocol.DeviceID {
devs := make([]protocol.DeviceID, 0, len(m.counts.Counts))
for _, dev := range m.counts.Counts {
if dev.Sequence > 0 {
id := protocol.DeviceIDFromBytes(dev.DeviceID)
if id == protocol.GlobalDeviceID || id == protocol.LocalDeviceID {
continue
}
devs[id] = struct{}{}
devs = append(devs, id)
}
}
m.mut.RUnlock()
devList := make([]protocol.DeviceID, 0, len(devs))
for dev := range devs {
devList = append(devList, dev)
}
return devList
return devs
}
func (m *metadataTracker) Created() time.Time {

8
lib/db/schemaupdater.go
View File

@ -228,7 +228,7 @@ func (db *schemaUpdater) updateSchema1to2() error {
for _, folderStr := range db.ListFolders() {
folder := []byte(folderStr)
var putErr error
err := db.withHave(folder, protocol.LocalDeviceID[:], nil, true, func(f FileIntf) bool {
err := t.withHave(folder, protocol.LocalDeviceID[:], nil, true, func(f FileIntf) bool {
sk, putErr = db.keyer.GenerateSequenceKey(sk, folder, f.SequenceNo())
if putErr != nil {
return false
@ -263,7 +263,7 @@ func (db *schemaUpdater) updateSchema2to3() error {
for _, folderStr := range db.ListFolders() {
folder := []byte(folderStr)
var putErr error
err := db.withGlobal(folder, nil, true, func(f FileIntf) bool {
err := t.withGlobal(folder, nil, true, func(f FileIntf) bool {
name := []byte(f.FileName())
dk, putErr = db.keyer.GenerateDeviceFileKey(dk, folder, protocol.LocalDeviceID[:], name)
if putErr != nil {
@ -339,7 +339,7 @@ func (db *schemaUpdater) updateSchema5to6() error {
for _, folderStr := range db.ListFolders() {
folder := []byte(folderStr)
var putErr error
err := db.withHave(folder, protocol.LocalDeviceID[:], nil, false, func(f FileIntf) bool {
err := t.withHave(folder, protocol.LocalDeviceID[:], nil, false, func(f FileIntf) bool {
if !f.IsInvalid() {
return true
}
@ -382,7 +382,7 @@ func (db *schemaUpdater) updateSchema6to7() error {
for _, folderStr := range db.ListFolders() {
folder := []byte(folderStr)
var delErr error
err := db.withNeedLocal(folder, false, func(f FileIntf) bool {
err := t.withNeedLocal(folder, false, func(f FileIntf) bool {
name := []byte(f.FileName())
global := f.(protocol.FileInfo)
gk, delErr = db.keyer.GenerateGlobalVersionKey(gk, folder, name)

178
lib/db/set.go
View File

@ -105,8 +105,12 @@ func (s *FileSet) recalcCounts() error {
return err
}
t, err := s.db.newReadWriteTransaction()
if err != nil {
return err
}
var deviceID protocol.DeviceID
err := s.db.withAllFolderTruncated([]byte(s.folder), func(device []byte, f FileInfoTruncated) bool {
err = t.withAllFolderTruncated([]byte(s.folder), func(device []byte, f FileInfoTruncated) bool {
copy(deviceID[:], device)
s.meta.addFile(deviceID, f)
return true
@ -183,75 +187,97 @@ func (s *FileSet) Update(device protocol.DeviceID, fs []protocol.FileInfo) {
}
}
func (s *FileSet) WithNeed(device protocol.DeviceID, fn Iterator) {
type Snapshot struct {
folder string
t readOnlyTransaction
meta *countsMap
}
func (s *FileSet) Snapshot() *Snapshot {
t, err := s.db.newReadOnlyTransaction()
if err != nil {
panic(err)
}
return &Snapshot{
folder: s.folder,
t: t,
meta: s.meta.Snapshot(),
}
}
func (s *Snapshot) Release() {
s.t.close()
}
func (s *Snapshot) WithNeed(device protocol.DeviceID, fn Iterator) {
l.Debugf("%s WithNeed(%v)", s.folder, device)
if err := s.db.withNeed([]byte(s.folder), device[:], false, nativeFileIterator(fn)); err != nil && !backend.IsClosed(err) {
if err := s.t.withNeed([]byte(s.folder), device[:], false, nativeFileIterator(fn)); err != nil && !backend.IsClosed(err) {
panic(err)
}
}
func (s *FileSet) WithNeedTruncated(device protocol.DeviceID, fn Iterator) {
func (s *Snapshot) WithNeedTruncated(device protocol.DeviceID, fn Iterator) {
l.Debugf("%s WithNeedTruncated(%v)", s.folder, device)
if err := s.db.withNeed([]byte(s.folder), device[:], true, nativeFileIterator(fn)); err != nil && !backend.IsClosed(err) {
if err := s.t.withNeed([]byte(s.folder), device[:], true, nativeFileIterator(fn)); err != nil && !backend.IsClosed(err) {
panic(err)
}
}
func (s *FileSet) WithHave(device protocol.DeviceID, fn Iterator) {
func (s *Snapshot) WithHave(device protocol.DeviceID, fn Iterator) {
l.Debugf("%s WithHave(%v)", s.folder, device)
if err := s.db.withHave([]byte(s.folder), device[:], nil, false, nativeFileIterator(fn)); err != nil && !backend.IsClosed(err) {
if err := s.t.withHave([]byte(s.folder), device[:], nil, false, nativeFileIterator(fn)); err != nil && !backend.IsClosed(err) {
panic(err)
}
}
func (s *FileSet) WithHaveTruncated(device protocol.DeviceID, fn Iterator) {
func (s *Snapshot) WithHaveTruncated(device protocol.DeviceID, fn Iterator) {
l.Debugf("%s WithHaveTruncated(%v)", s.folder, device)
if err := s.db.withHave([]byte(s.folder), device[:], nil, true, nativeFileIterator(fn)); err != nil && !backend.IsClosed(err) {
if err := s.t.withHave([]byte(s.folder), device[:], nil, true, nativeFileIterator(fn)); err != nil && !backend.IsClosed(err) {
panic(err)
}
}
func (s *FileSet) WithHaveSequence(startSeq int64, fn Iterator) {
func (s *Snapshot) WithHaveSequence(startSeq int64, fn Iterator) {
l.Debugf("%s WithHaveSequence(%v)", s.folder, startSeq)
if err := s.db.withHaveSequence([]byte(s.folder), startSeq, nativeFileIterator(fn)); err != nil && !backend.IsClosed(err) {
if err := s.t.withHaveSequence([]byte(s.folder), startSeq, nativeFileIterator(fn)); err != nil && !backend.IsClosed(err) {
panic(err)
}
}
// Except for an item with a path equal to prefix, only children of prefix are iterated.
// E.g. for prefix "dir", "dir/file" is iterated, but "dir.file" is not.
func (s *FileSet) WithPrefixedHaveTruncated(device protocol.DeviceID, prefix string, fn Iterator) {
func (s *Snapshot) WithPrefixedHaveTruncated(device protocol.DeviceID, prefix string, fn Iterator) {
l.Debugf(`%s WithPrefixedHaveTruncated(%v, "%v")`, s.folder, device, prefix)
if err := s.db.withHave([]byte(s.folder), device[:], []byte(osutil.NormalizedFilename(prefix)), true, nativeFileIterator(fn)); err != nil && !backend.IsClosed(err) {
if err := s.t.withHave([]byte(s.folder), device[:], []byte(osutil.NormalizedFilename(prefix)), true, nativeFileIterator(fn)); err != nil && !backend.IsClosed(err) {
panic(err)
}
}
func (s *FileSet) WithGlobal(fn Iterator) {
func (s *Snapshot) WithGlobal(fn Iterator) {
l.Debugf("%s WithGlobal()", s.folder)
if err := s.db.withGlobal([]byte(s.folder), nil, false, nativeFileIterator(fn)); err != nil && !backend.IsClosed(err) {
if err := s.t.withGlobal([]byte(s.folder), nil, false, nativeFileIterator(fn)); err != nil && !backend.IsClosed(err) {
panic(err)
}
}
func (s *FileSet) WithGlobalTruncated(fn Iterator) {
func (s *Snapshot) WithGlobalTruncated(fn Iterator) {
l.Debugf("%s WithGlobalTruncated()", s.folder)
if err := s.db.withGlobal([]byte(s.folder), nil, true, nativeFileIterator(fn)); err != nil && !backend.IsClosed(err) {
if err := s.t.withGlobal([]byte(s.folder), nil, true, nativeFileIterator(fn)); err != nil && !backend.IsClosed(err) {
panic(err)
}
}
// Except for an item with a path equal to prefix, only children of prefix are iterated.
// E.g. for prefix "dir", "dir/file" is iterated, but "dir.file" is not.
func (s *FileSet) WithPrefixedGlobalTruncated(prefix string, fn Iterator) {
func (s *Snapshot) WithPrefixedGlobalTruncated(prefix string, fn Iterator) {
l.Debugf(`%s WithPrefixedGlobalTruncated("%v")`, s.folder, prefix)
if err := s.db.withGlobal([]byte(s.folder), []byte(osutil.NormalizedFilename(prefix)), true, nativeFileIterator(fn)); err != nil && !backend.IsClosed(err) {
if err := s.t.withGlobal([]byte(s.folder), []byte(osutil.NormalizedFilename(prefix)), true, nativeFileIterator(fn)); err != nil && !backend.IsClosed(err) {
panic(err)
}
}
func (s *FileSet) Get(device protocol.DeviceID, file string) (protocol.FileInfo, bool) {
f, ok, err := s.db.getFileDirty([]byte(s.folder), device[:], []byte(osutil.NormalizedFilename(file)))
func (s *Snapshot) Get(device protocol.DeviceID, file string) (protocol.FileInfo, bool) {
f, ok, err := s.t.getFile([]byte(s.folder), device[:], []byte(osutil.NormalizedFilename(file)))
if backend.IsClosed(err) {
return protocol.FileInfo{}, false
} else if err != nil {
@ -261,8 +287,8 @@ func (s *FileSet) Get(device protocol.DeviceID, file string) (protocol.FileInfo,
return f, ok
}
func (s *FileSet) GetGlobal(file string) (protocol.FileInfo, bool) {
fi, ok, err := s.db.getGlobalDirty([]byte(s.folder), []byte(osutil.NormalizedFilename(file)), false)
func (s *Snapshot) GetGlobal(file string) (protocol.FileInfo, bool) {
_, fi, ok, err := s.t.getGlobal(nil, []byte(s.folder), []byte(osutil.NormalizedFilename(file)), false)
if backend.IsClosed(err) {
return protocol.FileInfo{}, false
} else if err != nil {
@ -276,8 +302,8 @@ func (s *FileSet) GetGlobal(file string) (protocol.FileInfo, bool) {
return f, true
}
func (s *FileSet) GetGlobalTruncated(file string) (FileInfoTruncated, bool) {
fi, ok, err := s.db.getGlobalDirty([]byte(s.folder), []byte(osutil.NormalizedFilename(file)), true)
func (s *Snapshot) GetGlobalTruncated(file string) (FileInfoTruncated, bool) {
_, fi, ok, err := s.t.getGlobal(nil, []byte(s.folder), []byte(osutil.NormalizedFilename(file)), true)
if backend.IsClosed(err) {
return FileInfoTruncated{}, false
} else if err != nil {
@ -291,8 +317,8 @@ func (s *FileSet) GetGlobalTruncated(file string) (FileInfoTruncated, bool) {
return f, true
}
func (s *FileSet) Availability(file string) []protocol.DeviceID {
av, err := s.db.availability([]byte(s.folder), []byte(osutil.NormalizedFilename(file)))
func (s *Snapshot) Availability(file string) []protocol.DeviceID {
av, err := s.t.availability([]byte(s.folder), []byte(osutil.NormalizedFilename(file)))
if backend.IsClosed(err) {
return nil
} else if err != nil {
@ -301,26 +327,110 @@ func (s *FileSet) Availability(file string) []protocol.DeviceID {
return av
}
func (s *FileSet) Sequence(device protocol.DeviceID) int64 {
return s.meta.Sequence(device)
func (s *Snapshot) Sequence(device protocol.DeviceID) int64 {
return s.meta.Counts(device, 0).Sequence
}
func (s *FileSet) LocalSize() Counts {
// RemoteSequence returns the change version for the given folder, as
// sent by remote peers. This is guaranteed to increment if the contents of
// the remote or global folder has changed.
func (s *Snapshot) RemoteSequence() int64 {
var ver int64
for _, device := range s.meta.devices() {
ver += s.Sequence(device)
}
return ver
}
func (s *Snapshot) LocalSize() Counts {
local := s.meta.Counts(protocol.LocalDeviceID, 0)
recvOnlyChanged := s.meta.Counts(protocol.LocalDeviceID, protocol.FlagLocalReceiveOnly)
return local.Add(recvOnlyChanged)
return local.Add(s.ReceiveOnlyChangedSize())
}
func (s *FileSet) ReceiveOnlyChangedSize() Counts {
func (s *Snapshot) ReceiveOnlyChangedSize() Counts {
return s.meta.Counts(protocol.LocalDeviceID, protocol.FlagLocalReceiveOnly)
}
func (s *FileSet) GlobalSize() Counts {
func (s *Snapshot) GlobalSize() Counts {
global := s.meta.Counts(protocol.GlobalDeviceID, 0)
recvOnlyChanged := s.meta.Counts(protocol.GlobalDeviceID, protocol.FlagLocalReceiveOnly)
return global.Add(recvOnlyChanged)
}
func (s *Snapshot) NeedSize() Counts {
var result Counts
s.WithNeedTruncated(protocol.LocalDeviceID, func(f FileIntf) bool {
switch {
case f.IsDeleted():
result.Deleted++
case f.IsDirectory():
result.Directories++
case f.IsSymlink():
result.Symlinks++
default:
result.Files++
result.Bytes += f.FileSize()
}
return true
})
return result
}
// LocalChangedFiles returns a paginated list of currently needed files in
// progress, queued, and to be queued on next puller iteration, as well as the
// total number of files currently needed.
func (s *Snapshot) LocalChangedFiles(page, perpage int) []FileInfoTruncated {
if s.ReceiveOnlyChangedSize().TotalItems() == 0 {
return nil
}
files := make([]FileInfoTruncated, 0, perpage)
skip := (page - 1) * perpage
get := perpage
s.WithHaveTruncated(protocol.LocalDeviceID, func(f FileIntf) bool {
if !f.IsReceiveOnlyChanged() {
return true
}
if skip > 0 {
skip--
return true
}
ft := f.(FileInfoTruncated)
files = append(files, ft)
get--
return get > 0
})
return files
}
// RemoteNeedFolderFiles returns paginated list of currently needed files in
// progress, queued, and to be queued on next puller iteration, as well as the
// total number of files currently needed.
func (s *Snapshot) RemoteNeedFolderFiles(device protocol.DeviceID, page, perpage int) []FileInfoTruncated {
files := make([]FileInfoTruncated, 0, perpage)
skip := (page - 1) * perpage
get := perpage
s.WithNeedTruncated(device, func(f FileIntf) bool {
if skip > 0 {
skip--
return true
}
files = append(files, f.(FileInfoTruncated))
get--
return get > 0
})
return files
}
func (s *FileSet) Sequence(device protocol.DeviceID) int64 {
return s.meta.Sequence(device)
}
func (s *FileSet) IndexID(device protocol.DeviceID) protocol.IndexID {
id, err := s.db.getIndexID(device[:], []byte(s.folder))
if backend.IsClosed(err) {

145
lib/db/set_test.go
View File

@ -46,7 +46,9 @@ func genBlocks(n int) []protocol.BlockInfo {
func globalList(s *db.FileSet) []protocol.FileInfo {
var fs []protocol.FileInfo
s.WithGlobal(func(fi db.FileIntf) bool {
snap := s.Snapshot()
defer snap.Release()
snap.WithGlobal(func(fi db.FileIntf) bool {
f := fi.(protocol.FileInfo)
fs = append(fs, f)
return true
@ -55,7 +57,9 @@ func globalList(s *db.FileSet) []protocol.FileInfo {
}
func globalListPrefixed(s *db.FileSet, prefix string) []db.FileInfoTruncated {
var fs []db.FileInfoTruncated
s.WithPrefixedGlobalTruncated(prefix, func(fi db.FileIntf) bool {
snap := s.Snapshot()
defer snap.Release()
snap.WithPrefixedGlobalTruncated(prefix, func(fi db.FileIntf) bool {
f := fi.(db.FileInfoTruncated)
fs = append(fs, f)
return true
@ -65,7 +69,9 @@ func globalListPrefixed(s *db.FileSet, prefix string) []db.FileInfoTruncated {
func haveList(s *db.FileSet, n protocol.DeviceID) []protocol.FileInfo {
var fs []protocol.FileInfo
s.WithHave(n, func(fi db.FileIntf) bool {
snap := s.Snapshot()
defer snap.Release()
snap.WithHave(n, func(fi db.FileIntf) bool {
f := fi.(protocol.FileInfo)
fs = append(fs, f)
return true
@ -75,7 +81,9 @@ func haveList(s *db.FileSet, n protocol.DeviceID) []protocol.FileInfo {
func haveListPrefixed(s *db.FileSet, n protocol.DeviceID, prefix string) []db.FileInfoTruncated {
var fs []db.FileInfoTruncated
s.WithPrefixedHaveTruncated(n, prefix, func(fi db.FileIntf) bool {
snap := s.Snapshot()
defer snap.Release()
snap.WithPrefixedHaveTruncated(n, prefix, func(fi db.FileIntf) bool {
f := fi.(db.FileInfoTruncated)
fs = append(fs, f)
return true
@ -85,7 +93,9 @@ func haveListPrefixed(s *db.FileSet, n protocol.DeviceID, prefix string) []db.Fi
func needList(s *db.FileSet, n protocol.DeviceID) []protocol.FileInfo {
var fs []protocol.FileInfo
s.WithNeed(n, func(fi db.FileIntf) bool {
snap := s.Snapshot()
defer snap.Release()
snap.WithNeed(n, func(fi db.FileIntf) bool {
f := fi.(protocol.FileInfo)
fs = append(fs, f)
return true
@ -206,7 +216,7 @@ func TestGlobalSet(t *testing.T) {
}
globalBytes += f.FileSize()
}
gs := m.GlobalSize()
gs := globalSize(m)
if gs.Files != globalFiles {
t.Errorf("Incorrect GlobalSize files; %d != %d", gs.Files, globalFiles)
}
@ -242,7 +252,7 @@ func TestGlobalSet(t *testing.T) {
}
haveBytes += f.FileSize()
}
ls := m.LocalSize()
ls := localSize(m)
if ls.Files != haveFiles {
t.Errorf("Incorrect LocalSize files; %d != %d", ls.Files, haveFiles)
}
@ -277,7 +287,9 @@ func TestGlobalSet(t *testing.T) {
t.Errorf("Need incorrect;\n A: %v !=\n E: %v", n, expectedRemoteNeed)
}
f, ok := m.Get(protocol.LocalDeviceID, "b")
snap := m.Snapshot()
defer snap.Release()
f, ok := snap.Get(protocol.LocalDeviceID, "b")
if !ok {
t.Error("Unexpectedly not OK")
}
@ -285,7 +297,7 @@ func TestGlobalSet(t *testing.T) {
t.Errorf("Get incorrect;\n A: %v !=\n E: %v", f, localTot[1])
}
f, ok = m.Get(remoteDevice0, "b")
f, ok = snap.Get(remoteDevice0, "b")
if !ok {
t.Error("Unexpectedly not OK")
}
@ -293,7 +305,7 @@ func TestGlobalSet(t *testing.T) {
t.Errorf("Get incorrect;\n A: %v !=\n E: %v", f, remote1[0])
}
f, ok = m.GetGlobal("b")
f, ok = snap.GetGlobal("b")
if !ok {
t.Error("Unexpectedly not OK")
}
@ -301,7 +313,7 @@ func TestGlobalSet(t *testing.T) {
t.Errorf("GetGlobal incorrect;\n A: %v !=\n E: %v", f, remote1[0])
}
f, ok = m.Get(protocol.LocalDeviceID, "zz")
f, ok = snap.Get(protocol.LocalDeviceID, "zz")
if ok {
t.Error("Unexpectedly OK")
}
@ -309,7 +321,7 @@ func TestGlobalSet(t *testing.T) {
t.Errorf("Get incorrect;\n A: %v !=\n E: %v", f, protocol.FileInfo{})
}
f, ok = m.GetGlobal("zz")
f, ok = snap.GetGlobal("zz")
if ok {
t.Error("Unexpectedly OK")
}
@ -318,15 +330,15 @@ func TestGlobalSet(t *testing.T) {
}
av := []protocol.DeviceID{protocol.LocalDeviceID, remoteDevice0}
a := m.Availability("a")
a := snap.Availability("a")
if !(len(a) == 2 && (a[0] == av[0] && a[1] == av[1] || a[0] == av[1] && a[1] == av[0])) {
t.Errorf("Availability incorrect;\n A: %v !=\n E: %v", a, av)
}
a = m.Availability("b")
a = snap.Availability("b")
if len(a) != 1 || a[0] != remoteDevice0 {
t.Errorf("Availability incorrect;\n A: %v !=\n E: %v", a, remoteDevice0)
}
a = m.Availability("d")
a = snap.Availability("d")
if len(a) != 1 || a[0] != protocol.LocalDeviceID {
t.Errorf("Availability incorrect;\n A: %v !=\n E: %v", a, protocol.LocalDeviceID)
}
@ -446,19 +458,22 @@ func TestInvalidAvailability(t *testing.T) {
replace(s, remoteDevice0, remote0Have)
replace(s, remoteDevice1, remote1Have)
if av := s.Availability("both"); len(av) != 2 {
snap := s.Snapshot()
defer snap.Release()
if av := snap.Availability("both"); len(av) != 2 {
t.Error("Incorrect availability for 'both':", av)
}
if av := s.Availability("r0only"); len(av) != 1 || av[0] != remoteDevice0 {
if av := snap.Availability("r0only"); len(av) != 1 || av[0] != remoteDevice0 {
t.Error("Incorrect availability for 'r0only':", av)
}
if av := s.Availability("r1only"); len(av) != 1 || av[0] != remoteDevice1 {
if av := snap.Availability("r1only"); len(av) != 1 || av[0] != remoteDevice1 {
t.Error("Incorrect availability for 'r1only':", av)
}
if av := s.Availability("none"); len(av) != 0 {
if av := snap.Availability("none"); len(av) != 0 {
t.Error("Incorrect availability for 'none':", av)
}
}
@ -826,20 +841,20 @@ func TestIssue4701(t *testing.T) {
s.Update(protocol.LocalDeviceID, localHave)
if c := s.LocalSize(); c.Files != 1 {
if c := localSize(s); c.Files != 1 {
t.Errorf("Expected 1 local file, got %v", c.Files)
}
if c := s.GlobalSize(); c.Files != 1 {
if c := globalSize(s); c.Files != 1 {
t.Errorf("Expected 1 global file, got %v", c.Files)
}
localHave[1].LocalFlags = 0
s.Update(protocol.LocalDeviceID, localHave)
if c := s.LocalSize(); c.Files != 2 {
if c := localSize(s); c.Files != 2 {
t.Errorf("Expected 2 local files, got %v", c.Files)
}
if c := s.GlobalSize(); c.Files != 2 {
if c := globalSize(s); c.Files != 2 {
t.Errorf("Expected 2 global files, got %v", c.Files)
}
@ -847,10 +862,10 @@ func TestIssue4701(t *testing.T) {
localHave[1].LocalFlags = protocol.FlagLocalIgnored
s.Update(protocol.LocalDeviceID, localHave)
if c := s.LocalSize(); c.Files != 0 {
if c := localSize(s); c.Files != 0 {
t.Errorf("Expected 0 local files, got %v", c.Files)
}
if c := s.GlobalSize(); c.Files != 0 {
if c := globalSize(s); c.Files != 0 {
t.Errorf("Expected 0 global files, got %v", c.Files)
}
}
@ -873,7 +888,9 @@ func TestWithHaveSequence(t *testing.T) {
replace(s, protocol.LocalDeviceID, localHave)
i := 2
s.WithHaveSequence(int64(i), func(fi db.FileIntf) bool {
snap := s.Snapshot()
defer snap.Release()
snap.WithHaveSequence(int64(i), func(fi db.FileIntf) bool {
if f := fi.(protocol.FileInfo); !f.IsEquivalent(localHave[i-1], 0) {
t.Fatalf("Got %v\nExpected %v", f, localHave[i-1])
}
@ -922,13 +939,15 @@ loop:
break loop
default:
}
s.WithHaveSequence(prevSeq+1, func(fi db.FileIntf) bool {
snap := s.Snapshot()
snap.WithHaveSequence(prevSeq+1, func(fi db.FileIntf) bool {
if fi.SequenceNo() < prevSeq+1 {
t.Fatal("Skipped ", prevSeq+1, fi.SequenceNo())
}
prevSeq = fi.SequenceNo()
return true
})
snap.Release()
}
}
@ -981,12 +1000,12 @@ func TestMoveGlobalBack(t *testing.T) {
t.Error("Expected no need for remote 0, got", need)
}
ls := s.LocalSize()
ls := localSize(s)
if haveBytes := localHave[0].Size; ls.Bytes != haveBytes {
t.Errorf("Incorrect LocalSize bytes; %d != %d", ls.Bytes, haveBytes)
}
gs := s.GlobalSize()
gs := globalSize(s)
if globalBytes := remote0Have[0].Size; gs.Bytes != globalBytes {
t.Errorf("Incorrect GlobalSize bytes; %d != %d", gs.Bytes, globalBytes)
}
@ -1007,12 +1026,12 @@ func TestMoveGlobalBack(t *testing.T) {
t.Error("Expected no local need, got", need)
}
ls = s.LocalSize()
ls = localSize(s)
if haveBytes := localHave[0].Size; ls.Bytes != haveBytes {
t.Errorf("Incorrect LocalSize bytes; %d != %d", ls.Bytes, haveBytes)
}
gs = s.GlobalSize()
gs = globalSize(s)
if globalBytes := localHave[0].Size; gs.Bytes != globalBytes {
t.Errorf("Incorrect GlobalSize bytes; %d != %d", gs.Bytes, globalBytes)
}
@ -1106,22 +1125,22 @@ func TestReceiveOnlyAccounting(t *testing.T) {
replace(s, protocol.LocalDeviceID, files)
replace(s, remote, files)
if n := s.LocalSize().Files; n != 3 {
if n := localSize(s).Files; n != 3 {
t.Fatal("expected 3 local files initially, not", n)
}
if n := s.LocalSize().Bytes; n != 30 {
if n := localSize(s).Bytes; n != 30 {
t.Fatal("expected 30 local bytes initially, not", n)
}
if n := s.GlobalSize().Files; n != 3 {
if n := globalSize(s).Files; n != 3 {
t.Fatal("expected 3 global files initially, not", n)
}
if n := s.GlobalSize().Bytes; n != 30 {
if n := globalSize(s).Bytes; n != 30 {
t.Fatal("expected 30 global bytes initially, not", n)
}
if n := s.ReceiveOnlyChangedSize().Files; n != 0 {
if n := receiveOnlyChangedSize(s).Files; n != 0 {
t.Fatal("expected 0 receive only changed files initially, not", n)
}
if n := s.ReceiveOnlyChangedSize().Bytes; n != 0 {
if n := receiveOnlyChangedSize(s).Bytes; n != 0 {
t.Fatal("expected 0 receive only changed bytes initially, not", n)
}
@ -1136,22 +1155,22 @@ func TestReceiveOnlyAccounting(t *testing.T) {
// Check that we see the files
if n := s.LocalSize().Files; n != 3 {
if n := localSize(s).Files; n != 3 {
t.Fatal("expected 3 local files after local change, not", n)
}
if n := s.LocalSize().Bytes; n != 120 {
if n := localSize(s).Bytes; n != 120 {
t.Fatal("expected 120 local bytes after local change, not", n)
}
if n := s.GlobalSize().Files; n != 3 {
if n := globalSize(s).Files; n != 3 {
t.Fatal("expected 3 global files after local change, not", n)
}
if n := s.GlobalSize().Bytes; n != 30 {
if n := globalSize(s).Bytes; n != 30 {
t.Fatal("expected 30 global files after local change, not", n)
}
if n := s.ReceiveOnlyChangedSize().Files; n != 1 {
if n := receiveOnlyChangedSize(s).Files; n != 1 {
t.Fatal("expected 1 receive only changed file after local change, not", n)
}
if n := s.ReceiveOnlyChangedSize().Bytes; n != 100 {
if n := receiveOnlyChangedSize(s).Bytes; n != 100 {
t.Fatal("expected 100 receive only changed btyes after local change, not", n)
}
@ -1167,22 +1186,22 @@ func TestReceiveOnlyAccounting(t *testing.T) {
// Check that we see the files, same data as initially
if n := s.LocalSize().Files; n != 3 {
if n := localSize(s).Files; n != 3 {
t.Fatal("expected 3 local files after revert, not", n)
}
if n := s.LocalSize().Bytes; n != 30 {
if n := localSize(s).Bytes; n != 30 {
t.Fatal("expected 30 local bytes after revert, not", n)
}
if n := s.GlobalSize().Files; n != 3 {
if n := globalSize(s).Files; n != 3 {
t.Fatal("expected 3 global files after revert, not", n)
}
if n := s.GlobalSize().Bytes; n != 30 {
if n := globalSize(s).Bytes; n != 30 {
t.Fatal("expected 30 global bytes after revert, not", n)
}
if n := s.ReceiveOnlyChangedSize().Files; n != 0 {
if n := receiveOnlyChangedSize(s).Files; n != 0 {
t.Fatal("expected 0 receive only changed files after revert, not", n)
}
if n := s.ReceiveOnlyChangedSize().Bytes; n != 0 {
if n := receiveOnlyChangedSize(s).Bytes; n != 0 {
t.Fatal("expected 0 receive only changed bytes after revert, not", n)
}
}
@ -1229,7 +1248,7 @@ func TestRemoteInvalidNotAccounted(t *testing.T) {
}
s.Update(remoteDevice0, files)
global := s.GlobalSize()
global := globalSize(s)
if global.Files != 1 {
t.Error("Expected one file in global size, not", global.Files)
}
@ -1386,12 +1405,14 @@ func TestSequenceIndex(t *testing.T) {
// a subset of those files if we manage to run before a complete
// update has happened since our last iteration.
latest = latest[:0]
s.WithHaveSequence(seq+1, func(f db.FileIntf) bool {
snap := s.Snapshot()
snap.WithHaveSequence(seq+1, func(f db.FileIntf) bool {
seen[f.FileName()] = f
latest = append(latest, f)
seq = f.SequenceNo()
return true
})
snap.Release()
// Calculate the spread in sequence number.
var max, min int64
@ -1449,7 +1470,9 @@ func TestIgnoreAfterReceiveOnly(t *testing.T) {
s.Update(protocol.LocalDeviceID, fs)
if f, ok := s.Get(protocol.LocalDeviceID, file); !ok {
snap := s.Snapshot()
defer snap.Release()
if f, ok := snap.Get(protocol.LocalDeviceID, file); !ok {
t.Error("File missing in db")
} else if f.IsReceiveOnlyChanged() {
t.Error("File is still receive-only changed")
@ -1462,3 +1485,21 @@ func replace(fs *db.FileSet, device protocol.DeviceID, files []protocol.FileInfo
fs.Drop(device)
fs.Update(device, files)
}
func localSize(fs *db.FileSet) db.Counts {
snap := fs.Snapshot()
defer snap.Release()
return snap.LocalSize()
}
func globalSize(fs *db.FileSet) db.Counts {
snap := fs.Snapshot()
defer snap.Release()
return snap.GlobalSize()
}
func receiveOnlyChangedSize(fs *db.FileSet) db.Counts {
snap := fs.Snapshot()
defer snap.Release()
return snap.ReceiveOnlyChangedSize()
}

346
lib/db/transactions.go
View File

@ -7,6 +7,8 @@
package db
import (
"bytes"
"github.com/syncthing/syncthing/lib/db/backend"
"github.com/syncthing/syncthing/lib/protocol"
)
@ -94,6 +96,291 @@ func (t readOnlyTransaction) getGlobal(keyBuf, folder, file []byte, truncate boo
return keyBuf, fi, true, nil
}
func (t *readOnlyTransaction) withHave(folder, device, prefix []byte, truncate bool, fn Iterator) error {
if len(prefix) > 0 {
unslashedPrefix := prefix
if bytes.HasSuffix(prefix, []byte{'/'}) {
unslashedPrefix = unslashedPrefix[:len(unslashedPrefix)-1]
} else {
prefix = append(prefix, '/')
}
key, err := t.keyer.GenerateDeviceFileKey(nil, folder, device, unslashedPrefix)
if err != nil {
return err
}
if f, ok, err := t.getFileTrunc(key, true); err != nil {
return err
} else if ok && !fn(f) {
return nil
}
}
key, err := t.keyer.GenerateDeviceFileKey(nil, folder, device, prefix)
if err != nil {
return err
}
dbi, err := t.NewPrefixIterator(key)
if err != nil {
return err
}
defer dbi.Release()
for dbi.Next() {
name := t.keyer.NameFromDeviceFileKey(dbi.Key())
if len(prefix) > 0 && !bytes.HasPrefix(name, prefix) {
return nil
}
f, err := unmarshalTrunc(dbi.Value(), truncate)
if err != nil {
l.Debugln("unmarshal error:", err)
continue
}
if !fn(f) {
return nil
}
}
return dbi.Error()
}
func (t *readOnlyTransaction) withHaveSequence(folder []byte, startSeq int64, fn Iterator) error {
first, err := t.keyer.GenerateSequenceKey(nil, folder, startSeq)
if err != nil {
return err
}
last, err := t.keyer.GenerateSequenceKey(nil, folder, maxInt64)
if err != nil {
return err
}
dbi, err := t.NewRangeIterator(first, last)
if err != nil {
return err
}
defer dbi.Release()
for dbi.Next() {
f, ok, err := t.getFileByKey(dbi.Value())
if err != nil {
return err
}
if !ok {
l.Debugln("missing file for sequence number", t.keyer.SequenceFromSequenceKey(dbi.Key()))
continue
}
if shouldDebug() {
if seq := t.keyer.SequenceFromSequenceKey(dbi.Key()); f.Sequence != seq {
l.Warnf("Sequence index corruption (folder %v, file %v): sequence %d != expected %d", string(folder), f.Name, f.Sequence, seq)
panic("sequence index corruption")
}
}
if !fn(f) {
return nil
}
}
return dbi.Error()
}
func (t *readOnlyTransaction) withGlobal(folder, prefix []byte, truncate bool, fn Iterator) error {
if len(prefix) > 0 {
unslashedPrefix := prefix
if bytes.HasSuffix(prefix, []byte{'/'}) {
unslashedPrefix = unslashedPrefix[:len(unslashedPrefix)-1]
} else {
prefix = append(prefix, '/')
}
if _, f, ok, err := t.getGlobal(nil, folder, unslashedPrefix, truncate); err != nil {
return err
} else if ok && !fn(f) {
return nil
}
}
key, err := t.keyer.GenerateGlobalVersionKey(nil, folder, prefix)
if err != nil {
return err
}
dbi, err := t.NewPrefixIterator(key)
if err != nil {
return err
}
defer dbi.Release()
var dk []byte
for dbi.Next() {
name := t.keyer.NameFromGlobalVersionKey(dbi.Key())
if len(prefix) > 0 && !bytes.HasPrefix(name, prefix) {
return nil
}
vl, ok := unmarshalVersionList(dbi.Value())
if !ok {
continue
}
dk, err = t.keyer.GenerateDeviceFileKey(dk, folder, vl.Versions[0].Device, name)
if err != nil {
return err
}
f, ok, err := t.getFileTrunc(dk, truncate)
if err != nil {
return err
}
if !ok {
continue
}
if !fn(f) {
return nil
}
}
if err != nil {
return err
}
return dbi.Error()
}
func (t *readOnlyTransaction) availability(folder, file []byte) ([]protocol.DeviceID, error) {
k, err := t.keyer.GenerateGlobalVersionKey(nil, folder, file)
if err != nil {
return nil, err
}
bs, err := t.Get(k)
if backend.IsNotFound(err) {
return nil, nil
}
if err != nil {
return nil, err
}
vl, ok := unmarshalVersionList(bs)
if !ok {
return nil, nil
}
var devices []protocol.DeviceID
for _, v := range vl.Versions {
if !v.Version.Equal(vl.Versions[0].Version) {
break
}
if v.Invalid {
continue
}
n := protocol.DeviceIDFromBytes(v.Device)
devices = append(devices, n)
}
return devices, nil
}
func (t *readOnlyTransaction) withNeed(folder, device []byte, truncate bool, fn Iterator) error {
if bytes.Equal(device, protocol.LocalDeviceID[:]) {
return t.withNeedLocal(folder, truncate, fn)
}
key, err := t.keyer.GenerateGlobalVersionKey(nil, folder, nil)
if err != nil {
return err
}
dbi, err := t.NewPrefixIterator(key.WithoutName())
if err != nil {
return err
}
defer dbi.Release()
var dk []byte
devID := protocol.DeviceIDFromBytes(device)
for dbi.Next() {
vl, ok := unmarshalVersionList(dbi.Value())
if !ok {
continue
}
haveFV, have := vl.Get(device)
// XXX: This marks Concurrent (i.e. conflicting) changes as
// needs. Maybe we should do that, but it needs special
// handling in the puller.
if have && haveFV.Version.GreaterEqual(vl.Versions[0].Version) {
continue
}
name := t.keyer.NameFromGlobalVersionKey(dbi.Key())
needVersion := vl.Versions[0].Version
needDevice := protocol.DeviceIDFromBytes(vl.Versions[0].Device)
for i := range vl.Versions {
if !vl.Versions[i].Version.Equal(needVersion) {
// We haven't found a valid copy of the file with the needed version.
break
}
if vl.Versions[i].Invalid {
// The file is marked invalid, don't use it.
continue
}
dk, err = t.keyer.GenerateDeviceFileKey(dk, folder, vl.Versions[i].Device, name)
if err != nil {
return err
}
gf, ok, err := t.getFileTrunc(dk, truncate)
if err != nil {
return err
}
if !ok {
continue
}
if gf.IsDeleted() && !have {
// We don't need deleted files that we don't have
break
}
l.Debugf("need folder=%q device=%v name=%q have=%v invalid=%v haveV=%v globalV=%v globalDev=%v", folder, devID, name, have, haveFV.Invalid, haveFV.Version, needVersion, needDevice)
if !fn(gf) {
return nil
}
// This file is handled, no need to look further in the version list
break
}
}
return dbi.Error()
}
func (t *readOnlyTransaction) withNeedLocal(folder []byte, truncate bool, fn Iterator) error {
key, err := t.keyer.GenerateNeedFileKey(nil, folder, nil)
if err != nil {
return err
}
dbi, err := t.NewPrefixIterator(key.WithoutName())
if err != nil {
return err
}
defer dbi.Release()
var keyBuf []byte
var f FileIntf
var ok bool
for dbi.Next() {
keyBuf, f, ok, err = t.getGlobal(keyBuf, folder, t.keyer.NameFromGlobalVersionKey(dbi.Key()), truncate)
if err != nil {
return err
}
if !ok {
continue
}
if !fn(f) {
return nil
}
}
return dbi.Error()
}
// A readWriteTransaction is a readOnlyTransaction plus a batch for writes.
// The batch will be committed on close() or by checkFlush() if it exceeds the
// batch size.
@ -353,6 +640,65 @@ func (t readWriteTransaction) deleteKeyPrefix(prefix []byte) error {
return dbi.Error()
}
func (t *readWriteTransaction) withAllFolderTruncated(folder []byte, fn func(device []byte, f FileInfoTruncated) bool) error {
key, err := t.keyer.GenerateDeviceFileKey(nil, folder, nil, nil)
if err != nil {
return err
}
dbi, err := t.NewPrefixIterator(key.WithoutNameAndDevice())
if err != nil {
return err
}
defer dbi.Release()
var gk, keyBuf []byte
for dbi.Next() {
device, ok := t.keyer.DeviceFromDeviceFileKey(dbi.Key())
if !ok {
// Not having the device in the index is bad. Clear it.
if err := t.Delete(dbi.Key()); err != nil {
return err
}
continue
}
var f FileInfoTruncated
// The iterator function may keep a reference to the unmarshalled
// struct, which in turn references the buffer it was unmarshalled
// from. dbi.Value() just returns an internal slice that it reuses, so
// we need to copy it.
err := f.Unmarshal(append([]byte{}, dbi.Value()...))
if err != nil {
return err
}
switch f.Name {
case "", ".", "..", "/": // A few obviously invalid filenames
l.Infof("Dropping invalid filename %q from database", f.Name)
name := []byte(f.Name)
gk, err = t.keyer.GenerateGlobalVersionKey(gk, folder, name)
if err != nil {
return err
}
keyBuf, err = t.removeFromGlobal(gk, keyBuf, folder, device, name, nil)
if err != nil {
return err
}
if err := t.Delete(dbi.Key()); err != nil {
return err
}
continue
}
if !fn(device, f) {
return nil
}
}
if err := dbi.Error(); err != nil {
return err
}
return t.commit()
}
type marshaller interface {
Marshal() ([]byte, error)
}

20
lib/model/folder.go
View File

@ -328,11 +328,16 @@ func (f *folder) scanSubdirs(subDirs []string) error {
subDirs[i] = sub
}
snap := f.fset.Snapshot()
// We release explicitly later in this function, however we might exit early
// and it's ok to release twice.
defer snap.Release()
// Clean the list of subitems to ensure that we start at a known
// directory, and don't scan subdirectories of things we've already
// scanned.
subDirs = unifySubs(subDirs, func(file string) bool {
_, ok := f.fset.Get(protocol.LocalDeviceID, file)
_, ok := snap.Get(protocol.LocalDeviceID, file)
return ok
})
@ -344,7 +349,7 @@ func (f *folder) scanSubdirs(subDirs []string) error {
Subs: subDirs,
Matcher: f.ignores,
TempLifetime: time.Duration(f.model.cfg.Options().KeepTemporariesH) * time.Hour,
CurrentFiler: cFiler{f.fset},
CurrentFiler: cFiler{snap},
Filesystem: mtimefs,
IgnorePerms: f.IgnorePerms,
AutoNormalize: f.AutoNormalize,
@ -369,7 +374,7 @@ func (f *folder) scanSubdirs(subDirs []string) error {
oldBatchFn := batchFn // can't reference batchFn directly (recursion)
batchFn = func(fs []protocol.FileInfo) error {
for i := range fs {
switch gf, ok := f.fset.GetGlobal(fs[i].Name); {
switch gf, ok := snap.GetGlobal(fs[i].Name); {
case !ok:
continue
case gf.IsEquivalentOptional(fs[i], f.ModTimeWindow(), false, false, protocol.FlagLocalReceiveOnly):
@ -426,10 +431,15 @@ func (f *folder) scanSubdirs(subDirs []string) error {
// ignored files.
var toIgnore []db.FileInfoTruncated
ignoredParent := ""
snap.Release()
snap = f.fset.Snapshot()
defer snap.Release()
for _, sub := range subDirs {
var iterError error
f.fset.WithPrefixedHaveTruncated(protocol.LocalDeviceID, sub, func(fi db.FileIntf) bool {
snap.WithPrefixedHaveTruncated(protocol.LocalDeviceID, sub, func(fi db.FileIntf) bool {
select {
case <-f.ctx.Done():
return false
@ -891,7 +901,7 @@ func unifySubs(dirs []string, exists func(dir string) bool) []string {
}
type cFiler struct {
*db.FileSet
*db.Snapshot
}
// Implements scanner.CurrentFiler

20
lib/model/folder_recvonly.go
View File

@ -79,7 +79,9 @@ func (f *receiveOnlyFolder) Revert() {
batch := make([]protocol.FileInfo, 0, maxBatchSizeFiles)
batchSizeBytes := 0
f.fset.WithHave(protocol.LocalDeviceID, func(intf db.FileIntf) bool {
snap := f.fset.Snapshot()
defer snap.Release()
snap.WithHave(protocol.LocalDeviceID, func(intf db.FileIntf) bool {
fi := intf.(protocol.FileInfo)
if !fi.IsReceiveOnlyChanged() {
// We're only interested in files that have changed locally in
@ -93,7 +95,7 @@ func (f *receiveOnlyFolder) Revert() {
// We'll delete files directly, directories get queued and
// handled below.
handled, err := delQueue.handle(fi)
handled, err := delQueue.handle(fi, snap)
if err != nil {
l.Infof("Revert: deleting %s: %v\n", fi.Name, err)
return true // continue
@ -138,7 +140,7 @@ func (f *receiveOnlyFolder) Revert() {
batchSizeBytes = 0
// Handle any queued directories
deleted, err := delQueue.flush()
deleted, err := delQueue.flush(snap)
if err != nil {
l.Infoln("Revert:", err)
}
@ -167,15 +169,15 @@ func (f *receiveOnlyFolder) Revert() {
// directories for last.
type deleteQueue struct {
handler interface {
deleteItemOnDisk(item protocol.FileInfo, scanChan chan<- string) error
deleteDirOnDisk(dir string, scanChan chan<- string) error
deleteItemOnDisk(item protocol.FileInfo, snap *db.Snapshot, scanChan chan<- string) error
deleteDirOnDisk(dir string, snap *db.Snapshot, scanChan chan<- string) error
}
ignores *ignore.Matcher
dirs []string
scanChan chan<- string
}
func (q *deleteQueue) handle(fi protocol.FileInfo) (bool, error) {
func (q *deleteQueue) handle(fi protocol.FileInfo, snap *db.Snapshot) (bool, error) {
// Things that are ignored but not marked deletable are not processed.
ign := q.ignores.Match(fi.Name)
if ign.IsIgnored() && !ign.IsDeletable() {
@ -189,11 +191,11 @@ func (q *deleteQueue) handle(fi protocol.FileInfo) (bool, error) {
}
// Kill it.
err := q.handler.deleteItemOnDisk(fi, q.scanChan)
err := q.handler.deleteItemOnDisk(fi, snap, q.scanChan)
return true, err
}
func (q *deleteQueue) flush() ([]string, error) {
func (q *deleteQueue) flush(snap *db.Snapshot) ([]string, error) {
// Process directories from the leaves inward.
sort.Sort(sort.Reverse(sort.StringSlice(q.dirs)))
@ -201,7 +203,7 @@ func (q *deleteQueue) flush() ([]string, error) {
var deleted []string
for _, dir := range q.dirs {
if err := q.handler.deleteDirOnDisk(dir, q.scanChan); err == nil {
if err := q.handler.deleteDirOnDisk(dir, snap, q.scanChan); err == nil {
deleted = append(deleted, dir)
} else if err != nil && firstError == nil {
firstError = err

46
lib/model/folder_recvonly_test.go
View File

@ -48,7 +48,7 @@ func TestRecvOnlyRevertDeletes(t *testing.T) {
m.Index(device1, "ro", knownFiles)
f.updateLocalsFromScanning(knownFiles)
size := m.GlobalSize("ro")
size := globalSize(t, m, "ro")
if size.Files != 1 || size.Directories != 1 {
t.Fatalf("Global: expected 1 file and 1 directory: %+v", size)
}
@ -60,15 +60,15 @@ func TestRecvOnlyRevertDeletes(t *testing.T) {
// We should now have two files and two directories.
size = m.GlobalSize("ro")
size = globalSize(t, m, "ro")
if size.Files != 2 || size.Directories != 2 {
t.Fatalf("Global: expected 2 files and 2 directories: %+v", size)
}
size = m.LocalSize("ro")
size = localSize(t, m, "ro")
if size.Files != 2 || size.Directories != 2 {
t.Fatalf("Local: expected 2 files and 2 directories: %+v", size)
}
size = m.ReceiveOnlyChangedSize("ro")
size = receiveOnlyChangedSize(t, m, "ro")
if size.Files+size.Directories == 0 {
t.Fatalf("ROChanged: expected something: %+v", size)
}
@ -93,11 +93,11 @@ func TestRecvOnlyRevertDeletes(t *testing.T) {
// We should now have one file and directory again.
size = m.GlobalSize("ro")
size = globalSize(t, m, "ro")
if size.Files != 1 || size.Directories != 1 {
t.Fatalf("Global: expected 1 files and 1 directories: %+v", size)
}
size = m.LocalSize("ro")
size = localSize(t, m, "ro")
if size.Files != 1 || size.Directories != 1 {
t.Fatalf("Local: expected 1 files and 1 directories: %+v", size)
}
@ -131,19 +131,19 @@ func TestRecvOnlyRevertNeeds(t *testing.T) {
// Everything should be in sync.
size := m.GlobalSize("ro")
size := globalSize(t, m, "ro")
if size.Files != 1 || size.Directories != 1 {
t.Fatalf("Global: expected 1 file and 1 directory: %+v", size)
}
size = m.LocalSize("ro")
size = localSize(t, m, "ro")
if size.Files != 1 || size.Directories != 1 {
t.Fatalf("Local: expected 1 file and 1 directory: %+v", size)
}
size = m.NeedSize("ro")
size = needSize(t, m, "ro")
if size.Files+size.Directories > 0 {
t.Fatalf("Need: expected nothing: %+v", size)
}
size = m.ReceiveOnlyChangedSize("ro")
size = receiveOnlyChangedSize(t, m, "ro")
if size.Files+size.Directories > 0 {
t.Fatalf("ROChanged: expected nothing: %+v", size)
}
@ -159,20 +159,20 @@ func TestRecvOnlyRevertNeeds(t *testing.T) {
// We now have a newer file than the rest of the cluster. Global state should reflect this.
size = m.GlobalSize("ro")
size = globalSize(t, m, "ro")
const sizeOfDir = 128
if size.Files != 1 || size.Bytes != sizeOfDir+int64(len(oldData)) {
t.Fatalf("Global: expected no change due to the new file: %+v", size)
}
size = m.LocalSize("ro")
size = localSize(t, m, "ro")
if size.Files != 1 || size.Bytes != sizeOfDir+int64(len(newData)) {
t.Fatalf("Local: expected the new file to be reflected: %+v", size)
}
size = m.NeedSize("ro")
size = needSize(t, m, "ro")
if size.Files+size.Directories > 0 {
t.Fatalf("Need: expected nothing: %+v", size)
}
size = m.ReceiveOnlyChangedSize("ro")
size = receiveOnlyChangedSize(t, m, "ro")
if size.Files+size.Directories == 0 {
t.Fatalf("ROChanged: expected something: %+v", size)
}
@ -181,15 +181,15 @@ func TestRecvOnlyRevertNeeds(t *testing.T) {
m.Revert("ro")
size = m.GlobalSize("ro")
size = globalSize(t, m, "ro")
if size.Files != 1 || size.Bytes != sizeOfDir+int64(len(oldData)) {
t.Fatalf("Global: expected the global size to revert: %+v", size)
}
size = m.LocalSize("ro")
size = localSize(t, m, "ro")
if size.Files != 1 || size.Bytes != sizeOfDir+int64(len(newData)) {
t.Fatalf("Local: expected the local size to remain: %+v", size)
}
size = m.NeedSize("ro")
size = needSize(t, m, "ro")
if size.Files != 1 || size.Bytes != int64(len(oldData)) {
t.Fatalf("Local: expected to need the old file data: %+v", size)
}
@ -227,19 +227,19 @@ func TestRecvOnlyUndoChanges(t *testing.T) {
// Everything should be in sync.
size := m.GlobalSize("ro")
size := globalSize(t, m, "ro")
if size.Files != 1 || size.Directories != 1 {
t.Fatalf("Global: expected 1 file and 1 directory: %+v", size)
}
size = m.LocalSize("ro")
size = localSize(t, m, "ro")
if size.Files != 1 || size.Directories != 1 {
t.Fatalf("Local: expected 1 file and 1 directory: %+v", size)
}
size = m.NeedSize("ro")
size = needSize(t, m, "ro")
if size.Files+size.Directories > 0 {
t.Fatalf("Need: expected nothing: %+v", size)
}
size = m.ReceiveOnlyChangedSize("ro")
size = receiveOnlyChangedSize(t, m, "ro")
if size.Files+size.Directories > 0 {
t.Fatalf("ROChanged: expected nothing: %+v", size)
}
@ -253,7 +253,7 @@ func TestRecvOnlyUndoChanges(t *testing.T) {
m.ScanFolder("ro")
size = m.ReceiveOnlyChangedSize("ro")
size = receiveOnlyChangedSize(t, m, "ro")
if size.Files != 2 {
t.Fatalf("Receive only: expected 2 files: %+v", size)
}
@ -266,7 +266,7 @@ func TestRecvOnlyUndoChanges(t *testing.T) {
m.ScanFolder("ro")
size = m.ReceiveOnlyChangedSize("ro")
size = receiveOnlyChangedSize(t, m, "ro")
if size.Files+size.Directories+size.Deleted != 0 {
t.Fatalf("Receive only: expected all zero: %+v", size)
}

12
lib/model/folder_sendonly.go
View File

@ -50,7 +50,9 @@ func (f *sendOnlyFolder) pull() bool {
batch := make([]protocol.FileInfo, 0, maxBatchSizeFiles)
batchSizeBytes := 0
f.fset.WithNeed(protocol.LocalDeviceID, func(intf db.FileIntf) bool {
snap := f.fset.Snapshot()
defer snap.Release()
snap.WithNeed(protocol.LocalDeviceID, func(intf db.FileIntf) bool {
if len(batch) == maxBatchSizeFiles || batchSizeBytes > maxBatchSizeBytes {
f.updateLocalsFromPulling(batch)
batch = batch[:0]
@ -66,7 +68,7 @@ func (f *sendOnlyFolder) pull() bool {
return true
}
curFile, ok := f.fset.Get(protocol.LocalDeviceID, intf.FileName())
curFile, ok := snap.Get(protocol.LocalDeviceID, intf.FileName())
if !ok {
if intf.IsDeleted() {
l.Debugln("Should never get a deleted file as needed when we don't have it")
@ -98,7 +100,9 @@ func (f *sendOnlyFolder) Override() {
f.setState(FolderScanning)
batch := make([]protocol.FileInfo, 0, maxBatchSizeFiles)
batchSizeBytes := 0
f.fset.WithNeed(protocol.LocalDeviceID, func(fi db.FileIntf) bool {
snap := f.fset.Snapshot()
defer snap.Release()
snap.WithNeed(protocol.LocalDeviceID, func(fi db.FileIntf) bool {
need := fi.(protocol.FileInfo)
if len(batch) == maxBatchSizeFiles || batchSizeBytes > maxBatchSizeBytes {
f.updateLocalsFromScanning(batch)
@ -106,7 +110,7 @@ func (f *sendOnlyFolder) Override() {
batchSizeBytes = 0
}
have, ok := f.fset.Get(protocol.LocalDeviceID, need.Name)
have, ok := snap.Get(protocol.LocalDeviceID, need.Name)
// Don't override files that are in a bad state (ignored,
// unsupported, must rescan, ...).
if ok && have.IsInvalid() {

87
lib/model/folder_sendrecv.go
View File

@ -149,10 +149,12 @@ func (f *sendReceiveFolder) pull() bool {
// If there is nothing to do, don't even enter pulling state.
abort := true
f.fset.WithNeed(protocol.LocalDeviceID, func(intf db.FileIntf) bool {
snap := f.fset.Snapshot()
snap.WithNeed(protocol.LocalDeviceID, func(intf db.FileIntf) bool {
abort = false
return false
})
snap.Release()
if abort {
return true
}
@ -234,6 +236,9 @@ func (f *sendReceiveFolder) pullerIteration(scanChan chan<- string) int {
f.pullErrors = make(map[string]string)
f.pullErrorsMut.Unlock()
snap := f.fset.Snapshot()
defer snap.Release()
pullChan := make(chan pullBlockState)
copyChan := make(chan copyBlocksState)
finisherChan := make(chan *sharedPullerState)
@ -272,11 +277,11 @@ func (f *sendReceiveFolder) pullerIteration(scanChan chan<- string) int {
doneWg.Add(1)
// finisherRoutine finishes when finisherChan is closed
go func() {
f.finisherRoutine(finisherChan, dbUpdateChan, scanChan)
f.finisherRoutine(snap, finisherChan, dbUpdateChan, scanChan)
doneWg.Done()
}()
changed, fileDeletions, dirDeletions, err := f.processNeeded(dbUpdateChan, copyChan, scanChan)
changed, fileDeletions, dirDeletions, err := f.processNeeded(snap, dbUpdateChan, copyChan, scanChan)
// Signal copy and puller routines that we are done with the in data for
// this iteration. Wait for them to finish.
@ -291,7 +296,7 @@ func (f *sendReceiveFolder) pullerIteration(scanChan chan<- string) int {
doneWg.Wait()
if err == nil {
f.processDeletions(fileDeletions, dirDeletions, dbUpdateChan, scanChan)
f.processDeletions(fileDeletions, dirDeletions, snap, dbUpdateChan, scanChan)
}
// Wait for db updates and scan scheduling to complete
@ -307,7 +312,7 @@ func (f *sendReceiveFolder) pullerIteration(scanChan chan<- string) int {
return changed
}
func (f *sendReceiveFolder) processNeeded(dbUpdateChan chan<- dbUpdateJob, copyChan chan<- copyBlocksState, scanChan chan<- string) (int, map[string]protocol.FileInfo, []protocol.FileInfo, error) {
func (f *sendReceiveFolder) processNeeded(snap *db.Snapshot, dbUpdateChan chan<- dbUpdateJob, copyChan chan<- copyBlocksState, scanChan chan<- string) (int, map[string]protocol.FileInfo, []protocol.FileInfo, error) {
changed := 0
var dirDeletions []protocol.FileInfo
fileDeletions := map[string]protocol.FileInfo{}
@ -317,7 +322,7 @@ func (f *sendReceiveFolder) processNeeded(dbUpdateChan chan<- dbUpdateJob, copyC
// Regular files to pull goes into the file queue, everything else
// (directories, symlinks and deletes) goes into the "process directly"
// pile.
f.fset.WithNeed(protocol.LocalDeviceID, func(intf db.FileIntf) bool {
snap.WithNeed(protocol.LocalDeviceID, func(intf db.FileIntf) bool {
select {
case <-f.ctx.Done():
return false
@ -359,9 +364,9 @@ func (f *sendReceiveFolder) processNeeded(dbUpdateChan chan<- dbUpdateJob, copyC
// files to delete inside them before we get to that point.
dirDeletions = append(dirDeletions, file)
} else if file.IsSymlink() {
f.deleteFile(file, dbUpdateChan, scanChan)
f.deleteFile(file, snap, dbUpdateChan, scanChan)
} else {
df, ok := f.fset.Get(protocol.LocalDeviceID, file.Name)
df, ok := snap.Get(protocol.LocalDeviceID, file.Name)
// Local file can be already deleted, but with a lower version
// number, hence the deletion coming in again as part of
// WithNeed, furthermore, the file can simply be of the wrong
@ -377,7 +382,7 @@ func (f *sendReceiveFolder) processNeeded(dbUpdateChan chan<- dbUpdateJob, copyC
}
case file.Type == protocol.FileInfoTypeFile:
curFile, hasCurFile := f.fset.Get(protocol.LocalDeviceID, file.Name)
curFile, hasCurFile := snap.Get(protocol.LocalDeviceID, file.Name)
if _, need := blockDiff(curFile.Blocks, file.Blocks); hasCurFile && len(need) == 0 {
// We are supposed to copy the entire file, and then fetch nothing. We
// are only updating metadata, so we don't actually *need* to make the
@ -396,13 +401,13 @@ func (f *sendReceiveFolder) processNeeded(dbUpdateChan chan<- dbUpdateJob, copyC
case file.IsDirectory() && !file.IsSymlink():
l.Debugln(f, "Handling directory", file.Name)
if f.checkParent(file.Name, scanChan) {
f.handleDir(file, dbUpdateChan, scanChan)
f.handleDir(file, snap, dbUpdateChan, scanChan)
}
case file.IsSymlink():
l.Debugln(f, "Handling symlink", file.Name)
if f.checkParent(file.Name, scanChan) {
f.handleSymlink(file, dbUpdateChan, scanChan)
f.handleSymlink(file, snap, dbUpdateChan, scanChan)
}
default:
@ -451,7 +456,7 @@ nextFile:
break
}
fi, ok := f.fset.GetGlobal(fileName)
fi, ok := snap.GetGlobal(fileName)
if !ok {
// File is no longer in the index. Mark it as done and drop it.
f.queue.Done(fileName)
@ -484,7 +489,7 @@ nextFile:
// desired state with the delete bit set is in the deletion
// map.
desired := fileDeletions[candidate.Name]
if err := f.renameFile(candidate, desired, fi, dbUpdateChan, scanChan); err != nil {
if err := f.renameFile(candidate, desired, fi, snap, dbUpdateChan, scanChan); err != nil {
// Failed to rename, try to handle files as separate
// deletions and updates.
break
@ -498,11 +503,11 @@ nextFile:
}
}
devices := f.fset.Availability(fileName)
devices := snap.Availability(fileName)
for _, dev := range devices {
if _, ok := f.model.Connection(dev); ok {
// Handle the file normally, by coping and pulling, etc.
f.handleFile(fi, copyChan, dbUpdateChan)
f.handleFile(fi, snap, copyChan)
continue nextFile
}
}
@ -513,7 +518,7 @@ nextFile:
return changed, fileDeletions, dirDeletions, nil
}
func (f *sendReceiveFolder) processDeletions(fileDeletions map[string]protocol.FileInfo, dirDeletions []protocol.FileInfo, dbUpdateChan chan<- dbUpdateJob, scanChan chan<- string) {
func (f *sendReceiveFolder) processDeletions(fileDeletions map[string]protocol.FileInfo, dirDeletions []protocol.FileInfo, snap *db.Snapshot, dbUpdateChan chan<- dbUpdateJob, scanChan chan<- string) {
for _, file := range fileDeletions {
select {
case <-f.ctx.Done():
@ -521,7 +526,7 @@ func (f *sendReceiveFolder) processDeletions(fileDeletions map[string]protocol.F
default:
}
f.deleteFile(file, dbUpdateChan, scanChan)
f.deleteFile(file, snap, dbUpdateChan, scanChan)
}
// Process in reverse order to delete depth first
@ -534,12 +539,12 @@ func (f *sendReceiveFolder) processDeletions(fileDeletions map[string]protocol.F
dir := dirDeletions[len(dirDeletions)-i-1]
l.Debugln(f, "Deleting dir", dir.Name)
f.deleteDir(dir, dbUpdateChan, scanChan)
f.deleteDir(dir, snap, dbUpdateChan, scanChan)
}
}
// handleDir creates or updates the given directory
func (f *sendReceiveFolder) handleDir(file protocol.FileInfo, dbUpdateChan chan<- dbUpdateJob, scanChan chan<- string) {
func (f *sendReceiveFolder) handleDir(file protocol.FileInfo, snap *db.Snapshot, dbUpdateChan chan<- dbUpdateJob, scanChan chan<- string) {
// Used in the defer closure below, updated by the function body. Take
// care not declare another err.
var err error
@ -567,7 +572,7 @@ func (f *sendReceiveFolder) handleDir(file protocol.FileInfo, dbUpdateChan chan<
}
if shouldDebug() {
curFile, _ := f.fset.Get(protocol.LocalDeviceID, file.Name)
curFile, _ := snap.Get(protocol.LocalDeviceID, file.Name)
l.Debugf("need dir\n\t%v\n\t%v", file, curFile)
}
@ -598,7 +603,7 @@ func (f *sendReceiveFolder) handleDir(file protocol.FileInfo, dbUpdateChan chan<
return f.moveForConflict(name, file.ModifiedBy.String(), scanChan)
}, curFile.Name)
} else {
err = f.deleteItemOnDisk(curFile, scanChan)
err = f.deleteItemOnDisk(curFile, snap, scanChan)
}
if err != nil {
f.newPullError(file.Name, err)
@ -693,7 +698,7 @@ func (f *sendReceiveFolder) checkParent(file string, scanChan chan<- string) boo
}
// handleSymlink creates or updates the given symlink
func (f *sendReceiveFolder) handleSymlink(file protocol.FileInfo, dbUpdateChan chan<- dbUpdateJob, scanChan chan<- string) {
func (f *sendReceiveFolder) handleSymlink(file protocol.FileInfo, snap *db.Snapshot, dbUpdateChan chan<- dbUpdateJob, scanChan chan<- string) {
// Used in the defer closure below, updated by the function body. Take
// care not declare another err.
var err error
@ -716,7 +721,7 @@ func (f *sendReceiveFolder) handleSymlink(file protocol.FileInfo, dbUpdateChan c
}()
if shouldDebug() {
curFile, _ := f.fset.Get(protocol.LocalDeviceID, file.Name)
curFile, _ := snap.Get(protocol.LocalDeviceID, file.Name)
l.Debugf("need symlink\n\t%v\n\t%v", file, curFile)
}
@ -750,7 +755,7 @@ func (f *sendReceiveFolder) handleSymlink(file protocol.FileInfo, dbUpdateChan c
return f.moveForConflict(name, file.ModifiedBy.String(), scanChan)
}, curFile.Name)
} else {
err = f.deleteItemOnDisk(curFile, scanChan)
err = f.deleteItemOnDisk(curFile, snap, scanChan)
}
if err != nil {
f.newPullError(file.Name, errors.Wrap(err, "symlink remove"))
@ -775,7 +780,7 @@ func (f *sendReceiveFolder) handleSymlink(file protocol.FileInfo, dbUpdateChan c
}
// deleteDir attempts to remove a directory that was deleted on a remote
func (f *sendReceiveFolder) deleteDir(file protocol.FileInfo, dbUpdateChan chan<- dbUpdateJob, scanChan chan<- string) {
func (f *sendReceiveFolder) deleteDir(file protocol.FileInfo, snap *db.Snapshot, dbUpdateChan chan<- dbUpdateJob, scanChan chan<- string) {
// Used in the defer closure below, updated by the function body. Take
// care not declare another err.
var err error
@ -797,7 +802,7 @@ func (f *sendReceiveFolder) deleteDir(file protocol.FileInfo, dbUpdateChan chan<
})
}()
if err = f.deleteDirOnDisk(file.Name, scanChan); err != nil {
if err = f.deleteDirOnDisk(file.Name, snap, scanChan); err != nil {
f.newPullError(file.Name, errors.Wrap(err, "delete dir"))
return
}
@ -806,8 +811,8 @@ func (f *sendReceiveFolder) deleteDir(file protocol.FileInfo, dbUpdateChan chan<
}
// deleteFile attempts to delete the given file
func (f *sendReceiveFolder) deleteFile(file protocol.FileInfo, dbUpdateChan chan<- dbUpdateJob, scanChan chan<- string) {
cur, hasCur := f.fset.Get(protocol.LocalDeviceID, file.Name)
func (f *sendReceiveFolder) deleteFile(file protocol.FileInfo, snap *db.Snapshot, dbUpdateChan chan<- dbUpdateJob, scanChan chan<- string) {
cur, hasCur := snap.Get(protocol.LocalDeviceID, file.Name)
f.deleteFileWithCurrent(file, cur, hasCur, dbUpdateChan, scanChan)
}
@ -895,7 +900,7 @@ func (f *sendReceiveFolder) deleteFileWithCurrent(file, cur protocol.FileInfo, h
// renameFile attempts to rename an existing file to a destination
// and set the right attributes on it.
func (f *sendReceiveFolder) renameFile(cur, source, target protocol.FileInfo, dbUpdateChan chan<- dbUpdateJob, scanChan chan<- string) error {
func (f *sendReceiveFolder) renameFile(cur, source, target protocol.FileInfo, snap *db.Snapshot, dbUpdateChan chan<- dbUpdateJob, scanChan chan<- string) error {
// Used in the defer closure below, updated by the function body. Take
// care not declare another err.
var err error
@ -937,7 +942,7 @@ func (f *sendReceiveFolder) renameFile(cur, source, target protocol.FileInfo, db
return err
}
// Check that the target corresponds to what we have in the DB
curTarget, ok := f.fset.Get(protocol.LocalDeviceID, target.Name)
curTarget, ok := snap.Get(protocol.LocalDeviceID, target.Name)
switch stat, serr := f.fs.Lstat(target.Name); {
case serr != nil && fs.IsNotExist(serr):
if !ok || curTarget.IsDeleted() {
@ -994,7 +999,7 @@ func (f *sendReceiveFolder) renameFile(cur, source, target protocol.FileInfo, db
// of the source and the creation of the target temp file. Fix-up the metadata,
// update the local index of the target file and rename from temp to real name.
if err = f.performFinish(target, curTarget, true, tempName, dbUpdateChan, scanChan); err != nil {
if err = f.performFinish(target, curTarget, true, tempName, snap, dbUpdateChan, scanChan); err != nil {
return err
}
@ -1039,8 +1044,8 @@ func (f *sendReceiveFolder) renameFile(cur, source, target protocol.FileInfo, db
// handleFile queues the copies and pulls as necessary for a single new or
// changed file.
func (f *sendReceiveFolder) handleFile(file protocol.FileInfo, copyChan chan<- copyBlocksState, dbUpdateChan chan<- dbUpdateJob) {
curFile, hasCurFile := f.fset.Get(protocol.LocalDeviceID, file.Name)
func (f *sendReceiveFolder) handleFile(file protocol.FileInfo, snap *db.Snapshot, copyChan chan<- copyBlocksState) {
curFile, hasCurFile := snap.Get(protocol.LocalDeviceID, file.Name)
have, _ := blockDiff(curFile.Blocks, file.Blocks)
@ -1493,7 +1498,7 @@ func (f *sendReceiveFolder) pullBlock(state pullBlockState, out chan<- *sharedPu
out <- state.sharedPullerState
}
func (f *sendReceiveFolder) performFinish(file, curFile protocol.FileInfo, hasCurFile bool, tempName string, dbUpdateChan chan<- dbUpdateJob, scanChan chan<- string) error {
func (f *sendReceiveFolder) performFinish(file, curFile protocol.FileInfo, hasCurFile bool, tempName string, snap *db.Snapshot, dbUpdateChan chan<- dbUpdateJob, scanChan chan<- string) error {
// Set the correct permission bits on the new file
if !f.IgnorePerms && !file.NoPermissions {
if err := f.fs.Chmod(tempName, fs.FileMode(file.Permissions&0777)); err != nil {
@ -1528,7 +1533,7 @@ func (f *sendReceiveFolder) performFinish(file, curFile protocol.FileInfo, hasCu
return f.moveForConflict(name, file.ModifiedBy.String(), scanChan)
}, curFile.Name)
} else {
err = f.deleteItemOnDisk(curFile, scanChan)
err = f.deleteItemOnDisk(curFile, snap, scanChan)
}
if err != nil {
return err
@ -1549,7 +1554,7 @@ func (f *sendReceiveFolder) performFinish(file, curFile protocol.FileInfo, hasCu
return nil
}
func (f *sendReceiveFolder) finisherRoutine(in <-chan *sharedPullerState, dbUpdateChan chan<- dbUpdateJob, scanChan chan<- string) {
func (f *sendReceiveFolder) finisherRoutine(snap *db.Snapshot, in <-chan *sharedPullerState, dbUpdateChan chan<- dbUpdateJob, scanChan chan<- string) {
for state := range in {
if closed, err := state.finalClose(); closed {
l.Debugln(f, "closing", state.file.Name)
@ -1557,7 +1562,7 @@ func (f *sendReceiveFolder) finisherRoutine(in <-chan *sharedPullerState, dbUpda
f.queue.Done(state.file.Name)
if err == nil {
err = f.performFinish(state.file, state.curFile, state.hasCurFile, state.tempName, dbUpdateChan, scanChan)
err = f.performFinish(state.file, state.curFile, state.hasCurFile, state.tempName, snap, dbUpdateChan, scanChan)
}
if err != nil {
@ -1809,7 +1814,7 @@ func (f *sendReceiveFolder) Errors() []FileError {
}
// deleteItemOnDisk deletes the file represented by old that is about to be replaced by new.
func (f *sendReceiveFolder) deleteItemOnDisk(item protocol.FileInfo, scanChan chan<- string) (err error) {
func (f *sendReceiveFolder) deleteItemOnDisk(item protocol.FileInfo, snap *db.Snapshot, scanChan chan<- string) (err error) {
defer func() {
err = errors.Wrap(err, contextRemovingOldItem)
}()
@ -1818,7 +1823,7 @@ func (f *sendReceiveFolder) deleteItemOnDisk(item protocol.FileInfo, scanChan ch
case item.IsDirectory():
// Directories aren't archived and need special treatment due
// to potential children.
return f.deleteDirOnDisk(item.Name, scanChan)
return f.deleteDirOnDisk(item.Name, snap, scanChan)
case !item.IsSymlink() && f.versioner != nil:
// If we should use versioning, let the versioner archive the
@ -1834,7 +1839,7 @@ func (f *sendReceiveFolder) deleteItemOnDisk(item protocol.FileInfo, scanChan ch
// deleteDirOnDisk attempts to delete a directory. It checks for files/dirs inside
// the directory and removes them if possible or returns an error if it fails
func (f *sendReceiveFolder) deleteDirOnDisk(dir string, scanChan chan<- string) error {
func (f *sendReceiveFolder) deleteDirOnDisk(dir string, snap *db.Snapshot, scanChan chan<- string) error {
if err := osutil.TraversesSymlink(f.fs, filepath.Dir(dir)); err != nil {
return err
}
@ -1853,7 +1858,7 @@ func (f *sendReceiveFolder) deleteDirOnDisk(dir string, scanChan chan<- string)
toBeDeleted = append(toBeDeleted, fullDirFile)
} else if f.ignores != nil && f.ignores.Match(fullDirFile).IsIgnored() {
hasIgnored = true
} else if cf, ok := f.fset.Get(protocol.LocalDeviceID, fullDirFile); !ok || cf.IsDeleted() || cf.IsInvalid() {
} else if cf, ok := snap.Get(protocol.LocalDeviceID, fullDirFile); !ok || cf.IsDeleted() || cf.IsInvalid() {
// Something appeared in the dir that we either are not aware of
// at all, that we think should be deleted or that is invalid,
// but not currently ignored -> schedule scan. The scanChan

41
lib/model/folder_sendrecv_test.go
View File

@ -148,9 +148,8 @@ func TestHandleFile(t *testing.T) {
defer cleanupSRFolder(f, m)
copyChan := make(chan copyBlocksState, 1)
dbUpdateChan := make(chan dbUpdateJob, 1)
f.handleFile(requiredFile, copyChan, dbUpdateChan)
f.handleFile(requiredFile, f.fset.Snapshot(), copyChan)
// Receive the results
toCopy := <-copyChan
@ -195,9 +194,8 @@ func TestHandleFileWithTemp(t *testing.T) {
}
copyChan := make(chan copyBlocksState, 1)
dbUpdateChan := make(chan dbUpdateJob, 1)
f.handleFile(requiredFile, copyChan, dbUpdateChan)
f.handleFile(requiredFile, f.fset.Snapshot(), copyChan)
// Receive the results
toCopy := <-copyChan
@ -247,13 +245,12 @@ func TestCopierFinder(t *testing.T) {
copyChan := make(chan copyBlocksState)
pullChan := make(chan pullBlockState, 4)
finisherChan := make(chan *sharedPullerState, 1)
dbUpdateChan := make(chan dbUpdateJob, 1)
// Run a single fetcher routine
go f.copierRoutine(copyChan, pullChan, finisherChan)
defer close(copyChan)
f.handleFile(requiredFile, copyChan, dbUpdateChan)
f.handleFile(requiredFile, f.fset.Snapshot(), copyChan)
timeout := time.After(10 * time.Second)
pulls := make([]pullBlockState, 4)
@ -378,7 +375,6 @@ func TestWeakHash(t *testing.T) {
copyChan := make(chan copyBlocksState)
pullChan := make(chan pullBlockState, expectBlocks)
finisherChan := make(chan *sharedPullerState, 1)
dbUpdateChan := make(chan dbUpdateJob, 1)
// Run a single fetcher routine
go fo.copierRoutine(copyChan, pullChan, finisherChan)
@ -386,7 +382,7 @@ func TestWeakHash(t *testing.T) {
// Test 1 - no weak hashing, file gets fully repulled (`expectBlocks` pulls).
fo.WeakHashThresholdPct = 101
fo.handleFile(desiredFile, copyChan, dbUpdateChan)
fo.handleFile(desiredFile, fo.fset.Snapshot(), copyChan)
var pulls []pullBlockState
timeout := time.After(10 * time.Second)
@ -415,7 +411,7 @@ func TestWeakHash(t *testing.T) {
// Test 2 - using weak hash, expectPulls blocks pulled.
fo.WeakHashThresholdPct = -1
fo.handleFile(desiredFile, copyChan, dbUpdateChan)
fo.handleFile(desiredFile, fo.fset.Snapshot(), copyChan)
pulls = pulls[:0]
for len(pulls) < expectPulls {
@ -495,9 +491,10 @@ func TestDeregisterOnFailInCopy(t *testing.T) {
finisherBufferChan := make(chan *sharedPullerState)
finisherChan := make(chan *sharedPullerState)
dbUpdateChan := make(chan dbUpdateJob, 1)
snap := f.fset.Snapshot()
copyChan, copyWg := startCopier(f, pullChan, finisherBufferChan)
go f.finisherRoutine(finisherChan, dbUpdateChan, make(chan string))
go f.finisherRoutine(snap, finisherChan, dbUpdateChan, make(chan string))
defer func() {
close(copyChan)
@ -507,7 +504,7 @@ func TestDeregisterOnFailInCopy(t *testing.T) {
close(finisherChan)
}()
f.handleFile(file, copyChan, dbUpdateChan)
f.handleFile(file, snap, copyChan)
// Receive a block at puller, to indicate that at least a single copier
// loop has been performed.
@ -589,6 +586,7 @@ func TestDeregisterOnFailInPull(t *testing.T) {
finisherBufferChan := make(chan *sharedPullerState)
finisherChan := make(chan *sharedPullerState)
dbUpdateChan := make(chan dbUpdateJob, 1)
snap := f.fset.Snapshot()
copyChan, copyWg := startCopier(f, pullChan, finisherBufferChan)
pullWg := sync.NewWaitGroup()
@ -597,7 +595,7 @@ func TestDeregisterOnFailInPull(t *testing.T) {
f.pullerRoutine(pullChan, finisherBufferChan)
pullWg.Done()
}()
go f.finisherRoutine(finisherChan, dbUpdateChan, make(chan string))
go f.finisherRoutine(snap, finisherChan, dbUpdateChan, make(chan string))
defer func() {
// Unblock copier and puller
go func() {
@ -612,7 +610,7 @@ func TestDeregisterOnFailInPull(t *testing.T) {
close(finisherChan)
}()
f.handleFile(file, copyChan, dbUpdateChan)
f.handleFile(file, snap, copyChan)
// Receive at finisher, we should error out as puller has nowhere to pull
// from.
@ -693,7 +691,7 @@ func TestIssue3164(t *testing.T) {
dbUpdateChan := make(chan dbUpdateJob, 1)
f.deleteDir(file, dbUpdateChan, make(chan string))
f.deleteDir(file, f.fset.Snapshot(), dbUpdateChan, make(chan string))
if _, err := ffs.Stat("issue3164"); !fs.IsNotExist(err) {
t.Fatal(err)
@ -832,7 +830,7 @@ func TestCopyOwner(t *testing.T) {
dbUpdateChan := make(chan dbUpdateJob, 1)
defer close(dbUpdateChan)
f.handleDir(dir, dbUpdateChan, nil)
f.handleDir(dir, f.fset.Snapshot(), dbUpdateChan, nil)
<-dbUpdateChan // empty the channel for later
info, err := f.fs.Lstat("foo/bar")
@ -858,16 +856,17 @@ func TestCopyOwner(t *testing.T) {
// but it's the way data is passed around. When the database update
// comes the finisher is done.
snap := f.fset.Snapshot()
finisherChan := make(chan *sharedPullerState)
copierChan, copyWg := startCopier(f, nil, finisherChan)
go f.finisherRoutine(finisherChan, dbUpdateChan, nil)
go f.finisherRoutine(snap, finisherChan, dbUpdateChan, nil)
defer func() {
close(copierChan)
copyWg.Wait()
close(finisherChan)
}()
f.handleFile(file, copierChan, nil)
f.handleFile(file, snap, copierChan)
<-dbUpdateChan
info, err = f.fs.Lstat("foo/bar/baz")
@ -886,7 +885,7 @@ func TestCopyOwner(t *testing.T) {
SymlinkTarget: "over the rainbow",
}
f.handleSymlink(symlink, dbUpdateChan, nil)
f.handleSymlink(symlink, snap, dbUpdateChan, nil)
<-dbUpdateChan
info, err = f.fs.Lstat("foo/bar/sym")
@ -921,7 +920,7 @@ func TestSRConflictReplaceFileByDir(t *testing.T) {
dbUpdateChan := make(chan dbUpdateJob, 1)
scanChan := make(chan string, 1)
f.handleDir(file, dbUpdateChan, scanChan)
f.handleDir(file, f.fset.Snapshot(), dbUpdateChan, scanChan)
if confls := existingConflicts(name, ffs); len(confls) != 1 {
t.Fatal("Expected one conflict, got", len(confls))
@ -954,7 +953,7 @@ func TestSRConflictReplaceFileByLink(t *testing.T) {
dbUpdateChan := make(chan dbUpdateJob, 1)
scanChan := make(chan string, 1)
f.handleSymlink(file, dbUpdateChan, scanChan)
f.handleSymlink(file, f.fset.Snapshot(), dbUpdateChan, scanChan)
if confls := existingConflicts(name, ffs); len(confls) != 1 {
t.Fatal("Expected one conflict, got", len(confls))
@ -996,7 +995,7 @@ func TestDeleteBehindSymlink(t *testing.T) {
fi.Version = fi.Version.Update(device1.Short())
scanChan := make(chan string, 1)
dbUpdateChan := make(chan dbUpdateJob, 1)
f.deleteFile(fi, dbUpdateChan, scanChan)
f.deleteFile(fi, f.fset.Snapshot(), dbUpdateChan, scanChan)
select {
case f := <-scanChan:
t.Fatalf("Received %v on scanChan", f)

31
lib/model/folder_summary.go
View File

@ -77,6 +77,11 @@ func (c *folderSummaryService) String() string {
func (c *folderSummaryService) Summary(folder string) (map[string]interface{}, error) {
var res = make(map[string]interface{})
snap, err := c.model.DBSnapshot(folder)
if err != nil {
return nil, err
}
errors, err := c.model.FolderErrors(folder)
if err != nil && err != ErrFolderPaused && err != errFolderNotRunning {
// Stats from the db can still be obtained if the folder is just paused/being started
@ -87,19 +92,31 @@ func (c *folderSummaryService) Summary(folder string) (map[string]interface{}, e
res["invalid"] = "" // Deprecated, retains external API for now
global := c.model.GlobalSize(folder)
global := snap.GlobalSize()
res["globalFiles"], res["globalDirectories"], res["globalSymlinks"], res["globalDeleted"], res["globalBytes"], res["globalTotalItems"] = global.Files, global.Directories, global.Symlinks, global.Deleted, global.Bytes, global.TotalItems()
local := c.model.LocalSize(folder)
local := snap.LocalSize()
res["localFiles"], res["localDirectories"], res["localSymlinks"], res["localDeleted"], res["localBytes"], res["localTotalItems"] = local.Files, local.Directories, local.Symlinks, local.Deleted, local.Bytes, local.TotalItems()
need := c.model.NeedSize(folder)
need := snap.NeedSize()
need.Bytes -= c.model.FolderProgressBytesCompleted(folder)
// This may happen if we are in progress of pulling files that were
// deleted globally after the pull started.
if need.Bytes < 0 {
need.Bytes = 0
}
res["needFiles"], res["needDirectories"], res["needSymlinks"], res["needDeletes"], res["needBytes"], res["needTotalItems"] = need.Files, need.Directories, need.Symlinks, need.Deleted, need.Bytes, need.TotalItems()
if c.cfg.Folders()[folder].Type == config.FolderTypeReceiveOnly {
fcfg, ok := c.cfg.Folder(folder)
if ok && fcfg.IgnoreDelete {
res["needDeletes"] = 0
}
if ok && fcfg.Type == config.FolderTypeReceiveOnly {
// Add statistics for things that have changed locally in a receive
// only folder.
ro := c.model.ReceiveOnlyChangedSize(folder)
ro := snap.ReceiveOnlyChangedSize()
res["receiveOnlyChangedFiles"] = ro.Files
res["receiveOnlyChangedDirectories"] = ro.Directories
res["receiveOnlyChangedSymlinks"] = ro.Symlinks
@ -115,8 +132,8 @@ func (c *folderSummaryService) Summary(folder string) (map[string]interface{}, e
res["error"] = err.Error()
}
ourSeq, _ := c.model.CurrentSequence(folder)
remoteSeq, _ := c.model.RemoteSequence(folder)
ourSeq := snap.Sequence(protocol.LocalDeviceID)
remoteSeq := snap.Sequence(protocol.GlobalDeviceID)
res["version"] = ourSeq + remoteSeq // legacy
res["sequence"] = ourSeq + remoteSeq // new name

237
lib/model/model.go
View File

@ -90,21 +90,14 @@ type Model interface {
GetFolderVersions(folder string) (map[string][]versioner.FileVersion, error)
RestoreFolderVersions(folder string, versions map[string]time.Time) (map[string]string, error)
LocalChangedFiles(folder string, page, perpage int) []db.FileInfoTruncated
DBSnapshot(folder string) (*db.Snapshot, error)
NeedFolderFiles(folder string, page, perpage int) ([]db.FileInfoTruncated, []db.FileInfoTruncated, []db.FileInfoTruncated)
RemoteNeedFolderFiles(device protocol.DeviceID, folder string, page, perpage int) ([]db.FileInfoTruncated, error)
FolderProgressBytesCompleted(folder string) int64
CurrentFolderFile(folder string, file string) (protocol.FileInfo, bool)
CurrentGlobalFile(folder string, file string) (protocol.FileInfo, bool)
Availability(folder string, file protocol.FileInfo, block protocol.BlockInfo) []Availability
GlobalSize(folder string) db.Counts
LocalSize(folder string) db.Counts
NeedSize(folder string) db.Counts
ReceiveOnlyChangedSize(folder string) db.Counts
CurrentSequence(folder string) (int64, bool)
RemoteSequence(folder string) (int64, bool)
Completion(device protocol.DeviceID, folder string) FolderCompletion
ConnectionStats() map[string]interface{}
DeviceStatistics() (map[string]stats.DeviceStatistics, error)
@ -764,7 +757,10 @@ func (m *model) Completion(device protocol.DeviceID, folder string) FolderComple
return FolderCompletion{} // Folder doesn't exist, so we hardly have any of it
}
tot := rf.GlobalSize().Bytes
snap := rf.Snapshot()
defer snap.Release()
tot := snap.GlobalSize().Bytes
if tot == 0 {
// Folder is empty, so we have all of it
return FolderCompletion{
@ -777,7 +773,7 @@ func (m *model) Completion(device protocol.DeviceID, folder string) FolderComple
m.pmut.RUnlock()
var need, items, fileNeed, downloaded, deletes int64
rf.WithNeedTruncated(device, func(f db.FileIntf) bool {
snap.WithNeedTruncated(device, func(f db.FileIntf) bool {
ft := f.(db.FileInfoTruncated)
// If the file is deleted, we account it only in the deleted column.
@ -822,84 +818,19 @@ func (m *model) Completion(device protocol.DeviceID, folder string) FolderComple
}
}
func addSizeOfFile(s *db.Counts, f db.FileIntf) {
switch {
case f.IsDeleted():
s.Deleted++
case f.IsDirectory():
s.Directories++
case f.IsSymlink():
s.Symlinks++
default:
s.Files++
}
s.Bytes += f.FileSize()
}
// GlobalSize returns the number of files, deleted files and total bytes for all
// files in the global model.
func (m *model) GlobalSize(folder string) db.Counts {
// DBSnapshot returns a snapshot of the database content relevant to the given folder.
func (m *model) DBSnapshot(folder string) (*db.Snapshot, error) {
m.fmut.RLock()
rf, ok := m.folderFiles[folder]
m.fmut.RUnlock()
if ok {
return rf.GlobalSize()
if !ok {
return nil, errFolderMissing
}
return db.Counts{}
return rf.Snapshot(), nil
}
// LocalSize returns the number of files, deleted files and total bytes for all
// files in the local folder.
func (m *model) LocalSize(folder string) db.Counts {
m.fmut.RLock()
rf, ok := m.folderFiles[folder]
m.fmut.RUnlock()
if ok {
return rf.LocalSize()
}
return db.Counts{}
}
// ReceiveOnlyChangedSize returns the number of files, deleted files and
// total bytes for all files that have changed locally in a receieve only
// folder.
func (m *model) ReceiveOnlyChangedSize(folder string) db.Counts {
m.fmut.RLock()
rf, ok := m.folderFiles[folder]
m.fmut.RUnlock()
if ok {
return rf.ReceiveOnlyChangedSize()
}
return db.Counts{}
}
// NeedSize returns the number of currently needed files and their total size
// minus the amount that has already been downloaded.
func (m *model) NeedSize(folder string) db.Counts {
m.fmut.RLock()
rf, ok := m.folderFiles[folder]
cfg := m.folderCfgs[folder]
m.fmut.RUnlock()
var result db.Counts
if ok {
rf.WithNeedTruncated(protocol.LocalDeviceID, func(f db.FileIntf) bool {
if cfg.IgnoreDelete && f.IsDeleted() {
return true
}
addSizeOfFile(&result, f)
return true
})
}
result.Bytes -= m.progressEmitter.BytesCompleted(folder)
// This may happen if we are in progress of pulling files that were
// deleted globally after the pull started.
if result.Bytes < 0 {
result.Bytes = 0
}
l.Debugf("%v NeedSize(%q): %v", m, folder, result)
return result
func (m *model) FolderProgressBytesCompleted(folder string) int64 {
return m.progressEmitter.BytesCompleted(folder)
}
// NeedFolderFiles returns paginated list of currently needed files in
@ -915,6 +846,8 @@ func (m *model) NeedFolderFiles(folder string, page, perpage int) ([]db.FileInfo
return nil, nil, nil
}
snap := rf.Snapshot()
defer snap.Release()
var progress, queued, rest []db.FileInfoTruncated
var seen map[string]struct{}
@ -929,14 +862,14 @@ func (m *model) NeedFolderFiles(folder string, page, perpage int) ([]db.FileInfo
seen = make(map[string]struct{}, len(progressNames)+len(queuedNames))
for i, name := range progressNames {
if f, ok := rf.GetGlobalTruncated(name); ok {
if f, ok := snap.GetGlobalTruncated(name); ok {
progress[i] = f
seen[name] = struct{}{}
}
}
for i, name := range queuedNames {
if f, ok := rf.GetGlobalTruncated(name); ok {
if f, ok := snap.GetGlobalTruncated(name); ok {
queued[i] = f
seen[name] = struct{}{}
}
@ -950,7 +883,7 @@ func (m *model) NeedFolderFiles(folder string, page, perpage int) ([]db.FileInfo
}
rest = make([]db.FileInfoTruncated, 0, perpage)
rf.WithNeedTruncated(protocol.LocalDeviceID, func(f db.FileIntf) bool {
snap.WithNeedTruncated(protocol.LocalDeviceID, func(f db.FileIntf) bool {
if cfg.IgnoreDelete && f.IsDeleted() {
return true
}
@ -970,77 +903,6 @@ func (m *model) NeedFolderFiles(folder string, page, perpage int) ([]db.FileInfo
return progress, queued, rest
}
// LocalChangedFiles returns a paginated list of currently needed files in
// progress, queued, and to be queued on next puller iteration, as well as the
// total number of files currently needed.
func (m *model) LocalChangedFiles(folder string, page, perpage int) []db.FileInfoTruncated {
m.fmut.RLock()
rf, ok := m.folderFiles[folder]
fcfg := m.folderCfgs[folder]
m.fmut.RUnlock()
if !ok {
return nil
}
if fcfg.Type != config.FolderTypeReceiveOnly {
return nil
}
if rf.ReceiveOnlyChangedSize().TotalItems() == 0 {
return nil
}
files := make([]db.FileInfoTruncated, 0, perpage)
skip := (page - 1) * perpage
get := perpage
rf.WithHaveTruncated(protocol.LocalDeviceID, func(f db.FileIntf) bool {
if !f.IsReceiveOnlyChanged() {
return true
}
if skip > 0 {
skip--
return true
}
ft := f.(db.FileInfoTruncated)
files = append(files, ft)
get--
return get > 0
})
return files
}
// RemoteNeedFolderFiles returns paginated list of currently needed files in
// progress, queued, and to be queued on next puller iteration, as well as the
// total number of files currently needed.
func (m *model) RemoteNeedFolderFiles(device protocol.DeviceID, folder string, page, perpage int) ([]db.FileInfoTruncated, error) {
m.fmut.RLock()
m.pmut.RLock()
err := m.checkDeviceFolderConnectedLocked(device, folder)
rf := m.folderFiles[folder]
m.pmut.RUnlock()
m.fmut.RUnlock()
if err != nil {
return nil, err
}
files := make([]db.FileInfoTruncated, 0, perpage)
skip := (page - 1) * perpage
get := perpage
rf.WithNeedTruncated(device, func(f db.FileIntf) bool {
if skip > 0 {
skip--
return true
}
files = append(files, f.(db.FileInfoTruncated))
get--
return get > 0
})
return files, nil
}
// Index is called when a new device is connected and we receive their full index.
// Implements the protocol.Model interface.
func (m *model) Index(deviceID protocol.DeviceID, folder string, fs []protocol.FileInfo) error {
@ -1752,7 +1614,9 @@ func (m *model) CurrentFolderFile(folder string, file string) (protocol.FileInfo
if !ok {
return protocol.FileInfo{}, false
}
return fs.Get(protocol.LocalDeviceID, file)
snap := fs.Snapshot()
defer snap.Release()
return snap.Get(protocol.LocalDeviceID, file)
}
func (m *model) CurrentGlobalFile(folder string, file string) (protocol.FileInfo, bool) {
@ -1762,7 +1626,9 @@ func (m *model) CurrentGlobalFile(folder string, file string) (protocol.FileInfo
if !ok {
return protocol.FileInfo{}, false
}
return fs.GetGlobal(file)
snap := fs.Snapshot()
defer snap.Release()
return snap.GetGlobal(file)
}
// Connection returns the current connection for device, and a boolean whether a connection was found.
@ -2075,7 +1941,9 @@ func (s *indexSender) sendIndexTo(ctx context.Context) error {
var err error
var f protocol.FileInfo
s.fset.WithHaveSequence(s.prevSequence+1, func(fi db.FileIntf) bool {
snap := s.fset.Snapshot()
defer snap.Release()
snap.WithHaveSequence(s.prevSequence+1, func(fi db.FileIntf) bool {
if err = batch.flushIfFull(); err != nil {
return false
}
@ -2347,45 +2215,6 @@ func (m *model) Revert(folder string) {
runner.Revert()
}
// CurrentSequence returns the change version for the given folder.
// This is guaranteed to increment if the contents of the local folder has
// changed.
func (m *model) CurrentSequence(folder string) (int64, bool) {
m.fmut.RLock()
fs, ok := m.folderFiles[folder]
m.fmut.RUnlock()
if !ok {
// The folder might not exist, since this can be called with a user
// specified folder name from the REST interface.
return 0, false
}
return fs.Sequence(protocol.LocalDeviceID), true
}
// RemoteSequence returns the change version for the given folder, as
// sent by remote peers. This is guaranteed to increment if the contents of
// the remote or global folder has changed.
func (m *model) RemoteSequence(folder string) (int64, bool) {
m.fmut.RLock()
fs, ok := m.folderFiles[folder]
cfg := m.folderCfgs[folder]
m.fmut.RUnlock()
if !ok {
// The folder might not exist, since this can be called with a user
// specified folder name from the REST interface.
return 0, false
}
var ver int64
for _, device := range cfg.Devices {
ver += fs.Sequence(device.DeviceID)
}
return ver, true
}
func (m *model) GlobalDirectoryTree(folder, prefix string, levels int, dirsonly bool) map[string]interface{} {
m.fmut.RLock()
files, ok := m.folderFiles[folder]
@ -2402,7 +2231,9 @@ func (m *model) GlobalDirectoryTree(folder, prefix string, levels int, dirsonly
prefix = prefix + sep
}
files.WithPrefixedGlobalTruncated(prefix, func(fi db.FileIntf) bool {
snap := files.Snapshot()
defer snap.Release()
snap.WithPrefixedGlobalTruncated(prefix, func(fi db.FileIntf) bool {
f := fi.(db.FileInfoTruncated)
// Don't include the prefix itself.
@ -2514,8 +2345,10 @@ func (m *model) Availability(folder string, file protocol.FileInfo, block protoc
}
var availabilities []Availability
snap := fs.Snapshot()
defer snap.Release()
next:
for _, device := range fs.Availability(file.Name) {
for _, device := range snap.Availability(file.Name) {
for _, pausedFolder := range m.remotePausedFolders[device] {
if pausedFolder == folder {
continue next

14
lib/model/model_test.go
View File

@ -2146,8 +2146,8 @@ func TestIssue3028(t *testing.T) {
// Get a count of how many files are there now
locorigfiles := m.LocalSize("default").Files
globorigfiles := m.GlobalSize("default").Files
locorigfiles := localSize(t, m, "default").Files
globorigfiles := globalSize(t, m, "default").Files
// Delete and rescan specifically these two
@ -2158,8 +2158,8 @@ func TestIssue3028(t *testing.T) {
// Verify that the number of files decreased by two and the number of
// deleted files increases by two
loc := m.LocalSize("default")
glob := m.GlobalSize("default")
loc := localSize(t, m, "default")
glob := globalSize(t, m, "default")
if loc.Files != locorigfiles-2 {
t.Errorf("Incorrect local accounting; got %d current files, expected %d", loc.Files, locorigfiles-2)
}
@ -2439,10 +2439,12 @@ func TestIssue3496(t *testing.T) {
fs := m.folderFiles["default"]
m.fmut.RUnlock()
var localFiles []protocol.FileInfo
fs.WithHave(protocol.LocalDeviceID, func(i db.FileIntf) bool {
snap := fs.Snapshot()
snap.WithHave(protocol.LocalDeviceID, func(i db.FileIntf) bool {
localFiles = append(localFiles, i.(protocol.FileInfo))
return true
})
snap.Release()
// Mark all files as deleted and fake it as update from device1
@ -2482,7 +2484,7 @@ func TestIssue3496(t *testing.T) {
t.Log(comp)
// Check that NeedSize does the correct thing
need := m.NeedSize("default")
need := needSize(t, m, "default")
if need.Files != 1 || need.Bytes != 1234 {
// The one we added synthetically above
t.Errorf("Incorrect need size; %d, %d != 1, 1234", need.Files, need.Bytes)

38
lib/model/testutils_test.go
View File

@ -9,6 +9,7 @@ package model
import (
"io/ioutil"
"os"
"testing"
"time"
"github.com/syncthing/syncthing/lib/config"
@ -171,3 +172,40 @@ func (c *alwaysChanged) Seen(fs fs.Filesystem, name string) bool {
func (c *alwaysChanged) Changed() bool {
return true
}
func localSize(t *testing.T, m Model, folder string) db.Counts {
t.Helper()
snap := dbSnapshot(t, m, folder)
defer snap.Release()
return snap.LocalSize()
}
func globalSize(t *testing.T, m Model, folder string) db.Counts {
t.Helper()
snap := dbSnapshot(t, m, folder)
defer snap.Release()
return snap.GlobalSize()
}
func receiveOnlyChangedSize(t *testing.T, m Model, folder string) db.Counts {
t.Helper()
snap := dbSnapshot(t, m, folder)
defer snap.Release()
return snap.ReceiveOnlyChangedSize()
}
func needSize(t *testing.T, m Model, folder string) db.Counts {
t.Helper()
snap := dbSnapshot(t, m, folder)
defer snap.Release()
return snap.NeedSize()
}
func dbSnapshot(t *testing.T, m Model, folder string) *db.Snapshot {
t.Helper()
snap, err := m.DBSnapshot(folder)
if err != nil {
t.Fatal(err)
}
return snap
}

7
lib/ur/usage_report.go
View File

@ -88,7 +88,12 @@ func (s *Service) reportData(urVersion int, preview bool) map[string]interface{}
var totFiles, maxFiles int
var totBytes, maxBytes int64
for folderID := range s.cfg.Folders() {
global := s.model.GlobalSize(folderID)
snap, err := s.model.DBSnapshot(folderID)
if err != nil {
continue
}
global := snap.GlobalSize()
snap.Release()
totFiles += int(global.Files)
totBytes += global.Bytes
if int(global.Files) > maxFiles {