diff --git a/gui/default/syncthing/core/syncthingController.js b/gui/default/syncthing/core/syncthingController.js index eb6b9c78c..c1ed4332d 100755 --- a/gui/default/syncthing/core/syncthingController.js +++ b/gui/default/syncthing/core/syncthingController.js @@ -350,6 +350,9 @@ angular.module('syncthing.core') var debouncedFuncs = {}; function refreshFolder(folder) { + if ($scope.folders[folder].paused) { + return; + } var key = "refreshFolder" + folder; if (!debouncedFuncs[key]) { debouncedFuncs[key] = debounce(function () { @@ -780,10 +783,6 @@ angular.module('syncthing.core') }; $scope.folderStatus = function (folderCfg) { - if (typeof $scope.model[folderCfg.id] === 'undefined') { - return 'unknown'; - } - if (folderCfg.paused) { return 'paused'; } @@ -791,7 +790,7 @@ angular.module('syncthing.core') var folderInfo = $scope.model[folderCfg.id]; // after restart syncthing process state may be empty - if (!folderInfo.state) { + if (typeof folderInfo === 'undefined' || !folderInfo.state) { return 'unknown'; } diff --git a/lib/api/api.go b/lib/api/api.go index fc5369f4c..e8d07f431 100644 --- a/lib/api/api.go +++ b/lib/api/api.go @@ -743,15 +743,18 @@ func (s *service) getDBRemoteNeed(w http.ResponseWriter, r *http.Request) { page, perpage := getPagingParams(qs) - if files, err := s.model.RemoteNeedFolderFiles(deviceID, folder, page, perpage); err != nil { + snap, err := s.model.DBSnapshot(folder) + if err != nil { http.Error(w, err.Error(), http.StatusNotFound) - } else { - sendJSON(w, map[string]interface{}{ - "files": toJsonFileInfoSlice(files), - "page": page, - "perpage": perpage, - }) + return } + defer snap.Release() + files := snap.RemoteNeedFolderFiles(deviceID, page, perpage) + sendJSON(w, map[string]interface{}{ + "files": toJsonFileInfoSlice(files), + "page": page, + "perpage": perpage, + }) } func (s *service) getDBLocalChanged(w http.ResponseWriter, r *http.Request) { @@ -761,7 +764,13 @@ func (s *service) getDBLocalChanged(w http.ResponseWriter, r *http.Request) { page, perpage := getPagingParams(qs) - files := s.model.LocalChangedFiles(folder, page, perpage) + snap, err := s.model.DBSnapshot(folder) + if err != nil { + http.Error(w, err.Error(), http.StatusNotFound) + return + } + defer snap.Release() + files := snap.LocalChangedFiles(page, perpage) sendJSON(w, map[string]interface{}{ "files": toJsonFileInfoSlice(files), diff --git a/lib/api/api_test.go b/lib/api/api_test.go index 17cc6e151..e7d9b7bf7 100644 --- a/lib/api/api_test.go +++ b/lib/api/api_test.go @@ -29,7 +29,6 @@ import ( "github.com/syncthing/syncthing/lib/events" "github.com/syncthing/syncthing/lib/fs" "github.com/syncthing/syncthing/lib/locations" - "github.com/syncthing/syncthing/lib/model" "github.com/syncthing/syncthing/lib/protocol" "github.com/syncthing/syncthing/lib/sync" "github.com/syncthing/syncthing/lib/tlsutil" @@ -528,8 +527,7 @@ func startHTTP(cfg *mockedConfig) (string, *suture.Supervisor, error) { // Instantiate the API service urService := ur.New(cfg, m, connections, false) - summaryService := model.NewFolderSummaryService(cfg, m, protocol.LocalDeviceID, events.NoopLogger) - svc := New(protocol.LocalDeviceID, cfg, assetDir, "syncthing", m, eventSub, diskEventSub, events.NoopLogger, discoverer, connections, urService, summaryService, errorLog, systemLog, cpu, nil, false).(*service) + svc := New(protocol.LocalDeviceID, cfg, assetDir, "syncthing", m, eventSub, diskEventSub, events.NoopLogger, discoverer, connections, urService, &mockedFolderSummaryService{}, errorLog, systemLog, cpu, nil, false).(*service) defer os.Remove(token) svc.started = addrChan diff --git a/lib/api/mocked_model_test.go b/lib/api/mocked_model_test.go index e967f0253..24eba2533 100644 --- a/lib/api/mocked_model_test.go +++ b/lib/api/mocked_model_test.go @@ -36,12 +36,8 @@ func (m *mockedModel) NeedFolderFiles(folder string, page, perpage int) ([]db.Fi return nil, nil, nil } -func (m *mockedModel) RemoteNeedFolderFiles(device protocol.DeviceID, folder string, page, perpage int) ([]db.FileInfoTruncated, error) { - return nil, nil -} - -func (m *mockedModel) NeedSize(folder string) db.Counts { - return db.Counts{} +func (m *mockedModel) FolderProgressBytesCompleted(_ string) int64 { + return 0 } func (m *mockedModel) ConnectionStats() map[string]interface{} { @@ -112,26 +108,6 @@ func (m *mockedModel) Connection(deviceID protocol.DeviceID) (connections.Connec return nil, false } -func (m *mockedModel) GlobalSize(folder string) db.Counts { - return db.Counts{} -} - -func (m *mockedModel) LocalSize(folder string) db.Counts { - return db.Counts{} -} - -func (m *mockedModel) ReceiveOnlyChangedSize(folder string) db.Counts { - return db.Counts{} -} - -func (m *mockedModel) CurrentSequence(folder string) (int64, bool) { - return 0, false -} - -func (m *mockedModel) RemoteSequence(folder string) (int64, bool) { - return 0, false -} - func (m *mockedModel) State(folder string) (string, time.Time, error) { return "", time.Time{}, nil } @@ -148,10 +124,6 @@ func (m *mockedModel) WatchError(folder string) error { return nil } -func (m *mockedModel) LocalChangedFiles(folder string, page, perpage int) []db.FileInfoTruncated { - return nil -} - func (m *mockedModel) Serve() {} func (m *mockedModel) Stop() {} @@ -188,3 +160,19 @@ func (m *mockedModel) GetHello(protocol.DeviceID) protocol.HelloIntf { } func (m *mockedModel) StartDeadlockDetector(timeout time.Duration) {} + +func (m *mockedModel) DBSnapshot(_ string) (*db.Snapshot, error) { + return nil, nil +} + +type mockedFolderSummaryService struct{} + +func (m *mockedFolderSummaryService) Serve() {} + +func (m *mockedFolderSummaryService) Stop() {} + +func (m *mockedFolderSummaryService) Summary(folder string) (map[string]interface{}, error) { + return map[string]interface{}{"mocked": true}, nil +} + +func (m *mockedFolderSummaryService) OnEventRequest() {} diff --git a/lib/db/benchmark_test.go b/lib/db/benchmark_test.go index cb5ceb49b..9d3482bd6 100644 --- a/lib/db/benchmark_test.go +++ b/lib/db/benchmark_test.go @@ -145,10 +145,12 @@ func BenchmarkNeedHalf(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { count := 0 - benchS.WithNeed(protocol.LocalDeviceID, func(fi db.FileIntf) bool { + snap := benchS.Snapshot() + snap.WithNeed(protocol.LocalDeviceID, func(fi db.FileIntf) bool { count++ return true }) + snap.Release() if count != len(secondHalf) { b.Errorf("wrong length %d != %d", count, len(secondHalf)) } @@ -167,10 +169,12 @@ func BenchmarkNeedHalfRemote(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { count := 0 - fset.WithNeed(remoteDevice0, func(fi db.FileIntf) bool { + snap := fset.Snapshot() + snap.WithNeed(remoteDevice0, func(fi db.FileIntf) bool { count++ return true }) + snap.Release() if count != len(secondHalf) { b.Errorf("wrong length %d != %d", count, len(secondHalf)) } @@ -186,10 +190,12 @@ func BenchmarkHave(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { count := 0 - benchS.WithHave(protocol.LocalDeviceID, func(fi db.FileIntf) bool { + snap := benchS.Snapshot() + snap.WithHave(protocol.LocalDeviceID, func(fi db.FileIntf) bool { count++ return true }) + snap.Release() if count != len(firstHalf) { b.Errorf("wrong length %d != %d", count, len(firstHalf)) } @@ -205,10 +211,12 @@ func BenchmarkGlobal(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { count := 0 - benchS.WithGlobal(func(fi db.FileIntf) bool { + snap := benchS.Snapshot() + snap.WithGlobal(func(fi db.FileIntf) bool { count++ return true }) + snap.Release() if count != len(files) { b.Errorf("wrong length %d != %d", count, len(files)) } @@ -224,10 +232,12 @@ func BenchmarkNeedHalfTruncated(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { count := 0 - benchS.WithNeedTruncated(protocol.LocalDeviceID, func(fi db.FileIntf) bool { + snap := benchS.Snapshot() + snap.WithNeedTruncated(protocol.LocalDeviceID, func(fi db.FileIntf) bool { count++ return true }) + snap.Release() if count != len(secondHalf) { b.Errorf("wrong length %d != %d", count, len(secondHalf)) } @@ -243,10 +253,12 @@ func BenchmarkHaveTruncated(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { count := 0 - benchS.WithHaveTruncated(protocol.LocalDeviceID, func(fi db.FileIntf) bool { + snap := benchS.Snapshot() + snap.WithHaveTruncated(protocol.LocalDeviceID, func(fi db.FileIntf) bool { count++ return true }) + snap.Release() if count != len(firstHalf) { b.Errorf("wrong length %d != %d", count, len(firstHalf)) } @@ -262,10 +274,12 @@ func BenchmarkGlobalTruncated(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { count := 0 - benchS.WithGlobalTruncated(func(fi db.FileIntf) bool { + snap := benchS.Snapshot() + snap.WithGlobalTruncated(func(fi db.FileIntf) bool { count++ return true }) + snap.Release() if count != len(files) { b.Errorf("wrong length %d != %d", count, len(files)) } diff --git a/lib/db/db_test.go b/lib/db/db_test.go index 095fd5860..ed7b0310e 100644 --- a/lib/db/db_test.go +++ b/lib/db/db_test.go @@ -72,7 +72,9 @@ func TestIgnoredFiles(t *testing.T) { // Local files should have the "ignored" bit in addition to just being // generally invalid if we want to look at the simulation of that bit. - fi, ok := fs.Get(protocol.LocalDeviceID, "foo") + snap := fs.Snapshot() + defer snap.Release() + fi, ok := snap.Get(protocol.LocalDeviceID, "foo") if !ok { t.Fatal("foo should exist") } @@ -83,7 +85,7 @@ func TestIgnoredFiles(t *testing.T) { t.Error("foo should be ignored") } - fi, ok = fs.Get(protocol.LocalDeviceID, "bar") + fi, ok = snap.Get(protocol.LocalDeviceID, "bar") if !ok { t.Fatal("bar should exist") } @@ -97,7 +99,7 @@ func TestIgnoredFiles(t *testing.T) { // Remote files have the invalid bit as usual, and the IsInvalid() method // should pick this up too. - fi, ok = fs.Get(protocol.DeviceID{42}, "baz") + fi, ok = snap.Get(protocol.DeviceID{42}, "baz") if !ok { t.Fatal("baz should exist") } @@ -108,7 +110,7 @@ func TestIgnoredFiles(t *testing.T) { t.Error("baz should be invalid") } - fi, ok = fs.Get(protocol.DeviceID{42}, "quux") + fi, ok = snap.Get(protocol.DeviceID{42}, "quux") if !ok { t.Fatal("quux should exist") } @@ -167,7 +169,11 @@ func TestUpdate0to3(t *testing.T) { t.Fatal(err) } - if _, ok, err := db.getFileDirty(folder, protocol.LocalDeviceID[:], []byte(slashPrefixed)); err != nil { + trans, err := db.newReadOnlyTransaction() + if err != nil { + t.Fatal(err) + } + if _, ok, err := trans.getFile(folder, protocol.LocalDeviceID[:], []byte(slashPrefixed)); err != nil { t.Fatal(err) } else if ok { t.Error("File prefixed by '/' was not removed during transition to schema 1") @@ -186,7 +192,11 @@ func TestUpdate0to3(t *testing.T) { } found := false - _ = db.withHaveSequence(folder, 0, func(fi FileIntf) bool { + trans, err = db.newReadOnlyTransaction() + if err != nil { + t.Fatal(err) + } + _ = trans.withHaveSequence(folder, 0, func(fi FileIntf) bool { f := fi.(protocol.FileInfo) l.Infoln(f) if found { @@ -213,7 +223,11 @@ func TestUpdate0to3(t *testing.T) { haveUpdate0to3[remoteDevice1][0].Name: haveUpdate0to3[remoteDevice1][0], haveUpdate0to3[remoteDevice0][2].Name: haveUpdate0to3[remoteDevice0][2], } - _ = db.withNeed(folder, protocol.LocalDeviceID[:], false, func(fi FileIntf) bool { + trans, err = db.newReadOnlyTransaction() + if err != nil { + t.Fatal(err) + } + _ = trans.withNeed(folder, protocol.LocalDeviceID[:], false, func(fi FileIntf) bool { e, ok := need[fi.FileName()] if !ok { t.Error("Got unexpected needed file:", fi.FileName()) diff --git a/lib/db/lowlevel.go b/lib/db/lowlevel.go index 9024ddfca..c08848d61 100644 --- a/lib/db/lowlevel.go +++ b/lib/db/lowlevel.go @@ -194,405 +194,6 @@ func (db *Lowlevel) updateLocalFiles(folder []byte, fs []protocol.FileInfo, meta return t.commit() } -func (db *Lowlevel) withHave(folder, device, prefix []byte, truncate bool, fn Iterator) error { - t, err := db.newReadOnlyTransaction() - if err != nil { - return err - } - defer t.close() - - if len(prefix) > 0 { - unslashedPrefix := prefix - if bytes.HasSuffix(prefix, []byte{'/'}) { - unslashedPrefix = unslashedPrefix[:len(unslashedPrefix)-1] - } else { - prefix = append(prefix, '/') - } - - key, err := db.keyer.GenerateDeviceFileKey(nil, folder, device, unslashedPrefix) - if err != nil { - return err - } - if f, ok, err := t.getFileTrunc(key, true); err != nil { - return err - } else if ok && !fn(f) { - return nil - } - } - - key, err := db.keyer.GenerateDeviceFileKey(nil, folder, device, prefix) - if err != nil { - return err - } - dbi, err := t.NewPrefixIterator(key) - if err != nil { - return err - } - defer dbi.Release() - - for dbi.Next() { - name := db.keyer.NameFromDeviceFileKey(dbi.Key()) - if len(prefix) > 0 && !bytes.HasPrefix(name, prefix) { - return nil - } - - f, err := unmarshalTrunc(dbi.Value(), truncate) - if err != nil { - l.Debugln("unmarshal error:", err) - continue - } - if !fn(f) { - return nil - } - } - return dbi.Error() -} - -func (db *Lowlevel) withHaveSequence(folder []byte, startSeq int64, fn Iterator) error { - t, err := db.newReadOnlyTransaction() - if err != nil { - return err - } - defer t.close() - - first, err := db.keyer.GenerateSequenceKey(nil, folder, startSeq) - if err != nil { - return err - } - last, err := db.keyer.GenerateSequenceKey(nil, folder, maxInt64) - if err != nil { - return err - } - dbi, err := t.NewRangeIterator(first, last) - if err != nil { - return err - } - defer dbi.Release() - - for dbi.Next() { - f, ok, err := t.getFileByKey(dbi.Value()) - if err != nil { - return err - } - if !ok { - l.Debugln("missing file for sequence number", db.keyer.SequenceFromSequenceKey(dbi.Key())) - continue - } - - if shouldDebug() { - if seq := db.keyer.SequenceFromSequenceKey(dbi.Key()); f.Sequence != seq { - l.Warnf("Sequence index corruption (folder %v, file %v): sequence %d != expected %d", string(folder), f.Name, f.Sequence, seq) - panic("sequence index corruption") - } - } - if !fn(f) { - return nil - } - } - return dbi.Error() -} - -func (db *Lowlevel) withAllFolderTruncated(folder []byte, fn func(device []byte, f FileInfoTruncated) bool) error { - t, err := db.newReadWriteTransaction() - if err != nil { - return err - } - defer t.close() - - key, err := db.keyer.GenerateDeviceFileKey(nil, folder, nil, nil) - if err != nil { - return err - } - dbi, err := t.NewPrefixIterator(key.WithoutNameAndDevice()) - if err != nil { - return err - } - defer dbi.Release() - - var gk, keyBuf []byte - for dbi.Next() { - device, ok := db.keyer.DeviceFromDeviceFileKey(dbi.Key()) - if !ok { - // Not having the device in the index is bad. Clear it. - if err := t.Delete(dbi.Key()); err != nil { - return err - } - continue - } - var f FileInfoTruncated - // The iterator function may keep a reference to the unmarshalled - // struct, which in turn references the buffer it was unmarshalled - // from. dbi.Value() just returns an internal slice that it reuses, so - // we need to copy it. - err := f.Unmarshal(append([]byte{}, dbi.Value()...)) - if err != nil { - return err - } - - switch f.Name { - case "", ".", "..", "/": // A few obviously invalid filenames - l.Infof("Dropping invalid filename %q from database", f.Name) - name := []byte(f.Name) - gk, err = db.keyer.GenerateGlobalVersionKey(gk, folder, name) - if err != nil { - return err - } - keyBuf, err = t.removeFromGlobal(gk, keyBuf, folder, device, name, nil) - if err != nil { - return err - } - if err := t.Delete(dbi.Key()); err != nil { - return err - } - continue - } - - if !fn(device, f) { - return nil - } - } - if err := dbi.Error(); err != nil { - return err - } - return t.commit() -} - -func (db *Lowlevel) getFileDirty(folder, device, file []byte) (protocol.FileInfo, bool, error) { - t, err := db.newReadOnlyTransaction() - if err != nil { - return protocol.FileInfo{}, false, err - } - defer t.close() - return t.getFile(folder, device, file) -} - -func (db *Lowlevel) getGlobalDirty(folder, file []byte, truncate bool) (FileIntf, bool, error) { - t, err := db.newReadOnlyTransaction() - if err != nil { - return nil, false, err - } - defer t.close() - _, f, ok, err := t.getGlobal(nil, folder, file, truncate) - return f, ok, err -} - -func (db *Lowlevel) withGlobal(folder, prefix []byte, truncate bool, fn Iterator) error { - t, err := db.newReadOnlyTransaction() - if err != nil { - return err - } - defer t.close() - - if len(prefix) > 0 { - unslashedPrefix := prefix - if bytes.HasSuffix(prefix, []byte{'/'}) { - unslashedPrefix = unslashedPrefix[:len(unslashedPrefix)-1] - } else { - prefix = append(prefix, '/') - } - - if _, f, ok, err := t.getGlobal(nil, folder, unslashedPrefix, truncate); err != nil { - return err - } else if ok && !fn(f) { - return nil - } - } - - key, err := db.keyer.GenerateGlobalVersionKey(nil, folder, prefix) - if err != nil { - return err - } - dbi, err := t.NewPrefixIterator(key) - if err != nil { - return err - } - defer dbi.Release() - - var dk []byte - for dbi.Next() { - name := db.keyer.NameFromGlobalVersionKey(dbi.Key()) - if len(prefix) > 0 && !bytes.HasPrefix(name, prefix) { - return nil - } - - vl, ok := unmarshalVersionList(dbi.Value()) - if !ok { - continue - } - - dk, err = db.keyer.GenerateDeviceFileKey(dk, folder, vl.Versions[0].Device, name) - if err != nil { - return err - } - - f, ok, err := t.getFileTrunc(dk, truncate) - if err != nil { - return err - } - if !ok { - continue - } - - if !fn(f) { - return nil - } - } - if err != nil { - return err - } - return dbi.Error() -} - -func (db *Lowlevel) availability(folder, file []byte) ([]protocol.DeviceID, error) { - k, err := db.keyer.GenerateGlobalVersionKey(nil, folder, file) - if err != nil { - return nil, err - } - bs, err := db.Get(k) - if backend.IsNotFound(err) { - return nil, nil - } - if err != nil { - return nil, err - } - - vl, ok := unmarshalVersionList(bs) - if !ok { - return nil, nil - } - - var devices []protocol.DeviceID - for _, v := range vl.Versions { - if !v.Version.Equal(vl.Versions[0].Version) { - break - } - if v.Invalid { - continue - } - n := protocol.DeviceIDFromBytes(v.Device) - devices = append(devices, n) - } - - return devices, nil -} - -func (db *Lowlevel) withNeed(folder, device []byte, truncate bool, fn Iterator) error { - if bytes.Equal(device, protocol.LocalDeviceID[:]) { - return db.withNeedLocal(folder, truncate, fn) - } - - t, err := db.newReadOnlyTransaction() - if err != nil { - return err - } - defer t.close() - - key, err := db.keyer.GenerateGlobalVersionKey(nil, folder, nil) - if err != nil { - return err - } - dbi, err := t.NewPrefixIterator(key.WithoutName()) - if err != nil { - return err - } - defer dbi.Release() - - var dk []byte - devID := protocol.DeviceIDFromBytes(device) - for dbi.Next() { - vl, ok := unmarshalVersionList(dbi.Value()) - if !ok { - continue - } - - haveFV, have := vl.Get(device) - // XXX: This marks Concurrent (i.e. conflicting) changes as - // needs. Maybe we should do that, but it needs special - // handling in the puller. - if have && haveFV.Version.GreaterEqual(vl.Versions[0].Version) { - continue - } - - name := db.keyer.NameFromGlobalVersionKey(dbi.Key()) - needVersion := vl.Versions[0].Version - needDevice := protocol.DeviceIDFromBytes(vl.Versions[0].Device) - - for i := range vl.Versions { - if !vl.Versions[i].Version.Equal(needVersion) { - // We haven't found a valid copy of the file with the needed version. - break - } - - if vl.Versions[i].Invalid { - // The file is marked invalid, don't use it. - continue - } - - dk, err = db.keyer.GenerateDeviceFileKey(dk, folder, vl.Versions[i].Device, name) - if err != nil { - return err - } - gf, ok, err := t.getFileTrunc(dk, truncate) - if err != nil { - return err - } - if !ok { - continue - } - - if gf.IsDeleted() && !have { - // We don't need deleted files that we don't have - break - } - - l.Debugf("need folder=%q device=%v name=%q have=%v invalid=%v haveV=%v globalV=%v globalDev=%v", folder, devID, name, have, haveFV.Invalid, haveFV.Version, needVersion, needDevice) - - if !fn(gf) { - return nil - } - - // This file is handled, no need to look further in the version list - break - } - } - return dbi.Error() -} - -func (db *Lowlevel) withNeedLocal(folder []byte, truncate bool, fn Iterator) error { - t, err := db.newReadOnlyTransaction() - if err != nil { - return err - } - defer t.close() - - key, err := db.keyer.GenerateNeedFileKey(nil, folder, nil) - if err != nil { - return err - } - dbi, err := t.NewPrefixIterator(key.WithoutName()) - if err != nil { - return err - } - defer dbi.Release() - - var keyBuf []byte - var f FileIntf - var ok bool - for dbi.Next() { - keyBuf, f, ok, err = t.getGlobal(keyBuf, folder, db.keyer.NameFromGlobalVersionKey(dbi.Key()), truncate) - if err != nil { - return err - } - if !ok { - continue - } - if !fn(f) { - return nil - } - } - return dbi.Error() -} - func (db *Lowlevel) dropFolder(folder []byte) error { t, err := db.newReadWriteTransaction() if err != nil { diff --git a/lib/db/meta.go b/lib/db/meta.go index 65e73035a..63a773d0c 100644 --- a/lib/db/meta.go +++ b/lib/db/meta.go @@ -15,12 +15,16 @@ import ( "github.com/syncthing/syncthing/lib/sync" ) -// metadataTracker keeps metadata on a per device, per local flag basis. -type metadataTracker struct { - mut sync.RWMutex +type countsMap struct { counts CountsSet indexes map[metaKey]int // device ID + local flags -> index in counts - dirty bool +} + +// metadataTracker keeps metadata on a per device, per local flag basis. +type metadataTracker struct { + countsMap + mut sync.RWMutex + dirty bool } type metaKey struct { @@ -30,8 +34,10 @@ type metaKey struct { func newMetadataTracker() *metadataTracker { return &metadataTracker{ - mut: sync.NewRWMutex(), - indexes: make(map[metaKey]int), + mut: sync.NewRWMutex(), + countsMap: countsMap{ + indexes: make(map[metaKey]int), + }, } } @@ -49,7 +55,7 @@ func (m *metadataTracker) Unmarshal(bs []byte) error { return nil } -// Unmarshal returns the protobuf representation of the metadataTracker +// Marshal returns the protobuf representation of the metadataTracker func (m *metadataTracker) Marshal() ([]byte, error) { return m.counts.Marshal() } @@ -260,16 +266,11 @@ func (m *metadataTracker) resetCounts(dev protocol.DeviceID) { m.mut.Unlock() } -// Counts returns the counts for the given device ID and flag. `flag` should -// be zero or have exactly one bit set. -func (m *metadataTracker) Counts(dev protocol.DeviceID, flag uint32) Counts { +func (m *countsMap) Counts(dev protocol.DeviceID, flag uint32) Counts { if bits.OnesCount32(flag) > 1 { panic("incorrect usage: set at most one bit in flag") } - m.mut.RLock() - defer m.mut.RUnlock() - idx, ok := m.indexes[metaKey{dev, flag}] if !ok { return Counts{} @@ -278,6 +279,37 @@ func (m *metadataTracker) Counts(dev protocol.DeviceID, flag uint32) Counts { return m.counts.Counts[idx] } +// Counts returns the counts for the given device ID and flag. `flag` should +// be zero or have exactly one bit set. +func (m *metadataTracker) Counts(dev protocol.DeviceID, flag uint32) Counts { + m.mut.RLock() + defer m.mut.RUnlock() + + return m.countsMap.Counts(dev, flag) +} + +// Snapshot returns a copy of the metadata for reading. +func (m *metadataTracker) Snapshot() *countsMap { + m.mut.RLock() + defer m.mut.RUnlock() + + c := &countsMap{ + counts: CountsSet{ + Counts: make([]Counts, len(m.counts.Counts)), + Created: m.counts.Created, + }, + indexes: make(map[metaKey]int, len(m.indexes)), + } + for k, v := range m.indexes { + c.indexes[k] = v + } + for i := range m.counts.Counts { + c.counts.Counts[i] = m.counts.Counts[i] + } + + return c +} + // nextLocalSeq allocates a new local sequence number func (m *metadataTracker) nextLocalSeq() int64 { m.mut.Lock() @@ -291,26 +323,25 @@ func (m *metadataTracker) nextLocalSeq() int64 { // devices returns the list of devices tracked, excluding the local device // (which we don't know the ID of) func (m *metadataTracker) devices() []protocol.DeviceID { - devs := make(map[protocol.DeviceID]struct{}, len(m.counts.Counts)) - m.mut.RLock() + defer m.mut.RUnlock() + return m.countsMap.devices() +} + +func (m *countsMap) devices() []protocol.DeviceID { + devs := make([]protocol.DeviceID, 0, len(m.counts.Counts)) + for _, dev := range m.counts.Counts { if dev.Sequence > 0 { id := protocol.DeviceIDFromBytes(dev.DeviceID) if id == protocol.GlobalDeviceID || id == protocol.LocalDeviceID { continue } - devs[id] = struct{}{} + devs = append(devs, id) } } - m.mut.RUnlock() - devList := make([]protocol.DeviceID, 0, len(devs)) - for dev := range devs { - devList = append(devList, dev) - } - - return devList + return devs } func (m *metadataTracker) Created() time.Time { diff --git a/lib/db/schemaupdater.go b/lib/db/schemaupdater.go index d91a32436..7f6c8b1b3 100644 --- a/lib/db/schemaupdater.go +++ b/lib/db/schemaupdater.go @@ -228,7 +228,7 @@ func (db *schemaUpdater) updateSchema1to2() error { for _, folderStr := range db.ListFolders() { folder := []byte(folderStr) var putErr error - err := db.withHave(folder, protocol.LocalDeviceID[:], nil, true, func(f FileIntf) bool { + err := t.withHave(folder, protocol.LocalDeviceID[:], nil, true, func(f FileIntf) bool { sk, putErr = db.keyer.GenerateSequenceKey(sk, folder, f.SequenceNo()) if putErr != nil { return false @@ -263,7 +263,7 @@ func (db *schemaUpdater) updateSchema2to3() error { for _, folderStr := range db.ListFolders() { folder := []byte(folderStr) var putErr error - err := db.withGlobal(folder, nil, true, func(f FileIntf) bool { + err := t.withGlobal(folder, nil, true, func(f FileIntf) bool { name := []byte(f.FileName()) dk, putErr = db.keyer.GenerateDeviceFileKey(dk, folder, protocol.LocalDeviceID[:], name) if putErr != nil { @@ -339,7 +339,7 @@ func (db *schemaUpdater) updateSchema5to6() error { for _, folderStr := range db.ListFolders() { folder := []byte(folderStr) var putErr error - err := db.withHave(folder, protocol.LocalDeviceID[:], nil, false, func(f FileIntf) bool { + err := t.withHave(folder, protocol.LocalDeviceID[:], nil, false, func(f FileIntf) bool { if !f.IsInvalid() { return true } @@ -382,7 +382,7 @@ func (db *schemaUpdater) updateSchema6to7() error { for _, folderStr := range db.ListFolders() { folder := []byte(folderStr) var delErr error - err := db.withNeedLocal(folder, false, func(f FileIntf) bool { + err := t.withNeedLocal(folder, false, func(f FileIntf) bool { name := []byte(f.FileName()) global := f.(protocol.FileInfo) gk, delErr = db.keyer.GenerateGlobalVersionKey(gk, folder, name) diff --git a/lib/db/set.go b/lib/db/set.go index e61180823..c7963a9d8 100644 --- a/lib/db/set.go +++ b/lib/db/set.go @@ -105,8 +105,12 @@ func (s *FileSet) recalcCounts() error { return err } + t, err := s.db.newReadWriteTransaction() + if err != nil { + return err + } var deviceID protocol.DeviceID - err := s.db.withAllFolderTruncated([]byte(s.folder), func(device []byte, f FileInfoTruncated) bool { + err = t.withAllFolderTruncated([]byte(s.folder), func(device []byte, f FileInfoTruncated) bool { copy(deviceID[:], device) s.meta.addFile(deviceID, f) return true @@ -183,75 +187,97 @@ func (s *FileSet) Update(device protocol.DeviceID, fs []protocol.FileInfo) { } } -func (s *FileSet) WithNeed(device protocol.DeviceID, fn Iterator) { +type Snapshot struct { + folder string + t readOnlyTransaction + meta *countsMap +} + +func (s *FileSet) Snapshot() *Snapshot { + t, err := s.db.newReadOnlyTransaction() + if err != nil { + panic(err) + } + return &Snapshot{ + folder: s.folder, + t: t, + meta: s.meta.Snapshot(), + } +} + +func (s *Snapshot) Release() { + s.t.close() +} + +func (s *Snapshot) WithNeed(device protocol.DeviceID, fn Iterator) { l.Debugf("%s WithNeed(%v)", s.folder, device) - if err := s.db.withNeed([]byte(s.folder), device[:], false, nativeFileIterator(fn)); err != nil && !backend.IsClosed(err) { + if err := s.t.withNeed([]byte(s.folder), device[:], false, nativeFileIterator(fn)); err != nil && !backend.IsClosed(err) { panic(err) } } -func (s *FileSet) WithNeedTruncated(device protocol.DeviceID, fn Iterator) { +func (s *Snapshot) WithNeedTruncated(device protocol.DeviceID, fn Iterator) { l.Debugf("%s WithNeedTruncated(%v)", s.folder, device) - if err := s.db.withNeed([]byte(s.folder), device[:], true, nativeFileIterator(fn)); err != nil && !backend.IsClosed(err) { + if err := s.t.withNeed([]byte(s.folder), device[:], true, nativeFileIterator(fn)); err != nil && !backend.IsClosed(err) { panic(err) } } -func (s *FileSet) WithHave(device protocol.DeviceID, fn Iterator) { +func (s *Snapshot) WithHave(device protocol.DeviceID, fn Iterator) { l.Debugf("%s WithHave(%v)", s.folder, device) - if err := s.db.withHave([]byte(s.folder), device[:], nil, false, nativeFileIterator(fn)); err != nil && !backend.IsClosed(err) { + if err := s.t.withHave([]byte(s.folder), device[:], nil, false, nativeFileIterator(fn)); err != nil && !backend.IsClosed(err) { panic(err) } } -func (s *FileSet) WithHaveTruncated(device protocol.DeviceID, fn Iterator) { +func (s *Snapshot) WithHaveTruncated(device protocol.DeviceID, fn Iterator) { l.Debugf("%s WithHaveTruncated(%v)", s.folder, device) - if err := s.db.withHave([]byte(s.folder), device[:], nil, true, nativeFileIterator(fn)); err != nil && !backend.IsClosed(err) { + if err := s.t.withHave([]byte(s.folder), device[:], nil, true, nativeFileIterator(fn)); err != nil && !backend.IsClosed(err) { panic(err) } } -func (s *FileSet) WithHaveSequence(startSeq int64, fn Iterator) { +func (s *Snapshot) WithHaveSequence(startSeq int64, fn Iterator) { l.Debugf("%s WithHaveSequence(%v)", s.folder, startSeq) - if err := s.db.withHaveSequence([]byte(s.folder), startSeq, nativeFileIterator(fn)); err != nil && !backend.IsClosed(err) { + if err := s.t.withHaveSequence([]byte(s.folder), startSeq, nativeFileIterator(fn)); err != nil && !backend.IsClosed(err) { panic(err) } } // Except for an item with a path equal to prefix, only children of prefix are iterated. // E.g. for prefix "dir", "dir/file" is iterated, but "dir.file" is not. -func (s *FileSet) WithPrefixedHaveTruncated(device protocol.DeviceID, prefix string, fn Iterator) { +func (s *Snapshot) WithPrefixedHaveTruncated(device protocol.DeviceID, prefix string, fn Iterator) { l.Debugf(`%s WithPrefixedHaveTruncated(%v, "%v")`, s.folder, device, prefix) - if err := s.db.withHave([]byte(s.folder), device[:], []byte(osutil.NormalizedFilename(prefix)), true, nativeFileIterator(fn)); err != nil && !backend.IsClosed(err) { + if err := s.t.withHave([]byte(s.folder), device[:], []byte(osutil.NormalizedFilename(prefix)), true, nativeFileIterator(fn)); err != nil && !backend.IsClosed(err) { panic(err) } } -func (s *FileSet) WithGlobal(fn Iterator) { +func (s *Snapshot) WithGlobal(fn Iterator) { l.Debugf("%s WithGlobal()", s.folder) - if err := s.db.withGlobal([]byte(s.folder), nil, false, nativeFileIterator(fn)); err != nil && !backend.IsClosed(err) { + if err := s.t.withGlobal([]byte(s.folder), nil, false, nativeFileIterator(fn)); err != nil && !backend.IsClosed(err) { panic(err) } } -func (s *FileSet) WithGlobalTruncated(fn Iterator) { +func (s *Snapshot) WithGlobalTruncated(fn Iterator) { l.Debugf("%s WithGlobalTruncated()", s.folder) - if err := s.db.withGlobal([]byte(s.folder), nil, true, nativeFileIterator(fn)); err != nil && !backend.IsClosed(err) { + if err := s.t.withGlobal([]byte(s.folder), nil, true, nativeFileIterator(fn)); err != nil && !backend.IsClosed(err) { panic(err) } } // Except for an item with a path equal to prefix, only children of prefix are iterated. // E.g. for prefix "dir", "dir/file" is iterated, but "dir.file" is not. -func (s *FileSet) WithPrefixedGlobalTruncated(prefix string, fn Iterator) { +func (s *Snapshot) WithPrefixedGlobalTruncated(prefix string, fn Iterator) { l.Debugf(`%s WithPrefixedGlobalTruncated("%v")`, s.folder, prefix) - if err := s.db.withGlobal([]byte(s.folder), []byte(osutil.NormalizedFilename(prefix)), true, nativeFileIterator(fn)); err != nil && !backend.IsClosed(err) { + if err := s.t.withGlobal([]byte(s.folder), []byte(osutil.NormalizedFilename(prefix)), true, nativeFileIterator(fn)); err != nil && !backend.IsClosed(err) { panic(err) } } -func (s *FileSet) Get(device protocol.DeviceID, file string) (protocol.FileInfo, bool) { - f, ok, err := s.db.getFileDirty([]byte(s.folder), device[:], []byte(osutil.NormalizedFilename(file))) +func (s *Snapshot) Get(device protocol.DeviceID, file string) (protocol.FileInfo, bool) { + f, ok, err := s.t.getFile([]byte(s.folder), device[:], []byte(osutil.NormalizedFilename(file))) if backend.IsClosed(err) { return protocol.FileInfo{}, false } else if err != nil { @@ -261,8 +287,8 @@ func (s *FileSet) Get(device protocol.DeviceID, file string) (protocol.FileInfo, return f, ok } -func (s *FileSet) GetGlobal(file string) (protocol.FileInfo, bool) { - fi, ok, err := s.db.getGlobalDirty([]byte(s.folder), []byte(osutil.NormalizedFilename(file)), false) +func (s *Snapshot) GetGlobal(file string) (protocol.FileInfo, bool) { + _, fi, ok, err := s.t.getGlobal(nil, []byte(s.folder), []byte(osutil.NormalizedFilename(file)), false) if backend.IsClosed(err) { return protocol.FileInfo{}, false } else if err != nil { @@ -276,8 +302,8 @@ func (s *FileSet) GetGlobal(file string) (protocol.FileInfo, bool) { return f, true } -func (s *FileSet) GetGlobalTruncated(file string) (FileInfoTruncated, bool) { - fi, ok, err := s.db.getGlobalDirty([]byte(s.folder), []byte(osutil.NormalizedFilename(file)), true) +func (s *Snapshot) GetGlobalTruncated(file string) (FileInfoTruncated, bool) { + _, fi, ok, err := s.t.getGlobal(nil, []byte(s.folder), []byte(osutil.NormalizedFilename(file)), true) if backend.IsClosed(err) { return FileInfoTruncated{}, false } else if err != nil { @@ -291,8 +317,8 @@ func (s *FileSet) GetGlobalTruncated(file string) (FileInfoTruncated, bool) { return f, true } -func (s *FileSet) Availability(file string) []protocol.DeviceID { - av, err := s.db.availability([]byte(s.folder), []byte(osutil.NormalizedFilename(file))) +func (s *Snapshot) Availability(file string) []protocol.DeviceID { + av, err := s.t.availability([]byte(s.folder), []byte(osutil.NormalizedFilename(file))) if backend.IsClosed(err) { return nil } else if err != nil { @@ -301,26 +327,110 @@ func (s *FileSet) Availability(file string) []protocol.DeviceID { return av } -func (s *FileSet) Sequence(device protocol.DeviceID) int64 { - return s.meta.Sequence(device) +func (s *Snapshot) Sequence(device protocol.DeviceID) int64 { + return s.meta.Counts(device, 0).Sequence } -func (s *FileSet) LocalSize() Counts { +// RemoteSequence returns the change version for the given folder, as +// sent by remote peers. This is guaranteed to increment if the contents of +// the remote or global folder has changed. +func (s *Snapshot) RemoteSequence() int64 { + var ver int64 + + for _, device := range s.meta.devices() { + ver += s.Sequence(device) + } + + return ver +} + +func (s *Snapshot) LocalSize() Counts { local := s.meta.Counts(protocol.LocalDeviceID, 0) - recvOnlyChanged := s.meta.Counts(protocol.LocalDeviceID, protocol.FlagLocalReceiveOnly) - return local.Add(recvOnlyChanged) + return local.Add(s.ReceiveOnlyChangedSize()) } -func (s *FileSet) ReceiveOnlyChangedSize() Counts { +func (s *Snapshot) ReceiveOnlyChangedSize() Counts { return s.meta.Counts(protocol.LocalDeviceID, protocol.FlagLocalReceiveOnly) } -func (s *FileSet) GlobalSize() Counts { +func (s *Snapshot) GlobalSize() Counts { global := s.meta.Counts(protocol.GlobalDeviceID, 0) recvOnlyChanged := s.meta.Counts(protocol.GlobalDeviceID, protocol.FlagLocalReceiveOnly) return global.Add(recvOnlyChanged) } +func (s *Snapshot) NeedSize() Counts { + var result Counts + s.WithNeedTruncated(protocol.LocalDeviceID, func(f FileIntf) bool { + switch { + case f.IsDeleted(): + result.Deleted++ + case f.IsDirectory(): + result.Directories++ + case f.IsSymlink(): + result.Symlinks++ + default: + result.Files++ + result.Bytes += f.FileSize() + } + return true + }) + return result +} + +// LocalChangedFiles returns a paginated list of currently needed files in +// progress, queued, and to be queued on next puller iteration, as well as the +// total number of files currently needed. +func (s *Snapshot) LocalChangedFiles(page, perpage int) []FileInfoTruncated { + if s.ReceiveOnlyChangedSize().TotalItems() == 0 { + return nil + } + + files := make([]FileInfoTruncated, 0, perpage) + + skip := (page - 1) * perpage + get := perpage + + s.WithHaveTruncated(protocol.LocalDeviceID, func(f FileIntf) bool { + if !f.IsReceiveOnlyChanged() { + return true + } + if skip > 0 { + skip-- + return true + } + ft := f.(FileInfoTruncated) + files = append(files, ft) + get-- + return get > 0 + }) + + return files +} + +// RemoteNeedFolderFiles returns paginated list of currently needed files in +// progress, queued, and to be queued on next puller iteration, as well as the +// total number of files currently needed. +func (s *Snapshot) RemoteNeedFolderFiles(device protocol.DeviceID, page, perpage int) []FileInfoTruncated { + files := make([]FileInfoTruncated, 0, perpage) + skip := (page - 1) * perpage + get := perpage + s.WithNeedTruncated(device, func(f FileIntf) bool { + if skip > 0 { + skip-- + return true + } + files = append(files, f.(FileInfoTruncated)) + get-- + return get > 0 + }) + return files +} + +func (s *FileSet) Sequence(device protocol.DeviceID) int64 { + return s.meta.Sequence(device) +} + func (s *FileSet) IndexID(device protocol.DeviceID) protocol.IndexID { id, err := s.db.getIndexID(device[:], []byte(s.folder)) if backend.IsClosed(err) { diff --git a/lib/db/set_test.go b/lib/db/set_test.go index 256cf0333..067b92894 100644 --- a/lib/db/set_test.go +++ b/lib/db/set_test.go @@ -46,7 +46,9 @@ func genBlocks(n int) []protocol.BlockInfo { func globalList(s *db.FileSet) []protocol.FileInfo { var fs []protocol.FileInfo - s.WithGlobal(func(fi db.FileIntf) bool { + snap := s.Snapshot() + defer snap.Release() + snap.WithGlobal(func(fi db.FileIntf) bool { f := fi.(protocol.FileInfo) fs = append(fs, f) return true @@ -55,7 +57,9 @@ func globalList(s *db.FileSet) []protocol.FileInfo { } func globalListPrefixed(s *db.FileSet, prefix string) []db.FileInfoTruncated { var fs []db.FileInfoTruncated - s.WithPrefixedGlobalTruncated(prefix, func(fi db.FileIntf) bool { + snap := s.Snapshot() + defer snap.Release() + snap.WithPrefixedGlobalTruncated(prefix, func(fi db.FileIntf) bool { f := fi.(db.FileInfoTruncated) fs = append(fs, f) return true @@ -65,7 +69,9 @@ func globalListPrefixed(s *db.FileSet, prefix string) []db.FileInfoTruncated { func haveList(s *db.FileSet, n protocol.DeviceID) []protocol.FileInfo { var fs []protocol.FileInfo - s.WithHave(n, func(fi db.FileIntf) bool { + snap := s.Snapshot() + defer snap.Release() + snap.WithHave(n, func(fi db.FileIntf) bool { f := fi.(protocol.FileInfo) fs = append(fs, f) return true @@ -75,7 +81,9 @@ func haveList(s *db.FileSet, n protocol.DeviceID) []protocol.FileInfo { func haveListPrefixed(s *db.FileSet, n protocol.DeviceID, prefix string) []db.FileInfoTruncated { var fs []db.FileInfoTruncated - s.WithPrefixedHaveTruncated(n, prefix, func(fi db.FileIntf) bool { + snap := s.Snapshot() + defer snap.Release() + snap.WithPrefixedHaveTruncated(n, prefix, func(fi db.FileIntf) bool { f := fi.(db.FileInfoTruncated) fs = append(fs, f) return true @@ -85,7 +93,9 @@ func haveListPrefixed(s *db.FileSet, n protocol.DeviceID, prefix string) []db.Fi func needList(s *db.FileSet, n protocol.DeviceID) []protocol.FileInfo { var fs []protocol.FileInfo - s.WithNeed(n, func(fi db.FileIntf) bool { + snap := s.Snapshot() + defer snap.Release() + snap.WithNeed(n, func(fi db.FileIntf) bool { f := fi.(protocol.FileInfo) fs = append(fs, f) return true @@ -206,7 +216,7 @@ func TestGlobalSet(t *testing.T) { } globalBytes += f.FileSize() } - gs := m.GlobalSize() + gs := globalSize(m) if gs.Files != globalFiles { t.Errorf("Incorrect GlobalSize files; %d != %d", gs.Files, globalFiles) } @@ -242,7 +252,7 @@ func TestGlobalSet(t *testing.T) { } haveBytes += f.FileSize() } - ls := m.LocalSize() + ls := localSize(m) if ls.Files != haveFiles { t.Errorf("Incorrect LocalSize files; %d != %d", ls.Files, haveFiles) } @@ -277,7 +287,9 @@ func TestGlobalSet(t *testing.T) { t.Errorf("Need incorrect;\n A: %v !=\n E: %v", n, expectedRemoteNeed) } - f, ok := m.Get(protocol.LocalDeviceID, "b") + snap := m.Snapshot() + defer snap.Release() + f, ok := snap.Get(protocol.LocalDeviceID, "b") if !ok { t.Error("Unexpectedly not OK") } @@ -285,7 +297,7 @@ func TestGlobalSet(t *testing.T) { t.Errorf("Get incorrect;\n A: %v !=\n E: %v", f, localTot[1]) } - f, ok = m.Get(remoteDevice0, "b") + f, ok = snap.Get(remoteDevice0, "b") if !ok { t.Error("Unexpectedly not OK") } @@ -293,7 +305,7 @@ func TestGlobalSet(t *testing.T) { t.Errorf("Get incorrect;\n A: %v !=\n E: %v", f, remote1[0]) } - f, ok = m.GetGlobal("b") + f, ok = snap.GetGlobal("b") if !ok { t.Error("Unexpectedly not OK") } @@ -301,7 +313,7 @@ func TestGlobalSet(t *testing.T) { t.Errorf("GetGlobal incorrect;\n A: %v !=\n E: %v", f, remote1[0]) } - f, ok = m.Get(protocol.LocalDeviceID, "zz") + f, ok = snap.Get(protocol.LocalDeviceID, "zz") if ok { t.Error("Unexpectedly OK") } @@ -309,7 +321,7 @@ func TestGlobalSet(t *testing.T) { t.Errorf("Get incorrect;\n A: %v !=\n E: %v", f, protocol.FileInfo{}) } - f, ok = m.GetGlobal("zz") + f, ok = snap.GetGlobal("zz") if ok { t.Error("Unexpectedly OK") } @@ -318,15 +330,15 @@ func TestGlobalSet(t *testing.T) { } av := []protocol.DeviceID{protocol.LocalDeviceID, remoteDevice0} - a := m.Availability("a") + a := snap.Availability("a") if !(len(a) == 2 && (a[0] == av[0] && a[1] == av[1] || a[0] == av[1] && a[1] == av[0])) { t.Errorf("Availability incorrect;\n A: %v !=\n E: %v", a, av) } - a = m.Availability("b") + a = snap.Availability("b") if len(a) != 1 || a[0] != remoteDevice0 { t.Errorf("Availability incorrect;\n A: %v !=\n E: %v", a, remoteDevice0) } - a = m.Availability("d") + a = snap.Availability("d") if len(a) != 1 || a[0] != protocol.LocalDeviceID { t.Errorf("Availability incorrect;\n A: %v !=\n E: %v", a, protocol.LocalDeviceID) } @@ -446,19 +458,22 @@ func TestInvalidAvailability(t *testing.T) { replace(s, remoteDevice0, remote0Have) replace(s, remoteDevice1, remote1Have) - if av := s.Availability("both"); len(av) != 2 { + snap := s.Snapshot() + defer snap.Release() + + if av := snap.Availability("both"); len(av) != 2 { t.Error("Incorrect availability for 'both':", av) } - if av := s.Availability("r0only"); len(av) != 1 || av[0] != remoteDevice0 { + if av := snap.Availability("r0only"); len(av) != 1 || av[0] != remoteDevice0 { t.Error("Incorrect availability for 'r0only':", av) } - if av := s.Availability("r1only"); len(av) != 1 || av[0] != remoteDevice1 { + if av := snap.Availability("r1only"); len(av) != 1 || av[0] != remoteDevice1 { t.Error("Incorrect availability for 'r1only':", av) } - if av := s.Availability("none"); len(av) != 0 { + if av := snap.Availability("none"); len(av) != 0 { t.Error("Incorrect availability for 'none':", av) } } @@ -826,20 +841,20 @@ func TestIssue4701(t *testing.T) { s.Update(protocol.LocalDeviceID, localHave) - if c := s.LocalSize(); c.Files != 1 { + if c := localSize(s); c.Files != 1 { t.Errorf("Expected 1 local file, got %v", c.Files) } - if c := s.GlobalSize(); c.Files != 1 { + if c := globalSize(s); c.Files != 1 { t.Errorf("Expected 1 global file, got %v", c.Files) } localHave[1].LocalFlags = 0 s.Update(protocol.LocalDeviceID, localHave) - if c := s.LocalSize(); c.Files != 2 { + if c := localSize(s); c.Files != 2 { t.Errorf("Expected 2 local files, got %v", c.Files) } - if c := s.GlobalSize(); c.Files != 2 { + if c := globalSize(s); c.Files != 2 { t.Errorf("Expected 2 global files, got %v", c.Files) } @@ -847,10 +862,10 @@ func TestIssue4701(t *testing.T) { localHave[1].LocalFlags = protocol.FlagLocalIgnored s.Update(protocol.LocalDeviceID, localHave) - if c := s.LocalSize(); c.Files != 0 { + if c := localSize(s); c.Files != 0 { t.Errorf("Expected 0 local files, got %v", c.Files) } - if c := s.GlobalSize(); c.Files != 0 { + if c := globalSize(s); c.Files != 0 { t.Errorf("Expected 0 global files, got %v", c.Files) } } @@ -873,7 +888,9 @@ func TestWithHaveSequence(t *testing.T) { replace(s, protocol.LocalDeviceID, localHave) i := 2 - s.WithHaveSequence(int64(i), func(fi db.FileIntf) bool { + snap := s.Snapshot() + defer snap.Release() + snap.WithHaveSequence(int64(i), func(fi db.FileIntf) bool { if f := fi.(protocol.FileInfo); !f.IsEquivalent(localHave[i-1], 0) { t.Fatalf("Got %v\nExpected %v", f, localHave[i-1]) } @@ -922,13 +939,15 @@ loop: break loop default: } - s.WithHaveSequence(prevSeq+1, func(fi db.FileIntf) bool { + snap := s.Snapshot() + snap.WithHaveSequence(prevSeq+1, func(fi db.FileIntf) bool { if fi.SequenceNo() < prevSeq+1 { t.Fatal("Skipped ", prevSeq+1, fi.SequenceNo()) } prevSeq = fi.SequenceNo() return true }) + snap.Release() } } @@ -981,12 +1000,12 @@ func TestMoveGlobalBack(t *testing.T) { t.Error("Expected no need for remote 0, got", need) } - ls := s.LocalSize() + ls := localSize(s) if haveBytes := localHave[0].Size; ls.Bytes != haveBytes { t.Errorf("Incorrect LocalSize bytes; %d != %d", ls.Bytes, haveBytes) } - gs := s.GlobalSize() + gs := globalSize(s) if globalBytes := remote0Have[0].Size; gs.Bytes != globalBytes { t.Errorf("Incorrect GlobalSize bytes; %d != %d", gs.Bytes, globalBytes) } @@ -1007,12 +1026,12 @@ func TestMoveGlobalBack(t *testing.T) { t.Error("Expected no local need, got", need) } - ls = s.LocalSize() + ls = localSize(s) if haveBytes := localHave[0].Size; ls.Bytes != haveBytes { t.Errorf("Incorrect LocalSize bytes; %d != %d", ls.Bytes, haveBytes) } - gs = s.GlobalSize() + gs = globalSize(s) if globalBytes := localHave[0].Size; gs.Bytes != globalBytes { t.Errorf("Incorrect GlobalSize bytes; %d != %d", gs.Bytes, globalBytes) } @@ -1106,22 +1125,22 @@ func TestReceiveOnlyAccounting(t *testing.T) { replace(s, protocol.LocalDeviceID, files) replace(s, remote, files) - if n := s.LocalSize().Files; n != 3 { + if n := localSize(s).Files; n != 3 { t.Fatal("expected 3 local files initially, not", n) } - if n := s.LocalSize().Bytes; n != 30 { + if n := localSize(s).Bytes; n != 30 { t.Fatal("expected 30 local bytes initially, not", n) } - if n := s.GlobalSize().Files; n != 3 { + if n := globalSize(s).Files; n != 3 { t.Fatal("expected 3 global files initially, not", n) } - if n := s.GlobalSize().Bytes; n != 30 { + if n := globalSize(s).Bytes; n != 30 { t.Fatal("expected 30 global bytes initially, not", n) } - if n := s.ReceiveOnlyChangedSize().Files; n != 0 { + if n := receiveOnlyChangedSize(s).Files; n != 0 { t.Fatal("expected 0 receive only changed files initially, not", n) } - if n := s.ReceiveOnlyChangedSize().Bytes; n != 0 { + if n := receiveOnlyChangedSize(s).Bytes; n != 0 { t.Fatal("expected 0 receive only changed bytes initially, not", n) } @@ -1136,22 +1155,22 @@ func TestReceiveOnlyAccounting(t *testing.T) { // Check that we see the files - if n := s.LocalSize().Files; n != 3 { + if n := localSize(s).Files; n != 3 { t.Fatal("expected 3 local files after local change, not", n) } - if n := s.LocalSize().Bytes; n != 120 { + if n := localSize(s).Bytes; n != 120 { t.Fatal("expected 120 local bytes after local change, not", n) } - if n := s.GlobalSize().Files; n != 3 { + if n := globalSize(s).Files; n != 3 { t.Fatal("expected 3 global files after local change, not", n) } - if n := s.GlobalSize().Bytes; n != 30 { + if n := globalSize(s).Bytes; n != 30 { t.Fatal("expected 30 global files after local change, not", n) } - if n := s.ReceiveOnlyChangedSize().Files; n != 1 { + if n := receiveOnlyChangedSize(s).Files; n != 1 { t.Fatal("expected 1 receive only changed file after local change, not", n) } - if n := s.ReceiveOnlyChangedSize().Bytes; n != 100 { + if n := receiveOnlyChangedSize(s).Bytes; n != 100 { t.Fatal("expected 100 receive only changed btyes after local change, not", n) } @@ -1167,22 +1186,22 @@ func TestReceiveOnlyAccounting(t *testing.T) { // Check that we see the files, same data as initially - if n := s.LocalSize().Files; n != 3 { + if n := localSize(s).Files; n != 3 { t.Fatal("expected 3 local files after revert, not", n) } - if n := s.LocalSize().Bytes; n != 30 { + if n := localSize(s).Bytes; n != 30 { t.Fatal("expected 30 local bytes after revert, not", n) } - if n := s.GlobalSize().Files; n != 3 { + if n := globalSize(s).Files; n != 3 { t.Fatal("expected 3 global files after revert, not", n) } - if n := s.GlobalSize().Bytes; n != 30 { + if n := globalSize(s).Bytes; n != 30 { t.Fatal("expected 30 global bytes after revert, not", n) } - if n := s.ReceiveOnlyChangedSize().Files; n != 0 { + if n := receiveOnlyChangedSize(s).Files; n != 0 { t.Fatal("expected 0 receive only changed files after revert, not", n) } - if n := s.ReceiveOnlyChangedSize().Bytes; n != 0 { + if n := receiveOnlyChangedSize(s).Bytes; n != 0 { t.Fatal("expected 0 receive only changed bytes after revert, not", n) } } @@ -1229,7 +1248,7 @@ func TestRemoteInvalidNotAccounted(t *testing.T) { } s.Update(remoteDevice0, files) - global := s.GlobalSize() + global := globalSize(s) if global.Files != 1 { t.Error("Expected one file in global size, not", global.Files) } @@ -1386,12 +1405,14 @@ func TestSequenceIndex(t *testing.T) { // a subset of those files if we manage to run before a complete // update has happened since our last iteration. latest = latest[:0] - s.WithHaveSequence(seq+1, func(f db.FileIntf) bool { + snap := s.Snapshot() + snap.WithHaveSequence(seq+1, func(f db.FileIntf) bool { seen[f.FileName()] = f latest = append(latest, f) seq = f.SequenceNo() return true }) + snap.Release() // Calculate the spread in sequence number. var max, min int64 @@ -1449,7 +1470,9 @@ func TestIgnoreAfterReceiveOnly(t *testing.T) { s.Update(protocol.LocalDeviceID, fs) - if f, ok := s.Get(protocol.LocalDeviceID, file); !ok { + snap := s.Snapshot() + defer snap.Release() + if f, ok := snap.Get(protocol.LocalDeviceID, file); !ok { t.Error("File missing in db") } else if f.IsReceiveOnlyChanged() { t.Error("File is still receive-only changed") @@ -1462,3 +1485,21 @@ func replace(fs *db.FileSet, device protocol.DeviceID, files []protocol.FileInfo fs.Drop(device) fs.Update(device, files) } + +func localSize(fs *db.FileSet) db.Counts { + snap := fs.Snapshot() + defer snap.Release() + return snap.LocalSize() +} + +func globalSize(fs *db.FileSet) db.Counts { + snap := fs.Snapshot() + defer snap.Release() + return snap.GlobalSize() +} + +func receiveOnlyChangedSize(fs *db.FileSet) db.Counts { + snap := fs.Snapshot() + defer snap.Release() + return snap.ReceiveOnlyChangedSize() +} diff --git a/lib/db/transactions.go b/lib/db/transactions.go index 132193d4f..c0fd528c0 100644 --- a/lib/db/transactions.go +++ b/lib/db/transactions.go @@ -7,6 +7,8 @@ package db import ( + "bytes" + "github.com/syncthing/syncthing/lib/db/backend" "github.com/syncthing/syncthing/lib/protocol" ) @@ -94,6 +96,291 @@ func (t readOnlyTransaction) getGlobal(keyBuf, folder, file []byte, truncate boo return keyBuf, fi, true, nil } +func (t *readOnlyTransaction) withHave(folder, device, prefix []byte, truncate bool, fn Iterator) error { + if len(prefix) > 0 { + unslashedPrefix := prefix + if bytes.HasSuffix(prefix, []byte{'/'}) { + unslashedPrefix = unslashedPrefix[:len(unslashedPrefix)-1] + } else { + prefix = append(prefix, '/') + } + + key, err := t.keyer.GenerateDeviceFileKey(nil, folder, device, unslashedPrefix) + if err != nil { + return err + } + if f, ok, err := t.getFileTrunc(key, true); err != nil { + return err + } else if ok && !fn(f) { + return nil + } + } + + key, err := t.keyer.GenerateDeviceFileKey(nil, folder, device, prefix) + if err != nil { + return err + } + dbi, err := t.NewPrefixIterator(key) + if err != nil { + return err + } + defer dbi.Release() + + for dbi.Next() { + name := t.keyer.NameFromDeviceFileKey(dbi.Key()) + if len(prefix) > 0 && !bytes.HasPrefix(name, prefix) { + return nil + } + + f, err := unmarshalTrunc(dbi.Value(), truncate) + if err != nil { + l.Debugln("unmarshal error:", err) + continue + } + if !fn(f) { + return nil + } + } + return dbi.Error() +} + +func (t *readOnlyTransaction) withHaveSequence(folder []byte, startSeq int64, fn Iterator) error { + first, err := t.keyer.GenerateSequenceKey(nil, folder, startSeq) + if err != nil { + return err + } + last, err := t.keyer.GenerateSequenceKey(nil, folder, maxInt64) + if err != nil { + return err + } + dbi, err := t.NewRangeIterator(first, last) + if err != nil { + return err + } + defer dbi.Release() + + for dbi.Next() { + f, ok, err := t.getFileByKey(dbi.Value()) + if err != nil { + return err + } + if !ok { + l.Debugln("missing file for sequence number", t.keyer.SequenceFromSequenceKey(dbi.Key())) + continue + } + + if shouldDebug() { + if seq := t.keyer.SequenceFromSequenceKey(dbi.Key()); f.Sequence != seq { + l.Warnf("Sequence index corruption (folder %v, file %v): sequence %d != expected %d", string(folder), f.Name, f.Sequence, seq) + panic("sequence index corruption") + } + } + if !fn(f) { + return nil + } + } + return dbi.Error() +} + +func (t *readOnlyTransaction) withGlobal(folder, prefix []byte, truncate bool, fn Iterator) error { + if len(prefix) > 0 { + unslashedPrefix := prefix + if bytes.HasSuffix(prefix, []byte{'/'}) { + unslashedPrefix = unslashedPrefix[:len(unslashedPrefix)-1] + } else { + prefix = append(prefix, '/') + } + + if _, f, ok, err := t.getGlobal(nil, folder, unslashedPrefix, truncate); err != nil { + return err + } else if ok && !fn(f) { + return nil + } + } + + key, err := t.keyer.GenerateGlobalVersionKey(nil, folder, prefix) + if err != nil { + return err + } + dbi, err := t.NewPrefixIterator(key) + if err != nil { + return err + } + defer dbi.Release() + + var dk []byte + for dbi.Next() { + name := t.keyer.NameFromGlobalVersionKey(dbi.Key()) + if len(prefix) > 0 && !bytes.HasPrefix(name, prefix) { + return nil + } + + vl, ok := unmarshalVersionList(dbi.Value()) + if !ok { + continue + } + + dk, err = t.keyer.GenerateDeviceFileKey(dk, folder, vl.Versions[0].Device, name) + if err != nil { + return err + } + + f, ok, err := t.getFileTrunc(dk, truncate) + if err != nil { + return err + } + if !ok { + continue + } + + if !fn(f) { + return nil + } + } + if err != nil { + return err + } + return dbi.Error() +} + +func (t *readOnlyTransaction) availability(folder, file []byte) ([]protocol.DeviceID, error) { + k, err := t.keyer.GenerateGlobalVersionKey(nil, folder, file) + if err != nil { + return nil, err + } + bs, err := t.Get(k) + if backend.IsNotFound(err) { + return nil, nil + } + if err != nil { + return nil, err + } + + vl, ok := unmarshalVersionList(bs) + if !ok { + return nil, nil + } + + var devices []protocol.DeviceID + for _, v := range vl.Versions { + if !v.Version.Equal(vl.Versions[0].Version) { + break + } + if v.Invalid { + continue + } + n := protocol.DeviceIDFromBytes(v.Device) + devices = append(devices, n) + } + + return devices, nil +} + +func (t *readOnlyTransaction) withNeed(folder, device []byte, truncate bool, fn Iterator) error { + if bytes.Equal(device, protocol.LocalDeviceID[:]) { + return t.withNeedLocal(folder, truncate, fn) + } + + key, err := t.keyer.GenerateGlobalVersionKey(nil, folder, nil) + if err != nil { + return err + } + dbi, err := t.NewPrefixIterator(key.WithoutName()) + if err != nil { + return err + } + defer dbi.Release() + + var dk []byte + devID := protocol.DeviceIDFromBytes(device) + for dbi.Next() { + vl, ok := unmarshalVersionList(dbi.Value()) + if !ok { + continue + } + + haveFV, have := vl.Get(device) + // XXX: This marks Concurrent (i.e. conflicting) changes as + // needs. Maybe we should do that, but it needs special + // handling in the puller. + if have && haveFV.Version.GreaterEqual(vl.Versions[0].Version) { + continue + } + + name := t.keyer.NameFromGlobalVersionKey(dbi.Key()) + needVersion := vl.Versions[0].Version + needDevice := protocol.DeviceIDFromBytes(vl.Versions[0].Device) + + for i := range vl.Versions { + if !vl.Versions[i].Version.Equal(needVersion) { + // We haven't found a valid copy of the file with the needed version. + break + } + + if vl.Versions[i].Invalid { + // The file is marked invalid, don't use it. + continue + } + + dk, err = t.keyer.GenerateDeviceFileKey(dk, folder, vl.Versions[i].Device, name) + if err != nil { + return err + } + gf, ok, err := t.getFileTrunc(dk, truncate) + if err != nil { + return err + } + if !ok { + continue + } + + if gf.IsDeleted() && !have { + // We don't need deleted files that we don't have + break + } + + l.Debugf("need folder=%q device=%v name=%q have=%v invalid=%v haveV=%v globalV=%v globalDev=%v", folder, devID, name, have, haveFV.Invalid, haveFV.Version, needVersion, needDevice) + + if !fn(gf) { + return nil + } + + // This file is handled, no need to look further in the version list + break + } + } + return dbi.Error() +} + +func (t *readOnlyTransaction) withNeedLocal(folder []byte, truncate bool, fn Iterator) error { + key, err := t.keyer.GenerateNeedFileKey(nil, folder, nil) + if err != nil { + return err + } + dbi, err := t.NewPrefixIterator(key.WithoutName()) + if err != nil { + return err + } + defer dbi.Release() + + var keyBuf []byte + var f FileIntf + var ok bool + for dbi.Next() { + keyBuf, f, ok, err = t.getGlobal(keyBuf, folder, t.keyer.NameFromGlobalVersionKey(dbi.Key()), truncate) + if err != nil { + return err + } + if !ok { + continue + } + if !fn(f) { + return nil + } + } + return dbi.Error() +} + // A readWriteTransaction is a readOnlyTransaction plus a batch for writes. // The batch will be committed on close() or by checkFlush() if it exceeds the // batch size. @@ -353,6 +640,65 @@ func (t readWriteTransaction) deleteKeyPrefix(prefix []byte) error { return dbi.Error() } +func (t *readWriteTransaction) withAllFolderTruncated(folder []byte, fn func(device []byte, f FileInfoTruncated) bool) error { + key, err := t.keyer.GenerateDeviceFileKey(nil, folder, nil, nil) + if err != nil { + return err + } + dbi, err := t.NewPrefixIterator(key.WithoutNameAndDevice()) + if err != nil { + return err + } + defer dbi.Release() + + var gk, keyBuf []byte + for dbi.Next() { + device, ok := t.keyer.DeviceFromDeviceFileKey(dbi.Key()) + if !ok { + // Not having the device in the index is bad. Clear it. + if err := t.Delete(dbi.Key()); err != nil { + return err + } + continue + } + var f FileInfoTruncated + // The iterator function may keep a reference to the unmarshalled + // struct, which in turn references the buffer it was unmarshalled + // from. dbi.Value() just returns an internal slice that it reuses, so + // we need to copy it. + err := f.Unmarshal(append([]byte{}, dbi.Value()...)) + if err != nil { + return err + } + + switch f.Name { + case "", ".", "..", "/": // A few obviously invalid filenames + l.Infof("Dropping invalid filename %q from database", f.Name) + name := []byte(f.Name) + gk, err = t.keyer.GenerateGlobalVersionKey(gk, folder, name) + if err != nil { + return err + } + keyBuf, err = t.removeFromGlobal(gk, keyBuf, folder, device, name, nil) + if err != nil { + return err + } + if err := t.Delete(dbi.Key()); err != nil { + return err + } + continue + } + + if !fn(device, f) { + return nil + } + } + if err := dbi.Error(); err != nil { + return err + } + return t.commit() +} + type marshaller interface { Marshal() ([]byte, error) } diff --git a/lib/model/folder.go b/lib/model/folder.go index 109720955..045e55bbb 100644 --- a/lib/model/folder.go +++ b/lib/model/folder.go @@ -328,11 +328,16 @@ func (f *folder) scanSubdirs(subDirs []string) error { subDirs[i] = sub } + snap := f.fset.Snapshot() + // We release explicitly later in this function, however we might exit early + // and it's ok to release twice. + defer snap.Release() + // Clean the list of subitems to ensure that we start at a known // directory, and don't scan subdirectories of things we've already // scanned. subDirs = unifySubs(subDirs, func(file string) bool { - _, ok := f.fset.Get(protocol.LocalDeviceID, file) + _, ok := snap.Get(protocol.LocalDeviceID, file) return ok }) @@ -344,7 +349,7 @@ func (f *folder) scanSubdirs(subDirs []string) error { Subs: subDirs, Matcher: f.ignores, TempLifetime: time.Duration(f.model.cfg.Options().KeepTemporariesH) * time.Hour, - CurrentFiler: cFiler{f.fset}, + CurrentFiler: cFiler{snap}, Filesystem: mtimefs, IgnorePerms: f.IgnorePerms, AutoNormalize: f.AutoNormalize, @@ -369,7 +374,7 @@ func (f *folder) scanSubdirs(subDirs []string) error { oldBatchFn := batchFn // can't reference batchFn directly (recursion) batchFn = func(fs []protocol.FileInfo) error { for i := range fs { - switch gf, ok := f.fset.GetGlobal(fs[i].Name); { + switch gf, ok := snap.GetGlobal(fs[i].Name); { case !ok: continue case gf.IsEquivalentOptional(fs[i], f.ModTimeWindow(), false, false, protocol.FlagLocalReceiveOnly): @@ -426,10 +431,15 @@ func (f *folder) scanSubdirs(subDirs []string) error { // ignored files. var toIgnore []db.FileInfoTruncated ignoredParent := "" + + snap.Release() + snap = f.fset.Snapshot() + defer snap.Release() + for _, sub := range subDirs { var iterError error - f.fset.WithPrefixedHaveTruncated(protocol.LocalDeviceID, sub, func(fi db.FileIntf) bool { + snap.WithPrefixedHaveTruncated(protocol.LocalDeviceID, sub, func(fi db.FileIntf) bool { select { case <-f.ctx.Done(): return false @@ -891,7 +901,7 @@ func unifySubs(dirs []string, exists func(dir string) bool) []string { } type cFiler struct { - *db.FileSet + *db.Snapshot } // Implements scanner.CurrentFiler diff --git a/lib/model/folder_recvonly.go b/lib/model/folder_recvonly.go index 7899e789b..1bc40f980 100644 --- a/lib/model/folder_recvonly.go +++ b/lib/model/folder_recvonly.go @@ -79,7 +79,9 @@ func (f *receiveOnlyFolder) Revert() { batch := make([]protocol.FileInfo, 0, maxBatchSizeFiles) batchSizeBytes := 0 - f.fset.WithHave(protocol.LocalDeviceID, func(intf db.FileIntf) bool { + snap := f.fset.Snapshot() + defer snap.Release() + snap.WithHave(protocol.LocalDeviceID, func(intf db.FileIntf) bool { fi := intf.(protocol.FileInfo) if !fi.IsReceiveOnlyChanged() { // We're only interested in files that have changed locally in @@ -93,7 +95,7 @@ func (f *receiveOnlyFolder) Revert() { // We'll delete files directly, directories get queued and // handled below. - handled, err := delQueue.handle(fi) + handled, err := delQueue.handle(fi, snap) if err != nil { l.Infof("Revert: deleting %s: %v\n", fi.Name, err) return true // continue @@ -138,7 +140,7 @@ func (f *receiveOnlyFolder) Revert() { batchSizeBytes = 0 // Handle any queued directories - deleted, err := delQueue.flush() + deleted, err := delQueue.flush(snap) if err != nil { l.Infoln("Revert:", err) } @@ -167,15 +169,15 @@ func (f *receiveOnlyFolder) Revert() { // directories for last. type deleteQueue struct { handler interface { - deleteItemOnDisk(item protocol.FileInfo, scanChan chan<- string) error - deleteDirOnDisk(dir string, scanChan chan<- string) error + deleteItemOnDisk(item protocol.FileInfo, snap *db.Snapshot, scanChan chan<- string) error + deleteDirOnDisk(dir string, snap *db.Snapshot, scanChan chan<- string) error } ignores *ignore.Matcher dirs []string scanChan chan<- string } -func (q *deleteQueue) handle(fi protocol.FileInfo) (bool, error) { +func (q *deleteQueue) handle(fi protocol.FileInfo, snap *db.Snapshot) (bool, error) { // Things that are ignored but not marked deletable are not processed. ign := q.ignores.Match(fi.Name) if ign.IsIgnored() && !ign.IsDeletable() { @@ -189,11 +191,11 @@ func (q *deleteQueue) handle(fi protocol.FileInfo) (bool, error) { } // Kill it. - err := q.handler.deleteItemOnDisk(fi, q.scanChan) + err := q.handler.deleteItemOnDisk(fi, snap, q.scanChan) return true, err } -func (q *deleteQueue) flush() ([]string, error) { +func (q *deleteQueue) flush(snap *db.Snapshot) ([]string, error) { // Process directories from the leaves inward. sort.Sort(sort.Reverse(sort.StringSlice(q.dirs))) @@ -201,7 +203,7 @@ func (q *deleteQueue) flush() ([]string, error) { var deleted []string for _, dir := range q.dirs { - if err := q.handler.deleteDirOnDisk(dir, q.scanChan); err == nil { + if err := q.handler.deleteDirOnDisk(dir, snap, q.scanChan); err == nil { deleted = append(deleted, dir) } else if err != nil && firstError == nil { firstError = err diff --git a/lib/model/folder_recvonly_test.go b/lib/model/folder_recvonly_test.go index 3ef140608..e2eb00726 100644 --- a/lib/model/folder_recvonly_test.go +++ b/lib/model/folder_recvonly_test.go @@ -48,7 +48,7 @@ func TestRecvOnlyRevertDeletes(t *testing.T) { m.Index(device1, "ro", knownFiles) f.updateLocalsFromScanning(knownFiles) - size := m.GlobalSize("ro") + size := globalSize(t, m, "ro") if size.Files != 1 || size.Directories != 1 { t.Fatalf("Global: expected 1 file and 1 directory: %+v", size) } @@ -60,15 +60,15 @@ func TestRecvOnlyRevertDeletes(t *testing.T) { // We should now have two files and two directories. - size = m.GlobalSize("ro") + size = globalSize(t, m, "ro") if size.Files != 2 || size.Directories != 2 { t.Fatalf("Global: expected 2 files and 2 directories: %+v", size) } - size = m.LocalSize("ro") + size = localSize(t, m, "ro") if size.Files != 2 || size.Directories != 2 { t.Fatalf("Local: expected 2 files and 2 directories: %+v", size) } - size = m.ReceiveOnlyChangedSize("ro") + size = receiveOnlyChangedSize(t, m, "ro") if size.Files+size.Directories == 0 { t.Fatalf("ROChanged: expected something: %+v", size) } @@ -93,11 +93,11 @@ func TestRecvOnlyRevertDeletes(t *testing.T) { // We should now have one file and directory again. - size = m.GlobalSize("ro") + size = globalSize(t, m, "ro") if size.Files != 1 || size.Directories != 1 { t.Fatalf("Global: expected 1 files and 1 directories: %+v", size) } - size = m.LocalSize("ro") + size = localSize(t, m, "ro") if size.Files != 1 || size.Directories != 1 { t.Fatalf("Local: expected 1 files and 1 directories: %+v", size) } @@ -131,19 +131,19 @@ func TestRecvOnlyRevertNeeds(t *testing.T) { // Everything should be in sync. - size := m.GlobalSize("ro") + size := globalSize(t, m, "ro") if size.Files != 1 || size.Directories != 1 { t.Fatalf("Global: expected 1 file and 1 directory: %+v", size) } - size = m.LocalSize("ro") + size = localSize(t, m, "ro") if size.Files != 1 || size.Directories != 1 { t.Fatalf("Local: expected 1 file and 1 directory: %+v", size) } - size = m.NeedSize("ro") + size = needSize(t, m, "ro") if size.Files+size.Directories > 0 { t.Fatalf("Need: expected nothing: %+v", size) } - size = m.ReceiveOnlyChangedSize("ro") + size = receiveOnlyChangedSize(t, m, "ro") if size.Files+size.Directories > 0 { t.Fatalf("ROChanged: expected nothing: %+v", size) } @@ -159,20 +159,20 @@ func TestRecvOnlyRevertNeeds(t *testing.T) { // We now have a newer file than the rest of the cluster. Global state should reflect this. - size = m.GlobalSize("ro") + size = globalSize(t, m, "ro") const sizeOfDir = 128 if size.Files != 1 || size.Bytes != sizeOfDir+int64(len(oldData)) { t.Fatalf("Global: expected no change due to the new file: %+v", size) } - size = m.LocalSize("ro") + size = localSize(t, m, "ro") if size.Files != 1 || size.Bytes != sizeOfDir+int64(len(newData)) { t.Fatalf("Local: expected the new file to be reflected: %+v", size) } - size = m.NeedSize("ro") + size = needSize(t, m, "ro") if size.Files+size.Directories > 0 { t.Fatalf("Need: expected nothing: %+v", size) } - size = m.ReceiveOnlyChangedSize("ro") + size = receiveOnlyChangedSize(t, m, "ro") if size.Files+size.Directories == 0 { t.Fatalf("ROChanged: expected something: %+v", size) } @@ -181,15 +181,15 @@ func TestRecvOnlyRevertNeeds(t *testing.T) { m.Revert("ro") - size = m.GlobalSize("ro") + size = globalSize(t, m, "ro") if size.Files != 1 || size.Bytes != sizeOfDir+int64(len(oldData)) { t.Fatalf("Global: expected the global size to revert: %+v", size) } - size = m.LocalSize("ro") + size = localSize(t, m, "ro") if size.Files != 1 || size.Bytes != sizeOfDir+int64(len(newData)) { t.Fatalf("Local: expected the local size to remain: %+v", size) } - size = m.NeedSize("ro") + size = needSize(t, m, "ro") if size.Files != 1 || size.Bytes != int64(len(oldData)) { t.Fatalf("Local: expected to need the old file data: %+v", size) } @@ -227,19 +227,19 @@ func TestRecvOnlyUndoChanges(t *testing.T) { // Everything should be in sync. - size := m.GlobalSize("ro") + size := globalSize(t, m, "ro") if size.Files != 1 || size.Directories != 1 { t.Fatalf("Global: expected 1 file and 1 directory: %+v", size) } - size = m.LocalSize("ro") + size = localSize(t, m, "ro") if size.Files != 1 || size.Directories != 1 { t.Fatalf("Local: expected 1 file and 1 directory: %+v", size) } - size = m.NeedSize("ro") + size = needSize(t, m, "ro") if size.Files+size.Directories > 0 { t.Fatalf("Need: expected nothing: %+v", size) } - size = m.ReceiveOnlyChangedSize("ro") + size = receiveOnlyChangedSize(t, m, "ro") if size.Files+size.Directories > 0 { t.Fatalf("ROChanged: expected nothing: %+v", size) } @@ -253,7 +253,7 @@ func TestRecvOnlyUndoChanges(t *testing.T) { m.ScanFolder("ro") - size = m.ReceiveOnlyChangedSize("ro") + size = receiveOnlyChangedSize(t, m, "ro") if size.Files != 2 { t.Fatalf("Receive only: expected 2 files: %+v", size) } @@ -266,7 +266,7 @@ func TestRecvOnlyUndoChanges(t *testing.T) { m.ScanFolder("ro") - size = m.ReceiveOnlyChangedSize("ro") + size = receiveOnlyChangedSize(t, m, "ro") if size.Files+size.Directories+size.Deleted != 0 { t.Fatalf("Receive only: expected all zero: %+v", size) } diff --git a/lib/model/folder_sendonly.go b/lib/model/folder_sendonly.go index f56ce16a6..1696217ae 100644 --- a/lib/model/folder_sendonly.go +++ b/lib/model/folder_sendonly.go @@ -50,7 +50,9 @@ func (f *sendOnlyFolder) pull() bool { batch := make([]protocol.FileInfo, 0, maxBatchSizeFiles) batchSizeBytes := 0 - f.fset.WithNeed(protocol.LocalDeviceID, func(intf db.FileIntf) bool { + snap := f.fset.Snapshot() + defer snap.Release() + snap.WithNeed(protocol.LocalDeviceID, func(intf db.FileIntf) bool { if len(batch) == maxBatchSizeFiles || batchSizeBytes > maxBatchSizeBytes { f.updateLocalsFromPulling(batch) batch = batch[:0] @@ -66,7 +68,7 @@ func (f *sendOnlyFolder) pull() bool { return true } - curFile, ok := f.fset.Get(protocol.LocalDeviceID, intf.FileName()) + curFile, ok := snap.Get(protocol.LocalDeviceID, intf.FileName()) if !ok { if intf.IsDeleted() { l.Debugln("Should never get a deleted file as needed when we don't have it") @@ -98,7 +100,9 @@ func (f *sendOnlyFolder) Override() { f.setState(FolderScanning) batch := make([]protocol.FileInfo, 0, maxBatchSizeFiles) batchSizeBytes := 0 - f.fset.WithNeed(protocol.LocalDeviceID, func(fi db.FileIntf) bool { + snap := f.fset.Snapshot() + defer snap.Release() + snap.WithNeed(protocol.LocalDeviceID, func(fi db.FileIntf) bool { need := fi.(protocol.FileInfo) if len(batch) == maxBatchSizeFiles || batchSizeBytes > maxBatchSizeBytes { f.updateLocalsFromScanning(batch) @@ -106,7 +110,7 @@ func (f *sendOnlyFolder) Override() { batchSizeBytes = 0 } - have, ok := f.fset.Get(protocol.LocalDeviceID, need.Name) + have, ok := snap.Get(protocol.LocalDeviceID, need.Name) // Don't override files that are in a bad state (ignored, // unsupported, must rescan, ...). if ok && have.IsInvalid() { diff --git a/lib/model/folder_sendrecv.go b/lib/model/folder_sendrecv.go index 506efc9aa..7f1d60465 100644 --- a/lib/model/folder_sendrecv.go +++ b/lib/model/folder_sendrecv.go @@ -149,10 +149,12 @@ func (f *sendReceiveFolder) pull() bool { // If there is nothing to do, don't even enter pulling state. abort := true - f.fset.WithNeed(protocol.LocalDeviceID, func(intf db.FileIntf) bool { + snap := f.fset.Snapshot() + snap.WithNeed(protocol.LocalDeviceID, func(intf db.FileIntf) bool { abort = false return false }) + snap.Release() if abort { return true } @@ -234,6 +236,9 @@ func (f *sendReceiveFolder) pullerIteration(scanChan chan<- string) int { f.pullErrors = make(map[string]string) f.pullErrorsMut.Unlock() + snap := f.fset.Snapshot() + defer snap.Release() + pullChan := make(chan pullBlockState) copyChan := make(chan copyBlocksState) finisherChan := make(chan *sharedPullerState) @@ -272,11 +277,11 @@ func (f *sendReceiveFolder) pullerIteration(scanChan chan<- string) int { doneWg.Add(1) // finisherRoutine finishes when finisherChan is closed go func() { - f.finisherRoutine(finisherChan, dbUpdateChan, scanChan) + f.finisherRoutine(snap, finisherChan, dbUpdateChan, scanChan) doneWg.Done() }() - changed, fileDeletions, dirDeletions, err := f.processNeeded(dbUpdateChan, copyChan, scanChan) + changed, fileDeletions, dirDeletions, err := f.processNeeded(snap, dbUpdateChan, copyChan, scanChan) // Signal copy and puller routines that we are done with the in data for // this iteration. Wait for them to finish. @@ -291,7 +296,7 @@ func (f *sendReceiveFolder) pullerIteration(scanChan chan<- string) int { doneWg.Wait() if err == nil { - f.processDeletions(fileDeletions, dirDeletions, dbUpdateChan, scanChan) + f.processDeletions(fileDeletions, dirDeletions, snap, dbUpdateChan, scanChan) } // Wait for db updates and scan scheduling to complete @@ -307,7 +312,7 @@ func (f *sendReceiveFolder) pullerIteration(scanChan chan<- string) int { return changed } -func (f *sendReceiveFolder) processNeeded(dbUpdateChan chan<- dbUpdateJob, copyChan chan<- copyBlocksState, scanChan chan<- string) (int, map[string]protocol.FileInfo, []protocol.FileInfo, error) { +func (f *sendReceiveFolder) processNeeded(snap *db.Snapshot, dbUpdateChan chan<- dbUpdateJob, copyChan chan<- copyBlocksState, scanChan chan<- string) (int, map[string]protocol.FileInfo, []protocol.FileInfo, error) { changed := 0 var dirDeletions []protocol.FileInfo fileDeletions := map[string]protocol.FileInfo{} @@ -317,7 +322,7 @@ func (f *sendReceiveFolder) processNeeded(dbUpdateChan chan<- dbUpdateJob, copyC // Regular files to pull goes into the file queue, everything else // (directories, symlinks and deletes) goes into the "process directly" // pile. - f.fset.WithNeed(protocol.LocalDeviceID, func(intf db.FileIntf) bool { + snap.WithNeed(protocol.LocalDeviceID, func(intf db.FileIntf) bool { select { case <-f.ctx.Done(): return false @@ -359,9 +364,9 @@ func (f *sendReceiveFolder) processNeeded(dbUpdateChan chan<- dbUpdateJob, copyC // files to delete inside them before we get to that point. dirDeletions = append(dirDeletions, file) } else if file.IsSymlink() { - f.deleteFile(file, dbUpdateChan, scanChan) + f.deleteFile(file, snap, dbUpdateChan, scanChan) } else { - df, ok := f.fset.Get(protocol.LocalDeviceID, file.Name) + df, ok := snap.Get(protocol.LocalDeviceID, file.Name) // Local file can be already deleted, but with a lower version // number, hence the deletion coming in again as part of // WithNeed, furthermore, the file can simply be of the wrong @@ -377,7 +382,7 @@ func (f *sendReceiveFolder) processNeeded(dbUpdateChan chan<- dbUpdateJob, copyC } case file.Type == protocol.FileInfoTypeFile: - curFile, hasCurFile := f.fset.Get(protocol.LocalDeviceID, file.Name) + curFile, hasCurFile := snap.Get(protocol.LocalDeviceID, file.Name) if _, need := blockDiff(curFile.Blocks, file.Blocks); hasCurFile && len(need) == 0 { // We are supposed to copy the entire file, and then fetch nothing. We // are only updating metadata, so we don't actually *need* to make the @@ -396,13 +401,13 @@ func (f *sendReceiveFolder) processNeeded(dbUpdateChan chan<- dbUpdateJob, copyC case file.IsDirectory() && !file.IsSymlink(): l.Debugln(f, "Handling directory", file.Name) if f.checkParent(file.Name, scanChan) { - f.handleDir(file, dbUpdateChan, scanChan) + f.handleDir(file, snap, dbUpdateChan, scanChan) } case file.IsSymlink(): l.Debugln(f, "Handling symlink", file.Name) if f.checkParent(file.Name, scanChan) { - f.handleSymlink(file, dbUpdateChan, scanChan) + f.handleSymlink(file, snap, dbUpdateChan, scanChan) } default: @@ -451,7 +456,7 @@ nextFile: break } - fi, ok := f.fset.GetGlobal(fileName) + fi, ok := snap.GetGlobal(fileName) if !ok { // File is no longer in the index. Mark it as done and drop it. f.queue.Done(fileName) @@ -484,7 +489,7 @@ nextFile: // desired state with the delete bit set is in the deletion // map. desired := fileDeletions[candidate.Name] - if err := f.renameFile(candidate, desired, fi, dbUpdateChan, scanChan); err != nil { + if err := f.renameFile(candidate, desired, fi, snap, dbUpdateChan, scanChan); err != nil { // Failed to rename, try to handle files as separate // deletions and updates. break @@ -498,11 +503,11 @@ nextFile: } } - devices := f.fset.Availability(fileName) + devices := snap.Availability(fileName) for _, dev := range devices { if _, ok := f.model.Connection(dev); ok { // Handle the file normally, by coping and pulling, etc. - f.handleFile(fi, copyChan, dbUpdateChan) + f.handleFile(fi, snap, copyChan) continue nextFile } } @@ -513,7 +518,7 @@ nextFile: return changed, fileDeletions, dirDeletions, nil } -func (f *sendReceiveFolder) processDeletions(fileDeletions map[string]protocol.FileInfo, dirDeletions []protocol.FileInfo, dbUpdateChan chan<- dbUpdateJob, scanChan chan<- string) { +func (f *sendReceiveFolder) processDeletions(fileDeletions map[string]protocol.FileInfo, dirDeletions []protocol.FileInfo, snap *db.Snapshot, dbUpdateChan chan<- dbUpdateJob, scanChan chan<- string) { for _, file := range fileDeletions { select { case <-f.ctx.Done(): @@ -521,7 +526,7 @@ func (f *sendReceiveFolder) processDeletions(fileDeletions map[string]protocol.F default: } - f.deleteFile(file, dbUpdateChan, scanChan) + f.deleteFile(file, snap, dbUpdateChan, scanChan) } // Process in reverse order to delete depth first @@ -534,12 +539,12 @@ func (f *sendReceiveFolder) processDeletions(fileDeletions map[string]protocol.F dir := dirDeletions[len(dirDeletions)-i-1] l.Debugln(f, "Deleting dir", dir.Name) - f.deleteDir(dir, dbUpdateChan, scanChan) + f.deleteDir(dir, snap, dbUpdateChan, scanChan) } } // handleDir creates or updates the given directory -func (f *sendReceiveFolder) handleDir(file protocol.FileInfo, dbUpdateChan chan<- dbUpdateJob, scanChan chan<- string) { +func (f *sendReceiveFolder) handleDir(file protocol.FileInfo, snap *db.Snapshot, dbUpdateChan chan<- dbUpdateJob, scanChan chan<- string) { // Used in the defer closure below, updated by the function body. Take // care not declare another err. var err error @@ -567,7 +572,7 @@ func (f *sendReceiveFolder) handleDir(file protocol.FileInfo, dbUpdateChan chan< } if shouldDebug() { - curFile, _ := f.fset.Get(protocol.LocalDeviceID, file.Name) + curFile, _ := snap.Get(protocol.LocalDeviceID, file.Name) l.Debugf("need dir\n\t%v\n\t%v", file, curFile) } @@ -598,7 +603,7 @@ func (f *sendReceiveFolder) handleDir(file protocol.FileInfo, dbUpdateChan chan< return f.moveForConflict(name, file.ModifiedBy.String(), scanChan) }, curFile.Name) } else { - err = f.deleteItemOnDisk(curFile, scanChan) + err = f.deleteItemOnDisk(curFile, snap, scanChan) } if err != nil { f.newPullError(file.Name, err) @@ -693,7 +698,7 @@ func (f *sendReceiveFolder) checkParent(file string, scanChan chan<- string) boo } // handleSymlink creates or updates the given symlink -func (f *sendReceiveFolder) handleSymlink(file protocol.FileInfo, dbUpdateChan chan<- dbUpdateJob, scanChan chan<- string) { +func (f *sendReceiveFolder) handleSymlink(file protocol.FileInfo, snap *db.Snapshot, dbUpdateChan chan<- dbUpdateJob, scanChan chan<- string) { // Used in the defer closure below, updated by the function body. Take // care not declare another err. var err error @@ -716,7 +721,7 @@ func (f *sendReceiveFolder) handleSymlink(file protocol.FileInfo, dbUpdateChan c }() if shouldDebug() { - curFile, _ := f.fset.Get(protocol.LocalDeviceID, file.Name) + curFile, _ := snap.Get(protocol.LocalDeviceID, file.Name) l.Debugf("need symlink\n\t%v\n\t%v", file, curFile) } @@ -750,7 +755,7 @@ func (f *sendReceiveFolder) handleSymlink(file protocol.FileInfo, dbUpdateChan c return f.moveForConflict(name, file.ModifiedBy.String(), scanChan) }, curFile.Name) } else { - err = f.deleteItemOnDisk(curFile, scanChan) + err = f.deleteItemOnDisk(curFile, snap, scanChan) } if err != nil { f.newPullError(file.Name, errors.Wrap(err, "symlink remove")) @@ -775,7 +780,7 @@ func (f *sendReceiveFolder) handleSymlink(file protocol.FileInfo, dbUpdateChan c } // deleteDir attempts to remove a directory that was deleted on a remote -func (f *sendReceiveFolder) deleteDir(file protocol.FileInfo, dbUpdateChan chan<- dbUpdateJob, scanChan chan<- string) { +func (f *sendReceiveFolder) deleteDir(file protocol.FileInfo, snap *db.Snapshot, dbUpdateChan chan<- dbUpdateJob, scanChan chan<- string) { // Used in the defer closure below, updated by the function body. Take // care not declare another err. var err error @@ -797,7 +802,7 @@ func (f *sendReceiveFolder) deleteDir(file protocol.FileInfo, dbUpdateChan chan< }) }() - if err = f.deleteDirOnDisk(file.Name, scanChan); err != nil { + if err = f.deleteDirOnDisk(file.Name, snap, scanChan); err != nil { f.newPullError(file.Name, errors.Wrap(err, "delete dir")) return } @@ -806,8 +811,8 @@ func (f *sendReceiveFolder) deleteDir(file protocol.FileInfo, dbUpdateChan chan< } // deleteFile attempts to delete the given file -func (f *sendReceiveFolder) deleteFile(file protocol.FileInfo, dbUpdateChan chan<- dbUpdateJob, scanChan chan<- string) { - cur, hasCur := f.fset.Get(protocol.LocalDeviceID, file.Name) +func (f *sendReceiveFolder) deleteFile(file protocol.FileInfo, snap *db.Snapshot, dbUpdateChan chan<- dbUpdateJob, scanChan chan<- string) { + cur, hasCur := snap.Get(protocol.LocalDeviceID, file.Name) f.deleteFileWithCurrent(file, cur, hasCur, dbUpdateChan, scanChan) } @@ -895,7 +900,7 @@ func (f *sendReceiveFolder) deleteFileWithCurrent(file, cur protocol.FileInfo, h // renameFile attempts to rename an existing file to a destination // and set the right attributes on it. -func (f *sendReceiveFolder) renameFile(cur, source, target protocol.FileInfo, dbUpdateChan chan<- dbUpdateJob, scanChan chan<- string) error { +func (f *sendReceiveFolder) renameFile(cur, source, target protocol.FileInfo, snap *db.Snapshot, dbUpdateChan chan<- dbUpdateJob, scanChan chan<- string) error { // Used in the defer closure below, updated by the function body. Take // care not declare another err. var err error @@ -937,7 +942,7 @@ func (f *sendReceiveFolder) renameFile(cur, source, target protocol.FileInfo, db return err } // Check that the target corresponds to what we have in the DB - curTarget, ok := f.fset.Get(protocol.LocalDeviceID, target.Name) + curTarget, ok := snap.Get(protocol.LocalDeviceID, target.Name) switch stat, serr := f.fs.Lstat(target.Name); { case serr != nil && fs.IsNotExist(serr): if !ok || curTarget.IsDeleted() { @@ -994,7 +999,7 @@ func (f *sendReceiveFolder) renameFile(cur, source, target protocol.FileInfo, db // of the source and the creation of the target temp file. Fix-up the metadata, // update the local index of the target file and rename from temp to real name. - if err = f.performFinish(target, curTarget, true, tempName, dbUpdateChan, scanChan); err != nil { + if err = f.performFinish(target, curTarget, true, tempName, snap, dbUpdateChan, scanChan); err != nil { return err } @@ -1039,8 +1044,8 @@ func (f *sendReceiveFolder) renameFile(cur, source, target protocol.FileInfo, db // handleFile queues the copies and pulls as necessary for a single new or // changed file. -func (f *sendReceiveFolder) handleFile(file protocol.FileInfo, copyChan chan<- copyBlocksState, dbUpdateChan chan<- dbUpdateJob) { - curFile, hasCurFile := f.fset.Get(protocol.LocalDeviceID, file.Name) +func (f *sendReceiveFolder) handleFile(file protocol.FileInfo, snap *db.Snapshot, copyChan chan<- copyBlocksState) { + curFile, hasCurFile := snap.Get(protocol.LocalDeviceID, file.Name) have, _ := blockDiff(curFile.Blocks, file.Blocks) @@ -1493,7 +1498,7 @@ func (f *sendReceiveFolder) pullBlock(state pullBlockState, out chan<- *sharedPu out <- state.sharedPullerState } -func (f *sendReceiveFolder) performFinish(file, curFile protocol.FileInfo, hasCurFile bool, tempName string, dbUpdateChan chan<- dbUpdateJob, scanChan chan<- string) error { +func (f *sendReceiveFolder) performFinish(file, curFile protocol.FileInfo, hasCurFile bool, tempName string, snap *db.Snapshot, dbUpdateChan chan<- dbUpdateJob, scanChan chan<- string) error { // Set the correct permission bits on the new file if !f.IgnorePerms && !file.NoPermissions { if err := f.fs.Chmod(tempName, fs.FileMode(file.Permissions&0777)); err != nil { @@ -1528,7 +1533,7 @@ func (f *sendReceiveFolder) performFinish(file, curFile protocol.FileInfo, hasCu return f.moveForConflict(name, file.ModifiedBy.String(), scanChan) }, curFile.Name) } else { - err = f.deleteItemOnDisk(curFile, scanChan) + err = f.deleteItemOnDisk(curFile, snap, scanChan) } if err != nil { return err @@ -1549,7 +1554,7 @@ func (f *sendReceiveFolder) performFinish(file, curFile protocol.FileInfo, hasCu return nil } -func (f *sendReceiveFolder) finisherRoutine(in <-chan *sharedPullerState, dbUpdateChan chan<- dbUpdateJob, scanChan chan<- string) { +func (f *sendReceiveFolder) finisherRoutine(snap *db.Snapshot, in <-chan *sharedPullerState, dbUpdateChan chan<- dbUpdateJob, scanChan chan<- string) { for state := range in { if closed, err := state.finalClose(); closed { l.Debugln(f, "closing", state.file.Name) @@ -1557,7 +1562,7 @@ func (f *sendReceiveFolder) finisherRoutine(in <-chan *sharedPullerState, dbUpda f.queue.Done(state.file.Name) if err == nil { - err = f.performFinish(state.file, state.curFile, state.hasCurFile, state.tempName, dbUpdateChan, scanChan) + err = f.performFinish(state.file, state.curFile, state.hasCurFile, state.tempName, snap, dbUpdateChan, scanChan) } if err != nil { @@ -1809,7 +1814,7 @@ func (f *sendReceiveFolder) Errors() []FileError { } // deleteItemOnDisk deletes the file represented by old that is about to be replaced by new. -func (f *sendReceiveFolder) deleteItemOnDisk(item protocol.FileInfo, scanChan chan<- string) (err error) { +func (f *sendReceiveFolder) deleteItemOnDisk(item protocol.FileInfo, snap *db.Snapshot, scanChan chan<- string) (err error) { defer func() { err = errors.Wrap(err, contextRemovingOldItem) }() @@ -1818,7 +1823,7 @@ func (f *sendReceiveFolder) deleteItemOnDisk(item protocol.FileInfo, scanChan ch case item.IsDirectory(): // Directories aren't archived and need special treatment due // to potential children. - return f.deleteDirOnDisk(item.Name, scanChan) + return f.deleteDirOnDisk(item.Name, snap, scanChan) case !item.IsSymlink() && f.versioner != nil: // If we should use versioning, let the versioner archive the @@ -1834,7 +1839,7 @@ func (f *sendReceiveFolder) deleteItemOnDisk(item protocol.FileInfo, scanChan ch // deleteDirOnDisk attempts to delete a directory. It checks for files/dirs inside // the directory and removes them if possible or returns an error if it fails -func (f *sendReceiveFolder) deleteDirOnDisk(dir string, scanChan chan<- string) error { +func (f *sendReceiveFolder) deleteDirOnDisk(dir string, snap *db.Snapshot, scanChan chan<- string) error { if err := osutil.TraversesSymlink(f.fs, filepath.Dir(dir)); err != nil { return err } @@ -1853,7 +1858,7 @@ func (f *sendReceiveFolder) deleteDirOnDisk(dir string, scanChan chan<- string) toBeDeleted = append(toBeDeleted, fullDirFile) } else if f.ignores != nil && f.ignores.Match(fullDirFile).IsIgnored() { hasIgnored = true - } else if cf, ok := f.fset.Get(protocol.LocalDeviceID, fullDirFile); !ok || cf.IsDeleted() || cf.IsInvalid() { + } else if cf, ok := snap.Get(protocol.LocalDeviceID, fullDirFile); !ok || cf.IsDeleted() || cf.IsInvalid() { // Something appeared in the dir that we either are not aware of // at all, that we think should be deleted or that is invalid, // but not currently ignored -> schedule scan. The scanChan diff --git a/lib/model/folder_sendrecv_test.go b/lib/model/folder_sendrecv_test.go index 2e5a81b17..0de6bf7dd 100644 --- a/lib/model/folder_sendrecv_test.go +++ b/lib/model/folder_sendrecv_test.go @@ -148,9 +148,8 @@ func TestHandleFile(t *testing.T) { defer cleanupSRFolder(f, m) copyChan := make(chan copyBlocksState, 1) - dbUpdateChan := make(chan dbUpdateJob, 1) - f.handleFile(requiredFile, copyChan, dbUpdateChan) + f.handleFile(requiredFile, f.fset.Snapshot(), copyChan) // Receive the results toCopy := <-copyChan @@ -195,9 +194,8 @@ func TestHandleFileWithTemp(t *testing.T) { } copyChan := make(chan copyBlocksState, 1) - dbUpdateChan := make(chan dbUpdateJob, 1) - f.handleFile(requiredFile, copyChan, dbUpdateChan) + f.handleFile(requiredFile, f.fset.Snapshot(), copyChan) // Receive the results toCopy := <-copyChan @@ -247,13 +245,12 @@ func TestCopierFinder(t *testing.T) { copyChan := make(chan copyBlocksState) pullChan := make(chan pullBlockState, 4) finisherChan := make(chan *sharedPullerState, 1) - dbUpdateChan := make(chan dbUpdateJob, 1) // Run a single fetcher routine go f.copierRoutine(copyChan, pullChan, finisherChan) defer close(copyChan) - f.handleFile(requiredFile, copyChan, dbUpdateChan) + f.handleFile(requiredFile, f.fset.Snapshot(), copyChan) timeout := time.After(10 * time.Second) pulls := make([]pullBlockState, 4) @@ -378,7 +375,6 @@ func TestWeakHash(t *testing.T) { copyChan := make(chan copyBlocksState) pullChan := make(chan pullBlockState, expectBlocks) finisherChan := make(chan *sharedPullerState, 1) - dbUpdateChan := make(chan dbUpdateJob, 1) // Run a single fetcher routine go fo.copierRoutine(copyChan, pullChan, finisherChan) @@ -386,7 +382,7 @@ func TestWeakHash(t *testing.T) { // Test 1 - no weak hashing, file gets fully repulled (`expectBlocks` pulls). fo.WeakHashThresholdPct = 101 - fo.handleFile(desiredFile, copyChan, dbUpdateChan) + fo.handleFile(desiredFile, fo.fset.Snapshot(), copyChan) var pulls []pullBlockState timeout := time.After(10 * time.Second) @@ -415,7 +411,7 @@ func TestWeakHash(t *testing.T) { // Test 2 - using weak hash, expectPulls blocks pulled. fo.WeakHashThresholdPct = -1 - fo.handleFile(desiredFile, copyChan, dbUpdateChan) + fo.handleFile(desiredFile, fo.fset.Snapshot(), copyChan) pulls = pulls[:0] for len(pulls) < expectPulls { @@ -495,9 +491,10 @@ func TestDeregisterOnFailInCopy(t *testing.T) { finisherBufferChan := make(chan *sharedPullerState) finisherChan := make(chan *sharedPullerState) dbUpdateChan := make(chan dbUpdateJob, 1) + snap := f.fset.Snapshot() copyChan, copyWg := startCopier(f, pullChan, finisherBufferChan) - go f.finisherRoutine(finisherChan, dbUpdateChan, make(chan string)) + go f.finisherRoutine(snap, finisherChan, dbUpdateChan, make(chan string)) defer func() { close(copyChan) @@ -507,7 +504,7 @@ func TestDeregisterOnFailInCopy(t *testing.T) { close(finisherChan) }() - f.handleFile(file, copyChan, dbUpdateChan) + f.handleFile(file, snap, copyChan) // Receive a block at puller, to indicate that at least a single copier // loop has been performed. @@ -589,6 +586,7 @@ func TestDeregisterOnFailInPull(t *testing.T) { finisherBufferChan := make(chan *sharedPullerState) finisherChan := make(chan *sharedPullerState) dbUpdateChan := make(chan dbUpdateJob, 1) + snap := f.fset.Snapshot() copyChan, copyWg := startCopier(f, pullChan, finisherBufferChan) pullWg := sync.NewWaitGroup() @@ -597,7 +595,7 @@ func TestDeregisterOnFailInPull(t *testing.T) { f.pullerRoutine(pullChan, finisherBufferChan) pullWg.Done() }() - go f.finisherRoutine(finisherChan, dbUpdateChan, make(chan string)) + go f.finisherRoutine(snap, finisherChan, dbUpdateChan, make(chan string)) defer func() { // Unblock copier and puller go func() { @@ -612,7 +610,7 @@ func TestDeregisterOnFailInPull(t *testing.T) { close(finisherChan) }() - f.handleFile(file, copyChan, dbUpdateChan) + f.handleFile(file, snap, copyChan) // Receive at finisher, we should error out as puller has nowhere to pull // from. @@ -693,7 +691,7 @@ func TestIssue3164(t *testing.T) { dbUpdateChan := make(chan dbUpdateJob, 1) - f.deleteDir(file, dbUpdateChan, make(chan string)) + f.deleteDir(file, f.fset.Snapshot(), dbUpdateChan, make(chan string)) if _, err := ffs.Stat("issue3164"); !fs.IsNotExist(err) { t.Fatal(err) @@ -832,7 +830,7 @@ func TestCopyOwner(t *testing.T) { dbUpdateChan := make(chan dbUpdateJob, 1) defer close(dbUpdateChan) - f.handleDir(dir, dbUpdateChan, nil) + f.handleDir(dir, f.fset.Snapshot(), dbUpdateChan, nil) <-dbUpdateChan // empty the channel for later info, err := f.fs.Lstat("foo/bar") @@ -858,16 +856,17 @@ func TestCopyOwner(t *testing.T) { // but it's the way data is passed around. When the database update // comes the finisher is done. + snap := f.fset.Snapshot() finisherChan := make(chan *sharedPullerState) copierChan, copyWg := startCopier(f, nil, finisherChan) - go f.finisherRoutine(finisherChan, dbUpdateChan, nil) + go f.finisherRoutine(snap, finisherChan, dbUpdateChan, nil) defer func() { close(copierChan) copyWg.Wait() close(finisherChan) }() - f.handleFile(file, copierChan, nil) + f.handleFile(file, snap, copierChan) <-dbUpdateChan info, err = f.fs.Lstat("foo/bar/baz") @@ -886,7 +885,7 @@ func TestCopyOwner(t *testing.T) { SymlinkTarget: "over the rainbow", } - f.handleSymlink(symlink, dbUpdateChan, nil) + f.handleSymlink(symlink, snap, dbUpdateChan, nil) <-dbUpdateChan info, err = f.fs.Lstat("foo/bar/sym") @@ -921,7 +920,7 @@ func TestSRConflictReplaceFileByDir(t *testing.T) { dbUpdateChan := make(chan dbUpdateJob, 1) scanChan := make(chan string, 1) - f.handleDir(file, dbUpdateChan, scanChan) + f.handleDir(file, f.fset.Snapshot(), dbUpdateChan, scanChan) if confls := existingConflicts(name, ffs); len(confls) != 1 { t.Fatal("Expected one conflict, got", len(confls)) @@ -954,7 +953,7 @@ func TestSRConflictReplaceFileByLink(t *testing.T) { dbUpdateChan := make(chan dbUpdateJob, 1) scanChan := make(chan string, 1) - f.handleSymlink(file, dbUpdateChan, scanChan) + f.handleSymlink(file, f.fset.Snapshot(), dbUpdateChan, scanChan) if confls := existingConflicts(name, ffs); len(confls) != 1 { t.Fatal("Expected one conflict, got", len(confls)) @@ -996,7 +995,7 @@ func TestDeleteBehindSymlink(t *testing.T) { fi.Version = fi.Version.Update(device1.Short()) scanChan := make(chan string, 1) dbUpdateChan := make(chan dbUpdateJob, 1) - f.deleteFile(fi, dbUpdateChan, scanChan) + f.deleteFile(fi, f.fset.Snapshot(), dbUpdateChan, scanChan) select { case f := <-scanChan: t.Fatalf("Received %v on scanChan", f) diff --git a/lib/model/folder_summary.go b/lib/model/folder_summary.go index 473139094..3c0ad3ac8 100644 --- a/lib/model/folder_summary.go +++ b/lib/model/folder_summary.go @@ -77,6 +77,11 @@ func (c *folderSummaryService) String() string { func (c *folderSummaryService) Summary(folder string) (map[string]interface{}, error) { var res = make(map[string]interface{}) + snap, err := c.model.DBSnapshot(folder) + if err != nil { + return nil, err + } + errors, err := c.model.FolderErrors(folder) if err != nil && err != ErrFolderPaused && err != errFolderNotRunning { // Stats from the db can still be obtained if the folder is just paused/being started @@ -87,19 +92,31 @@ func (c *folderSummaryService) Summary(folder string) (map[string]interface{}, e res["invalid"] = "" // Deprecated, retains external API for now - global := c.model.GlobalSize(folder) + global := snap.GlobalSize() res["globalFiles"], res["globalDirectories"], res["globalSymlinks"], res["globalDeleted"], res["globalBytes"], res["globalTotalItems"] = global.Files, global.Directories, global.Symlinks, global.Deleted, global.Bytes, global.TotalItems() - local := c.model.LocalSize(folder) + local := snap.LocalSize() res["localFiles"], res["localDirectories"], res["localSymlinks"], res["localDeleted"], res["localBytes"], res["localTotalItems"] = local.Files, local.Directories, local.Symlinks, local.Deleted, local.Bytes, local.TotalItems() - need := c.model.NeedSize(folder) + need := snap.NeedSize() + need.Bytes -= c.model.FolderProgressBytesCompleted(folder) + // This may happen if we are in progress of pulling files that were + // deleted globally after the pull started. + if need.Bytes < 0 { + need.Bytes = 0 + } res["needFiles"], res["needDirectories"], res["needSymlinks"], res["needDeletes"], res["needBytes"], res["needTotalItems"] = need.Files, need.Directories, need.Symlinks, need.Deleted, need.Bytes, need.TotalItems() - if c.cfg.Folders()[folder].Type == config.FolderTypeReceiveOnly { + fcfg, ok := c.cfg.Folder(folder) + + if ok && fcfg.IgnoreDelete { + res["needDeletes"] = 0 + } + + if ok && fcfg.Type == config.FolderTypeReceiveOnly { // Add statistics for things that have changed locally in a receive // only folder. - ro := c.model.ReceiveOnlyChangedSize(folder) + ro := snap.ReceiveOnlyChangedSize() res["receiveOnlyChangedFiles"] = ro.Files res["receiveOnlyChangedDirectories"] = ro.Directories res["receiveOnlyChangedSymlinks"] = ro.Symlinks @@ -115,8 +132,8 @@ func (c *folderSummaryService) Summary(folder string) (map[string]interface{}, e res["error"] = err.Error() } - ourSeq, _ := c.model.CurrentSequence(folder) - remoteSeq, _ := c.model.RemoteSequence(folder) + ourSeq := snap.Sequence(protocol.LocalDeviceID) + remoteSeq := snap.Sequence(protocol.GlobalDeviceID) res["version"] = ourSeq + remoteSeq // legacy res["sequence"] = ourSeq + remoteSeq // new name diff --git a/lib/model/model.go b/lib/model/model.go index 279caf3a9..112242ea2 100644 --- a/lib/model/model.go +++ b/lib/model/model.go @@ -90,21 +90,14 @@ type Model interface { GetFolderVersions(folder string) (map[string][]versioner.FileVersion, error) RestoreFolderVersions(folder string, versions map[string]time.Time) (map[string]string, error) - LocalChangedFiles(folder string, page, perpage int) []db.FileInfoTruncated + DBSnapshot(folder string) (*db.Snapshot, error) NeedFolderFiles(folder string, page, perpage int) ([]db.FileInfoTruncated, []db.FileInfoTruncated, []db.FileInfoTruncated) - RemoteNeedFolderFiles(device protocol.DeviceID, folder string, page, perpage int) ([]db.FileInfoTruncated, error) + FolderProgressBytesCompleted(folder string) int64 + CurrentFolderFile(folder string, file string) (protocol.FileInfo, bool) CurrentGlobalFile(folder string, file string) (protocol.FileInfo, bool) Availability(folder string, file protocol.FileInfo, block protocol.BlockInfo) []Availability - GlobalSize(folder string) db.Counts - LocalSize(folder string) db.Counts - NeedSize(folder string) db.Counts - ReceiveOnlyChangedSize(folder string) db.Counts - - CurrentSequence(folder string) (int64, bool) - RemoteSequence(folder string) (int64, bool) - Completion(device protocol.DeviceID, folder string) FolderCompletion ConnectionStats() map[string]interface{} DeviceStatistics() (map[string]stats.DeviceStatistics, error) @@ -764,7 +757,10 @@ func (m *model) Completion(device protocol.DeviceID, folder string) FolderComple return FolderCompletion{} // Folder doesn't exist, so we hardly have any of it } - tot := rf.GlobalSize().Bytes + snap := rf.Snapshot() + defer snap.Release() + + tot := snap.GlobalSize().Bytes if tot == 0 { // Folder is empty, so we have all of it return FolderCompletion{ @@ -777,7 +773,7 @@ func (m *model) Completion(device protocol.DeviceID, folder string) FolderComple m.pmut.RUnlock() var need, items, fileNeed, downloaded, deletes int64 - rf.WithNeedTruncated(device, func(f db.FileIntf) bool { + snap.WithNeedTruncated(device, func(f db.FileIntf) bool { ft := f.(db.FileInfoTruncated) // If the file is deleted, we account it only in the deleted column. @@ -822,84 +818,19 @@ func (m *model) Completion(device protocol.DeviceID, folder string) FolderComple } } -func addSizeOfFile(s *db.Counts, f db.FileIntf) { - switch { - case f.IsDeleted(): - s.Deleted++ - case f.IsDirectory(): - s.Directories++ - case f.IsSymlink(): - s.Symlinks++ - default: - s.Files++ - } - s.Bytes += f.FileSize() -} - -// GlobalSize returns the number of files, deleted files and total bytes for all -// files in the global model. -func (m *model) GlobalSize(folder string) db.Counts { +// DBSnapshot returns a snapshot of the database content relevant to the given folder. +func (m *model) DBSnapshot(folder string) (*db.Snapshot, error) { m.fmut.RLock() rf, ok := m.folderFiles[folder] m.fmut.RUnlock() - if ok { - return rf.GlobalSize() + if !ok { + return nil, errFolderMissing } - return db.Counts{} + return rf.Snapshot(), nil } -// LocalSize returns the number of files, deleted files and total bytes for all -// files in the local folder. -func (m *model) LocalSize(folder string) db.Counts { - m.fmut.RLock() - rf, ok := m.folderFiles[folder] - m.fmut.RUnlock() - if ok { - return rf.LocalSize() - } - return db.Counts{} -} - -// ReceiveOnlyChangedSize returns the number of files, deleted files and -// total bytes for all files that have changed locally in a receieve only -// folder. -func (m *model) ReceiveOnlyChangedSize(folder string) db.Counts { - m.fmut.RLock() - rf, ok := m.folderFiles[folder] - m.fmut.RUnlock() - if ok { - return rf.ReceiveOnlyChangedSize() - } - return db.Counts{} -} - -// NeedSize returns the number of currently needed files and their total size -// minus the amount that has already been downloaded. -func (m *model) NeedSize(folder string) db.Counts { - m.fmut.RLock() - rf, ok := m.folderFiles[folder] - cfg := m.folderCfgs[folder] - m.fmut.RUnlock() - - var result db.Counts - if ok { - rf.WithNeedTruncated(protocol.LocalDeviceID, func(f db.FileIntf) bool { - if cfg.IgnoreDelete && f.IsDeleted() { - return true - } - - addSizeOfFile(&result, f) - return true - }) - } - result.Bytes -= m.progressEmitter.BytesCompleted(folder) - // This may happen if we are in progress of pulling files that were - // deleted globally after the pull started. - if result.Bytes < 0 { - result.Bytes = 0 - } - l.Debugf("%v NeedSize(%q): %v", m, folder, result) - return result +func (m *model) FolderProgressBytesCompleted(folder string) int64 { + return m.progressEmitter.BytesCompleted(folder) } // NeedFolderFiles returns paginated list of currently needed files in @@ -915,6 +846,8 @@ func (m *model) NeedFolderFiles(folder string, page, perpage int) ([]db.FileInfo return nil, nil, nil } + snap := rf.Snapshot() + defer snap.Release() var progress, queued, rest []db.FileInfoTruncated var seen map[string]struct{} @@ -929,14 +862,14 @@ func (m *model) NeedFolderFiles(folder string, page, perpage int) ([]db.FileInfo seen = make(map[string]struct{}, len(progressNames)+len(queuedNames)) for i, name := range progressNames { - if f, ok := rf.GetGlobalTruncated(name); ok { + if f, ok := snap.GetGlobalTruncated(name); ok { progress[i] = f seen[name] = struct{}{} } } for i, name := range queuedNames { - if f, ok := rf.GetGlobalTruncated(name); ok { + if f, ok := snap.GetGlobalTruncated(name); ok { queued[i] = f seen[name] = struct{}{} } @@ -950,7 +883,7 @@ func (m *model) NeedFolderFiles(folder string, page, perpage int) ([]db.FileInfo } rest = make([]db.FileInfoTruncated, 0, perpage) - rf.WithNeedTruncated(protocol.LocalDeviceID, func(f db.FileIntf) bool { + snap.WithNeedTruncated(protocol.LocalDeviceID, func(f db.FileIntf) bool { if cfg.IgnoreDelete && f.IsDeleted() { return true } @@ -970,77 +903,6 @@ func (m *model) NeedFolderFiles(folder string, page, perpage int) ([]db.FileInfo return progress, queued, rest } -// LocalChangedFiles returns a paginated list of currently needed files in -// progress, queued, and to be queued on next puller iteration, as well as the -// total number of files currently needed. -func (m *model) LocalChangedFiles(folder string, page, perpage int) []db.FileInfoTruncated { - m.fmut.RLock() - rf, ok := m.folderFiles[folder] - fcfg := m.folderCfgs[folder] - m.fmut.RUnlock() - - if !ok { - return nil - } - if fcfg.Type != config.FolderTypeReceiveOnly { - return nil - } - if rf.ReceiveOnlyChangedSize().TotalItems() == 0 { - return nil - } - - files := make([]db.FileInfoTruncated, 0, perpage) - - skip := (page - 1) * perpage - get := perpage - - rf.WithHaveTruncated(protocol.LocalDeviceID, func(f db.FileIntf) bool { - if !f.IsReceiveOnlyChanged() { - return true - } - if skip > 0 { - skip-- - return true - } - ft := f.(db.FileInfoTruncated) - files = append(files, ft) - get-- - return get > 0 - }) - - return files -} - -// RemoteNeedFolderFiles returns paginated list of currently needed files in -// progress, queued, and to be queued on next puller iteration, as well as the -// total number of files currently needed. -func (m *model) RemoteNeedFolderFiles(device protocol.DeviceID, folder string, page, perpage int) ([]db.FileInfoTruncated, error) { - m.fmut.RLock() - m.pmut.RLock() - err := m.checkDeviceFolderConnectedLocked(device, folder) - rf := m.folderFiles[folder] - m.pmut.RUnlock() - m.fmut.RUnlock() - if err != nil { - return nil, err - } - - files := make([]db.FileInfoTruncated, 0, perpage) - skip := (page - 1) * perpage - get := perpage - rf.WithNeedTruncated(device, func(f db.FileIntf) bool { - if skip > 0 { - skip-- - return true - } - files = append(files, f.(db.FileInfoTruncated)) - get-- - return get > 0 - }) - - return files, nil -} - // Index is called when a new device is connected and we receive their full index. // Implements the protocol.Model interface. func (m *model) Index(deviceID protocol.DeviceID, folder string, fs []protocol.FileInfo) error { @@ -1752,7 +1614,9 @@ func (m *model) CurrentFolderFile(folder string, file string) (protocol.FileInfo if !ok { return protocol.FileInfo{}, false } - return fs.Get(protocol.LocalDeviceID, file) + snap := fs.Snapshot() + defer snap.Release() + return snap.Get(protocol.LocalDeviceID, file) } func (m *model) CurrentGlobalFile(folder string, file string) (protocol.FileInfo, bool) { @@ -1762,7 +1626,9 @@ func (m *model) CurrentGlobalFile(folder string, file string) (protocol.FileInfo if !ok { return protocol.FileInfo{}, false } - return fs.GetGlobal(file) + snap := fs.Snapshot() + defer snap.Release() + return snap.GetGlobal(file) } // Connection returns the current connection for device, and a boolean whether a connection was found. @@ -2075,7 +1941,9 @@ func (s *indexSender) sendIndexTo(ctx context.Context) error { var err error var f protocol.FileInfo - s.fset.WithHaveSequence(s.prevSequence+1, func(fi db.FileIntf) bool { + snap := s.fset.Snapshot() + defer snap.Release() + snap.WithHaveSequence(s.prevSequence+1, func(fi db.FileIntf) bool { if err = batch.flushIfFull(); err != nil { return false } @@ -2347,45 +2215,6 @@ func (m *model) Revert(folder string) { runner.Revert() } -// CurrentSequence returns the change version for the given folder. -// This is guaranteed to increment if the contents of the local folder has -// changed. -func (m *model) CurrentSequence(folder string) (int64, bool) { - m.fmut.RLock() - fs, ok := m.folderFiles[folder] - m.fmut.RUnlock() - if !ok { - // The folder might not exist, since this can be called with a user - // specified folder name from the REST interface. - return 0, false - } - - return fs.Sequence(protocol.LocalDeviceID), true -} - -// RemoteSequence returns the change version for the given folder, as -// sent by remote peers. This is guaranteed to increment if the contents of -// the remote or global folder has changed. -func (m *model) RemoteSequence(folder string) (int64, bool) { - m.fmut.RLock() - fs, ok := m.folderFiles[folder] - cfg := m.folderCfgs[folder] - m.fmut.RUnlock() - - if !ok { - // The folder might not exist, since this can be called with a user - // specified folder name from the REST interface. - return 0, false - } - - var ver int64 - for _, device := range cfg.Devices { - ver += fs.Sequence(device.DeviceID) - } - - return ver, true -} - func (m *model) GlobalDirectoryTree(folder, prefix string, levels int, dirsonly bool) map[string]interface{} { m.fmut.RLock() files, ok := m.folderFiles[folder] @@ -2402,7 +2231,9 @@ func (m *model) GlobalDirectoryTree(folder, prefix string, levels int, dirsonly prefix = prefix + sep } - files.WithPrefixedGlobalTruncated(prefix, func(fi db.FileIntf) bool { + snap := files.Snapshot() + defer snap.Release() + snap.WithPrefixedGlobalTruncated(prefix, func(fi db.FileIntf) bool { f := fi.(db.FileInfoTruncated) // Don't include the prefix itself. @@ -2514,8 +2345,10 @@ func (m *model) Availability(folder string, file protocol.FileInfo, block protoc } var availabilities []Availability + snap := fs.Snapshot() + defer snap.Release() next: - for _, device := range fs.Availability(file.Name) { + for _, device := range snap.Availability(file.Name) { for _, pausedFolder := range m.remotePausedFolders[device] { if pausedFolder == folder { continue next diff --git a/lib/model/model_test.go b/lib/model/model_test.go index 2a0da0b71..21b3fbb9b 100644 --- a/lib/model/model_test.go +++ b/lib/model/model_test.go @@ -2146,8 +2146,8 @@ func TestIssue3028(t *testing.T) { // Get a count of how many files are there now - locorigfiles := m.LocalSize("default").Files - globorigfiles := m.GlobalSize("default").Files + locorigfiles := localSize(t, m, "default").Files + globorigfiles := globalSize(t, m, "default").Files // Delete and rescan specifically these two @@ -2158,8 +2158,8 @@ func TestIssue3028(t *testing.T) { // Verify that the number of files decreased by two and the number of // deleted files increases by two - loc := m.LocalSize("default") - glob := m.GlobalSize("default") + loc := localSize(t, m, "default") + glob := globalSize(t, m, "default") if loc.Files != locorigfiles-2 { t.Errorf("Incorrect local accounting; got %d current files, expected %d", loc.Files, locorigfiles-2) } @@ -2439,10 +2439,12 @@ func TestIssue3496(t *testing.T) { fs := m.folderFiles["default"] m.fmut.RUnlock() var localFiles []protocol.FileInfo - fs.WithHave(protocol.LocalDeviceID, func(i db.FileIntf) bool { + snap := fs.Snapshot() + snap.WithHave(protocol.LocalDeviceID, func(i db.FileIntf) bool { localFiles = append(localFiles, i.(protocol.FileInfo)) return true }) + snap.Release() // Mark all files as deleted and fake it as update from device1 @@ -2482,7 +2484,7 @@ func TestIssue3496(t *testing.T) { t.Log(comp) // Check that NeedSize does the correct thing - need := m.NeedSize("default") + need := needSize(t, m, "default") if need.Files != 1 || need.Bytes != 1234 { // The one we added synthetically above t.Errorf("Incorrect need size; %d, %d != 1, 1234", need.Files, need.Bytes) diff --git a/lib/model/testutils_test.go b/lib/model/testutils_test.go index 44181087f..d2c662d4f 100644 --- a/lib/model/testutils_test.go +++ b/lib/model/testutils_test.go @@ -9,6 +9,7 @@ package model import ( "io/ioutil" "os" + "testing" "time" "github.com/syncthing/syncthing/lib/config" @@ -171,3 +172,40 @@ func (c *alwaysChanged) Seen(fs fs.Filesystem, name string) bool { func (c *alwaysChanged) Changed() bool { return true } + +func localSize(t *testing.T, m Model, folder string) db.Counts { + t.Helper() + snap := dbSnapshot(t, m, folder) + defer snap.Release() + return snap.LocalSize() +} + +func globalSize(t *testing.T, m Model, folder string) db.Counts { + t.Helper() + snap := dbSnapshot(t, m, folder) + defer snap.Release() + return snap.GlobalSize() +} + +func receiveOnlyChangedSize(t *testing.T, m Model, folder string) db.Counts { + t.Helper() + snap := dbSnapshot(t, m, folder) + defer snap.Release() + return snap.ReceiveOnlyChangedSize() +} + +func needSize(t *testing.T, m Model, folder string) db.Counts { + t.Helper() + snap := dbSnapshot(t, m, folder) + defer snap.Release() + return snap.NeedSize() +} + +func dbSnapshot(t *testing.T, m Model, folder string) *db.Snapshot { + t.Helper() + snap, err := m.DBSnapshot(folder) + if err != nil { + t.Fatal(err) + } + return snap +} diff --git a/lib/ur/usage_report.go b/lib/ur/usage_report.go index 1d45061c0..52293af1b 100644 --- a/lib/ur/usage_report.go +++ b/lib/ur/usage_report.go @@ -88,7 +88,12 @@ func (s *Service) reportData(urVersion int, preview bool) map[string]interface{} var totFiles, maxFiles int var totBytes, maxBytes int64 for folderID := range s.cfg.Folders() { - global := s.model.GlobalSize(folderID) + snap, err := s.model.DBSnapshot(folderID) + if err != nil { + continue + } + global := snap.GlobalSize() + snap.Release() totFiles += int(global.Files) totBytes += global.Bytes if int(global.Files) > maxFiles {