From f822b105503398acdc37922595ed2317c6c80759 Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Thu, 12 Jul 2018 11:15:57 +0300 Subject: [PATCH] all: Add receive only folder type (#5027) Adds a receive only folder type that does not send changes, and where the user can optionally revert local changes. Also changes some of the icons to make the three folder types distinguishable. --- cmd/syncthing/gui.go | 20 ++ cmd/syncthing/mocked_model_test.go | 6 + gui/default/index.html | 10 +- .../syncthing/core/syncthingController.js | 16 ++ .../syncthing/folder/editFolderModalView.html | 2 + lib/config/foldertype.go | 5 + lib/db/leveldb_dbinstance.go | 2 +- lib/db/leveldb_test.go | 4 +- lib/db/leveldb_transactions.go | 8 +- lib/db/meta.go | 135 ++++++--- lib/db/meta_test.go | 82 ++++++ lib/db/set.go | 18 +- lib/db/set_test.go | 118 +++++++- lib/db/structs.go | 25 ++ lib/db/structs.pb.go | 116 +++++--- lib/db/structs.proto | 15 +- lib/model/folder.go | 5 +- lib/model/folder_recvonly.go | 210 ++++++++++++++ lib/model/folder_recvonly_test.go | 261 ++++++++++++++++++ lib/model/folder_sendonly.go | 2 +- lib/model/folder_sendrecv.go | 32 ++- lib/model/folder_sendrecv_test.go | 4 +- lib/model/model.go | 41 ++- lib/protocol/bep_extensions.go | 30 +- lib/protocol/deviceid.go | 12 +- lib/protocol/protocol.go | 5 +- lib/protocol/protocol_test.go | 16 +- lib/scanner/walk.go | 11 +- lib/scanner/walk_test.go | 69 ++++- 29 files changed, 1136 insertions(+), 144 deletions(-) create mode 100644 lib/db/meta_test.go create mode 100644 lib/model/folder_recvonly.go create mode 100644 lib/model/folder_recvonly_test.go diff --git a/cmd/syncthing/gui.go b/cmd/syncthing/gui.go index 90f0b2e78..e57db76db 100644 --- a/cmd/syncthing/gui.go +++ b/cmd/syncthing/gui.go @@ -85,6 +85,7 @@ type modelIntf interface { GlobalDirectoryTree(folder, prefix string, levels int, dirsonly bool) map[string]interface{} Completion(device protocol.DeviceID, folder string) model.FolderCompletion Override(folder string) + Revert(folder string) NeedFolderFiles(folder string, page, perpage int) ([]db.FileInfoTruncated, []db.FileInfoTruncated, []db.FileInfoTruncated) RemoteNeedFolderFiles(device protocol.DeviceID, folder string, page, perpage int) ([]db.FileInfoTruncated, error) NeedSize(folder string) db.Counts @@ -107,6 +108,7 @@ type modelIntf interface { Connection(deviceID protocol.DeviceID) (connections.Connection, bool) GlobalSize(folder string) db.Counts LocalSize(folder string) db.Counts + ReceiveOnlyChangedSize(folder string) db.Counts CurrentSequence(folder string) (int64, bool) RemoteSequence(folder string) (int64, bool) State(folder string) (string, time.Time, error) @@ -293,6 +295,7 @@ func (s *apiService) Serve() { postRestMux.HandleFunc("/rest/db/prio", s.postDBPrio) // folder file [perpage] [page] postRestMux.HandleFunc("/rest/db/ignores", s.postDBIgnores) // folder postRestMux.HandleFunc("/rest/db/override", s.postDBOverride) // folder + postRestMux.HandleFunc("/rest/db/revert", s.postDBRevert) // folder postRestMux.HandleFunc("/rest/db/scan", s.postDBScan) // folder [sub...] [delay] postRestMux.HandleFunc("/rest/folder/versions", s.postFolderVersionsRestore) // folder postRestMux.HandleFunc("/rest/system/config", s.postSystemConfig) // @@ -712,6 +715,17 @@ func folderSummary(cfg configIntf, m modelIntf, folder string) (map[string]inter need := m.NeedSize(folder) res["needFiles"], res["needDirectories"], res["needSymlinks"], res["needDeletes"], res["needBytes"] = need.Files, need.Directories, need.Symlinks, need.Deleted, need.Bytes + if cfg.Folders()[folder].Type == config.FolderTypeReceiveOnly { + // Add statistics for things that have changed locally in a receive + // only folder. + ro := m.ReceiveOnlyChangedSize(folder) + res["receiveOnlyChangedFiles"] = ro.Files + res["receiveOnlyChangedDirectories"] = ro.Directories + res["receiveOnlyChangedSymlinks"] = ro.Symlinks + res["receiveOnlyChangedDeletes"] = ro.Deleted + res["receiveOnlyChangedBytes"] = ro.Bytes + } + res["inSyncFiles"], res["inSyncBytes"] = global.Files-need.Files, global.Bytes-need.Bytes res["state"], res["stateChanged"], err = m.State(folder) @@ -748,6 +762,12 @@ func (s *apiService) postDBOverride(w http.ResponseWriter, r *http.Request) { go s.model.Override(folder) } +func (s *apiService) postDBRevert(w http.ResponseWriter, r *http.Request) { + var qs = r.URL.Query() + var folder = qs.Get("folder") + go s.model.Revert(folder) +} + func getPagingParams(qs url.Values) (int, int) { page, err := strconv.Atoi(qs.Get("page")) if err != nil || page < 1 { diff --git a/cmd/syncthing/mocked_model_test.go b/cmd/syncthing/mocked_model_test.go index 5841b99a0..04e854aa1 100644 --- a/cmd/syncthing/mocked_model_test.go +++ b/cmd/syncthing/mocked_model_test.go @@ -29,6 +29,8 @@ func (m *mockedModel) Completion(device protocol.DeviceID, folder string) model. func (m *mockedModel) Override(folder string) {} +func (m *mockedModel) Revert(folder string) {} + func (m *mockedModel) NeedFolderFiles(folder string, page, perpage int) ([]db.FileInfoTruncated, []db.FileInfoTruncated, []db.FileInfoTruncated) { return nil, nil, nil } @@ -117,6 +119,10 @@ func (m *mockedModel) LocalSize(folder string) db.Counts { return db.Counts{} } +func (m *mockedModel) ReceiveOnlyChangedSize(folder string) db.Counts { + return db.Counts{} +} + func (m *mockedModel) CurrentSequence(folder string) (int64, bool) { return 0, false } diff --git a/gui/default/index.html b/gui/default/index.html index 7dd6786ae..c2a10c0eb 100644 --- a/gui/default/index.html +++ b/gui/default/index.html @@ -301,7 +301,9 @@

◼ @@ -386,9 +388,10 @@ -  Folder Type +  Folder Type Send Only + Receive Only @@ -478,6 +481,9 @@ +
diff --git a/lib/config/foldertype.go b/lib/config/foldertype.go index 1c82bb42d..e76ef0884 100644 --- a/lib/config/foldertype.go +++ b/lib/config/foldertype.go @@ -11,6 +11,7 @@ type FolderType int const ( FolderTypeSendReceive FolderType = iota // default is sendreceive FolderTypeSendOnly + FolderTypeReceiveOnly ) func (t FolderType) String() string { @@ -19,6 +20,8 @@ func (t FolderType) String() string { return "sendreceive" case FolderTypeSendOnly: return "sendonly" + case FolderTypeReceiveOnly: + return "receiveonly" default: return "unknown" } @@ -34,6 +37,8 @@ func (t *FolderType) UnmarshalText(bs []byte) error { *t = FolderTypeSendReceive case "readonly", "sendonly": *t = FolderTypeSendOnly + case "receiveonly": + *t = FolderTypeReceiveOnly default: *t = FolderTypeSendReceive } diff --git a/lib/db/leveldb_dbinstance.go b/lib/db/leveldb_dbinstance.go index 87b13afa5..7d67ea671 100644 --- a/lib/db/leveldb_dbinstance.go +++ b/lib/db/leveldb_dbinstance.go @@ -586,7 +586,7 @@ func (db *Instance) checkGlobals(folder []byte, meta *metadataTracker) { if i == 0 { if fi, ok := db.getFile(fk); ok { - meta.addFile(globalDeviceID, fi) + meta.addFile(protocol.GlobalDeviceID, fi) } } } diff --git a/lib/db/leveldb_test.go b/lib/db/leveldb_test.go index 85196259e..dcdb5f41a 100644 --- a/lib/db/leveldb_test.go +++ b/lib/db/leveldb_test.go @@ -305,7 +305,7 @@ func TestUpdate0to3(t *testing.T) { t.Error("Unexpected additional file via sequence", f.FileName()) return true } - if e := haveUpdate0to3[protocol.LocalDeviceID][0]; f.IsEquivalent(e, true, true) { + if e := haveUpdate0to3[protocol.LocalDeviceID][0]; f.IsEquivalentOptional(e, true, true, 0) { found = true } else { t.Errorf("Wrong file via sequence, got %v, expected %v", f, e) @@ -330,7 +330,7 @@ func TestUpdate0to3(t *testing.T) { } f := fi.(protocol.FileInfo) delete(need, f.Name) - if !f.IsEquivalent(e, true, true) { + if !f.IsEquivalentOptional(e, true, true, 0) { t.Errorf("Wrong needed file, got %v, expected %v", f, e) } return true diff --git a/lib/db/leveldb_transactions.go b/lib/db/leveldb_transactions.go index a9ce5ddb7..64496380f 100644 --- a/lib/db/leveldb_transactions.go +++ b/lib/db/leveldb_transactions.go @@ -140,11 +140,11 @@ func (t readWriteTransaction) updateGlobal(gk, folder, device []byte, file proto if oldFile, ok := t.getFile(folder, oldGlobalFV.Device, name); ok { // A failure to get the file here is surprising and our // global size data will be incorrect until a restart... - meta.removeFile(globalDeviceID, oldFile) + meta.removeFile(protocol.GlobalDeviceID, oldFile) } // Add the new global to the global size counter - meta.addFile(globalDeviceID, newGlobal) + meta.addFile(protocol.GlobalDeviceID, newGlobal) l.Debugf(`new global for "%v" after update: %v`, file.Name, fl) t.Put(gk, mustMarshal(&fl)) @@ -197,7 +197,7 @@ func (t readWriteTransaction) removeFromGlobal(gk, folder, device, file []byte, // didn't exist anyway, apparently continue } - meta.removeFile(globalDeviceID, f) + meta.removeFile(protocol.GlobalDeviceID, f) removed = true } fl.Versions = append(fl.Versions[:i], fl.Versions[i+1:]...) @@ -215,7 +215,7 @@ func (t readWriteTransaction) removeFromGlobal(gk, folder, device, file []byte, if f, ok := t.getFile(folder, fl.Versions[0].Device, file); ok { // A failure to get the file here is surprising and our // global size data will be incorrect until a restart... - meta.addFile(globalDeviceID, f) + meta.addFile(protocol.GlobalDeviceID, f) } } } diff --git a/lib/db/meta.go b/lib/db/meta.go index 09015b174..93dc2a59d 100644 --- a/lib/db/meta.go +++ b/lib/db/meta.go @@ -7,25 +7,30 @@ package db import ( + "bytes" + "math/bits" "time" "github.com/syncthing/syncthing/lib/protocol" "github.com/syncthing/syncthing/lib/sync" ) -// like protocol.LocalDeviceID but with 0xf8 in all positions -var globalDeviceID = protocol.DeviceID{0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8} - +// metadataTracker keeps metadata on a per device, per local flag basis. type metadataTracker struct { mut sync.RWMutex counts CountsSet - indexes map[protocol.DeviceID]int // device ID -> index in counts + indexes map[metaKey]int // device ID + local flags -> index in counts +} + +type metaKey struct { + dev protocol.DeviceID + flags uint32 } func newMetadataTracker() *metadataTracker { return &metadataTracker{ mut: sync.NewRWMutex(), - indexes: make(map[protocol.DeviceID]int), + indexes: make(map[metaKey]int), } } @@ -38,7 +43,7 @@ func (m *metadataTracker) Unmarshal(bs []byte) error { // Initialize the index map for i, c := range m.counts.Counts { - m.indexes[protocol.DeviceIDFromBytes(c.DeviceID)] = i + m.indexes[metaKey{protocol.DeviceIDFromBytes(c.DeviceID), c.LocalFlags}] = i } return nil } @@ -72,14 +77,15 @@ func (m *metadataTracker) fromDB(db *Instance, folder []byte) error { // countsPtr returns a pointer to the corresponding Counts struct, if // necessary allocating one in the process -func (m *metadataTracker) countsPtr(dev protocol.DeviceID) *Counts { +func (m *metadataTracker) countsPtr(dev protocol.DeviceID, flags uint32) *Counts { // must be called with the mutex held - idx, ok := m.indexes[dev] + key := metaKey{dev, flags} + idx, ok := m.indexes[key] if !ok { idx = len(m.counts.Counts) - m.counts.Counts = append(m.counts.Counts, Counts{DeviceID: dev[:]}) - m.indexes[dev] = idx + m.counts.Counts = append(m.counts.Counts, Counts{DeviceID: dev[:], LocalFlags: flags}) + m.indexes[key] = idx } return &m.counts.Counts[idx] } @@ -87,12 +93,23 @@ func (m *metadataTracker) countsPtr(dev protocol.DeviceID) *Counts { // addFile adds a file to the counts, adjusting the sequence number as // appropriate func (m *metadataTracker) addFile(dev protocol.DeviceID, f FileIntf) { - if f.IsInvalid() { - return + m.mut.Lock() + + if flags := f.FileLocalFlags(); flags == 0 { + // Account regular files in the zero-flags bucket. + m.addFileLocked(dev, 0, f) + } else { + // Account in flag specific buckets. + eachFlagBit(flags, func(flag uint32) { + m.addFileLocked(dev, flag, f) + }) } - m.mut.Lock() - cp := m.countsPtr(dev) + m.mut.Unlock() +} + +func (m *metadataTracker) addFileLocked(dev protocol.DeviceID, flags uint32, f FileIntf) { + cp := m.countsPtr(dev, flags) switch { case f.IsDeleted(): @@ -109,18 +126,27 @@ func (m *metadataTracker) addFile(dev protocol.DeviceID, f FileIntf) { if seq := f.SequenceNo(); seq > cp.Sequence { cp.Sequence = seq } - - m.mut.Unlock() } // removeFile removes a file from the counts func (m *metadataTracker) removeFile(dev protocol.DeviceID, f FileIntf) { - if f.IsInvalid() { - return + m.mut.Lock() + + if flags := f.FileLocalFlags(); flags == 0 { + // Remove regular files from the zero-flags bucket + m.removeFileLocked(dev, 0, f) + } else { + // Remove from flag specific buckets. + eachFlagBit(flags, func(flag uint32) { + m.removeFileLocked(dev, flag, f) + }) } - m.mut.Lock() - cp := m.countsPtr(dev) + m.mut.Unlock() +} + +func (m *metadataTracker) removeFileLocked(dev protocol.DeviceID, flags uint32, f FileIntf) { + cp := m.countsPtr(dev, f.FileLocalFlags()) switch { case f.IsDeleted(): @@ -153,14 +179,19 @@ func (m *metadataTracker) removeFile(dev protocol.DeviceID, f FileIntf) { cp.Symlinks = 0 m.counts.Created = 0 } - - m.mut.Unlock() } // resetAll resets all metadata for the given device func (m *metadataTracker) resetAll(dev protocol.DeviceID) { m.mut.Lock() - *m.countsPtr(dev) = Counts{DeviceID: dev[:]} + for i, c := range m.counts.Counts { + if bytes.Equal(c.DeviceID, dev[:]) { + m.counts.Counts[i] = Counts{ + DeviceID: c.DeviceID, + LocalFlags: c.LocalFlags, + } + } + } m.mut.Unlock() } @@ -169,23 +200,30 @@ func (m *metadataTracker) resetAll(dev protocol.DeviceID) { func (m *metadataTracker) resetCounts(dev protocol.DeviceID) { m.mut.Lock() - c := m.countsPtr(dev) - c.Bytes = 0 - c.Deleted = 0 - c.Directories = 0 - c.Files = 0 - c.Symlinks = 0 - // c.Sequence deliberately untouched + for i, c := range m.counts.Counts { + if bytes.Equal(c.DeviceID, dev[:]) { + m.counts.Counts[i] = Counts{ + DeviceID: c.DeviceID, + Sequence: c.Sequence, + LocalFlags: c.LocalFlags, + } + } + } m.mut.Unlock() } -// Counts returns the counts for the given device ID -func (m *metadataTracker) Counts(dev protocol.DeviceID) Counts { +// Counts returns the counts for the given device ID and flag. `flag` should +// be zero or have exactly one bit set. +func (m *metadataTracker) Counts(dev protocol.DeviceID, flag uint32) Counts { + if bits.OnesCount32(flag) > 1 { + panic("incorrect usage: set at most one bit in flag") + } + m.mut.RLock() defer m.mut.RUnlock() - idx, ok := m.indexes[dev] + idx, ok := m.indexes[metaKey{dev, flag}] if !ok { return Counts{} } @@ -198,7 +236,7 @@ func (m *metadataTracker) nextSeq(dev protocol.DeviceID) int64 { m.mut.Lock() defer m.mut.Unlock() - c := m.countsPtr(dev) + c := m.countsPtr(dev, 0) c.Sequence++ return c.Sequence } @@ -206,21 +244,26 @@ func (m *metadataTracker) nextSeq(dev protocol.DeviceID) int64 { // devices returns the list of devices tracked, excluding the local device // (which we don't know the ID of) func (m *metadataTracker) devices() []protocol.DeviceID { - devs := make([]protocol.DeviceID, 0, len(m.counts.Counts)) + devs := make(map[protocol.DeviceID]struct{}, len(m.counts.Counts)) m.mut.RLock() for _, dev := range m.counts.Counts { if dev.Sequence > 0 { id := protocol.DeviceIDFromBytes(dev.DeviceID) - if id == globalDeviceID || id == protocol.LocalDeviceID { + if id == protocol.GlobalDeviceID || id == protocol.LocalDeviceID { continue } - devs = append(devs, id) + devs[id] = struct{}{} } } m.mut.RUnlock() - return devs + devList := make([]protocol.DeviceID, 0, len(devs)) + for dev := range devs { + devList = append(devList, dev) + } + + return devList } func (m *metadataTracker) Created() time.Time { @@ -234,3 +277,19 @@ func (m *metadataTracker) SetCreated() { m.counts.Created = time.Now().UnixNano() m.mut.Unlock() } + +// eachFlagBit calls the function once for every bit that is set in flags +func eachFlagBit(flags uint32, fn func(flag uint32)) { + // Test each bit from the right, as long as there are bits left in the + // flag set. Clear any bits found and stop testing as soon as there are + // no more bits set. + + currentBit := uint32(1 << 0) + for flags != 0 { + if flags¤tBit != 0 { + fn(currentBit) + flags &^= currentBit + } + currentBit <<= 1 + } +} diff --git a/lib/db/meta_test.go b/lib/db/meta_test.go new file mode 100644 index 000000000..4d807f656 --- /dev/null +++ b/lib/db/meta_test.go @@ -0,0 +1,82 @@ +// Copyright (C) 2018 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at https://mozilla.org/MPL/2.0/. + +package db + +import ( + "math/bits" + "sort" + "testing" + + "github.com/syncthing/syncthing/lib/protocol" +) + +func TestEachFlagBit(t *testing.T) { + cases := []struct { + flags uint32 + iterations int + }{ + {0, 0}, + {1<<0 | 1<<3, 2}, + {1 << 0, 1}, + {1 << 31, 1}, + {1<<10 | 1<<20 | 1<<30, 3}, + } + + for _, tc := range cases { + var flags uint32 + iterations := 0 + + eachFlagBit(tc.flags, func(f uint32) { + iterations++ + flags |= f + if bits.OnesCount32(f) != 1 { + t.Error("expected exactly one bit to be set in every call") + } + }) + + if flags != tc.flags { + t.Errorf("expected 0x%x flags, got 0x%x", tc.flags, flags) + } + if iterations != tc.iterations { + t.Errorf("expected %d iterations, got %d", tc.iterations, iterations) + } + } +} + +func TestMetaDevices(t *testing.T) { + d1 := protocol.DeviceID{1} + d2 := protocol.DeviceID{2} + meta := newMetadataTracker() + + meta.addFile(d1, protocol.FileInfo{Sequence: 1}) + meta.addFile(d1, protocol.FileInfo{Sequence: 2, LocalFlags: 1}) + meta.addFile(d2, protocol.FileInfo{Sequence: 1}) + meta.addFile(d2, protocol.FileInfo{Sequence: 2, LocalFlags: 2}) + meta.addFile(protocol.LocalDeviceID, protocol.FileInfo{Sequence: 1}) + + // There are five device/flags combos + if l := len(meta.counts.Counts); l < 5 { + t.Error("expected at least five buckets, not", l) + } + + // There are only two non-local devices + devs := meta.devices() + if l := len(devs); l != 2 { + t.Fatal("expected two devices, not", l) + } + + // Check that we got the two devices we expect + sort.Slice(devs, func(a, b int) bool { + return devs[a].Compare(devs[b]) == -1 + }) + if devs[0] != d1 { + t.Error("first device should be d1") + } + if devs[1] != d2 { + t.Error("second device should be d2") + } +} diff --git a/lib/db/set.go b/lib/db/set.go index 314ed4538..8093fb3c0 100644 --- a/lib/db/set.go +++ b/lib/db/set.go @@ -37,8 +37,12 @@ type FileSet struct { type FileIntf interface { FileSize() int64 FileName() string + FileLocalFlags() uint32 IsDeleted() bool IsInvalid() bool + IsIgnored() bool + IsUnsupported() bool + MustRescan() bool IsDirectory() bool IsSymlink() bool HasPermissionBits() bool @@ -248,15 +252,23 @@ func (s *FileSet) Availability(file string) []protocol.DeviceID { } func (s *FileSet) Sequence(device protocol.DeviceID) int64 { - return s.meta.Counts(device).Sequence + return s.meta.Counts(device, 0).Sequence } func (s *FileSet) LocalSize() Counts { - return s.meta.Counts(protocol.LocalDeviceID) + local := s.meta.Counts(protocol.LocalDeviceID, 0) + recvOnlyChanged := s.meta.Counts(protocol.LocalDeviceID, protocol.FlagLocalReceiveOnly) + return local.Add(recvOnlyChanged) +} + +func (s *FileSet) ReceiveOnlyChangedSize() Counts { + return s.meta.Counts(protocol.LocalDeviceID, protocol.FlagLocalReceiveOnly) } func (s *FileSet) GlobalSize() Counts { - return s.meta.Counts(globalDeviceID) + global := s.meta.Counts(protocol.GlobalDeviceID, 0) + recvOnlyChanged := s.meta.Counts(protocol.GlobalDeviceID, protocol.FlagLocalReceiveOnly) + return global.Add(recvOnlyChanged) } func (s *FileSet) IndexID(device protocol.DeviceID) protocol.IndexID { diff --git a/lib/db/set_test.go b/lib/db/set_test.go index fd0cb540e..53087a579 100644 --- a/lib/db/set_test.go +++ b/lib/db/set_test.go @@ -906,7 +906,7 @@ func TestWithHaveSequence(t *testing.T) { i := 2 s.WithHaveSequence(int64(i), func(fi db.FileIntf) bool { - if f := fi.(protocol.FileInfo); !f.IsEquivalent(localHave[i-1], false, false) { + if f := fi.(protocol.FileInfo); !f.IsEquivalent(localHave[i-1]) { t.Fatalf("Got %v\nExpected %v", f, localHave[i-1]) } i++ @@ -917,7 +917,7 @@ func TestWithHaveSequence(t *testing.T) { func TestIssue4925(t *testing.T) { ldb := db.OpenMemory() - folder := "test)" + folder := "test" s := db.NewFileSet(folder, fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb) localHave := fileList{ @@ -955,7 +955,7 @@ func TestMoveGlobalBack(t *testing.T) { if need := needList(s, protocol.LocalDeviceID); len(need) != 1 { t.Error("Expected 1 local need, got", need) - } else if !need[0].IsEquivalent(remote0Have[0], false, false) { + } else if !need[0].IsEquivalent(remote0Have[0]) { t.Errorf("Local need incorrect;\n A: %v !=\n E: %v", need[0], remote0Have[0]) } @@ -981,7 +981,7 @@ func TestMoveGlobalBack(t *testing.T) { if need := needList(s, remoteDevice0); len(need) != 1 { t.Error("Expected 1 need for remote 0, got", need) - } else if !need[0].IsEquivalent(localHave[0], false, false) { + } else if !need[0].IsEquivalent(localHave[0]) { t.Errorf("Need for remote 0 incorrect;\n A: %v !=\n E: %v", need[0], localHave[0]) } @@ -1017,7 +1017,7 @@ func TestIssue5007(t *testing.T) { if need := needList(s, protocol.LocalDeviceID); len(need) != 1 { t.Fatal("Expected 1 local need, got", need) - } else if !need[0].IsEquivalent(fs[0], false, false) { + } else if !need[0].IsEquivalent(fs[0]) { t.Fatalf("Local need incorrect;\n A: %v !=\n E: %v", need[0], fs[0]) } @@ -1052,7 +1052,7 @@ func TestNeedDeleted(t *testing.T) { if need := needList(s, protocol.LocalDeviceID); len(need) != 1 { t.Fatal("Expected 1 local need, got", need) - } else if !need[0].IsEquivalent(fs[0], false, false) { + } else if !need[0].IsEquivalent(fs[0]) { t.Fatalf("Local need incorrect;\n A: %v !=\n E: %v", need[0], fs[0]) } @@ -1065,6 +1065,110 @@ func TestNeedDeleted(t *testing.T) { } } +func TestReceiveOnlyAccounting(t *testing.T) { + ldb := db.OpenMemory() + + folder := "test" + s := db.NewFileSet(folder, fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb) + + local := protocol.DeviceID{1} + remote := protocol.DeviceID{2} + + // Three files that have been created by the remote device + + version := protocol.Vector{Counters: []protocol.Counter{{ID: remote.Short(), Value: 1}}} + files := fileList{ + protocol.FileInfo{Name: "f1", Size: 10, Sequence: 1, Version: version}, + protocol.FileInfo{Name: "f2", Size: 10, Sequence: 1, Version: version}, + protocol.FileInfo{Name: "f3", Size: 10, Sequence: 1, Version: version}, + } + + // We have synced them locally + + replace(s, protocol.LocalDeviceID, files) + replace(s, remote, files) + + if n := s.LocalSize().Files; n != 3 { + t.Fatal("expected 3 local files initially, not", n) + } + if n := s.LocalSize().Bytes; n != 30 { + t.Fatal("expected 30 local bytes initially, not", n) + } + if n := s.GlobalSize().Files; n != 3 { + t.Fatal("expected 3 global files initially, not", n) + } + if n := s.GlobalSize().Bytes; n != 30 { + t.Fatal("expected 30 global bytes initially, not", n) + } + if n := s.ReceiveOnlyChangedSize().Files; n != 0 { + t.Fatal("expected 0 receive only changed files initially, not", n) + } + if n := s.ReceiveOnlyChangedSize().Bytes; n != 0 { + t.Fatal("expected 0 receive only changed bytes initially, not", n) + } + + // Detected a local change in a receive only folder + + changed := files[0] + changed.Version = changed.Version.Update(local.Short()) + changed.Size = 100 + changed.ModifiedBy = local.Short() + changed.LocalFlags = protocol.FlagLocalReceiveOnly + s.Update(protocol.LocalDeviceID, []protocol.FileInfo{changed}) + + // Check that we see the files + + if n := s.LocalSize().Files; n != 3 { + t.Fatal("expected 3 local files after local change, not", n) + } + if n := s.LocalSize().Bytes; n != 120 { + t.Fatal("expected 120 local bytes after local change, not", n) + } + if n := s.GlobalSize().Files; n != 3 { + t.Fatal("expected 3 global files after local change, not", n) + } + if n := s.GlobalSize().Bytes; n != 120 { + t.Fatal("expected 120 global bytes after local change, not", n) + } + if n := s.ReceiveOnlyChangedSize().Files; n != 1 { + t.Fatal("expected 1 receive only changed file after local change, not", n) + } + if n := s.ReceiveOnlyChangedSize().Bytes; n != 100 { + t.Fatal("expected 100 receive only changed btyes after local change, not", n) + } + + // Fake a revert. That's a two step process, first converting our + // changed file into a less preferred variant, then pulling down the old + // version. + + changed.Version = protocol.Vector{} + changed.LocalFlags &^= protocol.FlagLocalReceiveOnly + s.Update(protocol.LocalDeviceID, []protocol.FileInfo{changed}) + + s.Update(protocol.LocalDeviceID, []protocol.FileInfo{files[0]}) + + // Check that we see the files, same data as initially + + if n := s.LocalSize().Files; n != 3 { + t.Fatal("expected 3 local files after revert, not", n) + } + if n := s.LocalSize().Bytes; n != 30 { + t.Fatal("expected 30 local bytes after revert, not", n) + } + if n := s.GlobalSize().Files; n != 3 { + t.Fatal("expected 3 global files after revert, not", n) + } + if n := s.GlobalSize().Bytes; n != 30 { + t.Fatal("expected 30 global bytes after revert, not", n) + } + if n := s.ReceiveOnlyChangedSize().Files; n != 0 { + t.Fatal("expected 0 receive only changed files after revert, not", n) + } + if n := s.ReceiveOnlyChangedSize().Bytes; n != 0 { + t.Fatal("expected 0 receive only changed bytes after revert, not", n) + } +} + func TestNeedAfterUnignore(t *testing.T) { ldb := db.OpenMemory() @@ -1090,7 +1194,7 @@ func TestNeedAfterUnignore(t *testing.T) { if need := needList(s, protocol.LocalDeviceID); len(need) != 1 { t.Fatal("Expected one local need, got", need) - } else if !need[0].IsEquivalent(remote, false, false) { + } else if !need[0].IsEquivalent(remote) { t.Fatalf("Got %v, expected %v", need[0], remote) } } diff --git a/lib/db/structs.go b/lib/db/structs.go index 91aff8409..985a432dd 100644 --- a/lib/db/structs.go +++ b/lib/db/structs.go @@ -40,6 +40,10 @@ func (f FileInfoTruncated) IsInvalid() bool { return f.RawInvalid || f.LocalFlags&protocol.LocalInvalidFlags != 0 } +func (f FileInfoTruncated) IsUnsupported() bool { + return f.LocalFlags&protocol.FlagLocalUnsupported != 0 +} + func (f FileInfoTruncated) IsIgnored() bool { return f.LocalFlags&protocol.FlagLocalIgnored != 0 } @@ -48,6 +52,10 @@ func (f FileInfoTruncated) MustRescan() bool { return f.LocalFlags&protocol.FlagLocalMustRescan != 0 } +func (f FileInfoTruncated) IsReceiveOnlyChanged() bool { + return f.LocalFlags&protocol.FlagLocalReceiveOnly != 0 +} + func (f FileInfoTruncated) IsDirectory() bool { return f.Type == protocol.FileInfoTypeDirectory } @@ -86,6 +94,10 @@ func (f FileInfoTruncated) FileName() string { return f.Name } +func (f FileInfoTruncated) FileLocalFlags() uint32 { + return f.LocalFlags +} + func (f FileInfoTruncated) ModTime() time.Time { return time.Unix(f.ModifiedS, int64(f.ModifiedNs)) } @@ -110,3 +122,16 @@ func (f FileInfoTruncated) ConvertToIgnoredFileInfo(by protocol.ShortID) protoco LocalFlags: protocol.FlagLocalIgnored, } } + +func (c Counts) Add(other Counts) Counts { + return Counts{ + Files: c.Files + other.Files, + Directories: c.Directories + other.Directories, + Symlinks: c.Symlinks + other.Symlinks, + Deleted: c.Deleted + other.Deleted, + Bytes: c.Bytes + other.Bytes, + Sequence: c.Sequence + other.Sequence, + DeviceID: protocol.EmptyDeviceID[:], + LocalFlags: c.LocalFlags | other.LocalFlags, + } +} diff --git a/lib/db/structs.pb.go b/lib/db/structs.pb.go index e11fc5359..e3a9e4b7e 100644 --- a/lib/db/structs.pb.go +++ b/lib/db/structs.pb.go @@ -91,6 +91,7 @@ type Counts struct { Bytes int64 `protobuf:"varint,5,opt,name=bytes,proto3" json:"bytes,omitempty"` Sequence int64 `protobuf:"varint,6,opt,name=sequence,proto3" json:"sequence,omitempty"` DeviceID []byte `protobuf:"bytes,17,opt,name=deviceID,proto3" json:"deviceID,omitempty"` + LocalFlags uint32 `protobuf:"varint,18,opt,name=localFlags,proto3" json:"localFlags,omitempty"` } func (m *Counts) Reset() { *m = Counts{} } @@ -357,6 +358,13 @@ func (m *Counts) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintStructs(dAtA, i, uint64(len(m.DeviceID))) i += copy(dAtA[i:], m.DeviceID) } + if m.LocalFlags != 0 { + dAtA[i] = 0x90 + i++ + dAtA[i] = 0x1 + i++ + i = encodeVarintStructs(dAtA, i, uint64(m.LocalFlags)) + } return i, nil } @@ -526,6 +534,9 @@ func (m *Counts) ProtoSize() (n int) { if l > 0 { n += 2 + l + sovStructs(uint64(l)) } + if m.LocalFlags != 0 { + n += 2 + sovStructs(uint64(m.LocalFlags)) + } return n } @@ -1312,6 +1323,25 @@ func (m *Counts) Unmarshal(dAtA []byte) error { m.DeviceID = []byte{} } iNdEx = postIndex + case 18: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field LocalFlags", wireType) + } + m.LocalFlags = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowStructs + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.LocalFlags |= (uint32(b) & 0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipStructs(dAtA[iNdEx:]) @@ -1541,47 +1571,47 @@ var ( func init() { proto.RegisterFile("structs.proto", fileDescriptorStructs) } var fileDescriptorStructs = []byte{ - // 663 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x53, 0xcd, 0x6a, 0xdb, 0x40, - 0x10, 0xb6, 0x62, 0xf9, 0x6f, 0x6c, 0xa7, 0xc9, 0x12, 0x82, 0x30, 0xd4, 0x16, 0x86, 0x82, 0x28, - 0xd4, 0x6e, 0x13, 0x7a, 0x69, 0x6f, 0x6a, 0x08, 0x18, 0x4a, 0x5b, 0xd6, 0x21, 0xa7, 0x82, 0xd1, - 0xcf, 0xda, 0x59, 0x22, 0x6b, 0x1d, 0xed, 0x3a, 0x41, 0x79, 0x92, 0x1e, 0xf3, 0x30, 0x3d, 0xe4, - 0xd8, 0x73, 0x0f, 0x26, 0x75, 0x2f, 0x7d, 0x8c, 0xb2, 0xbb, 0x92, 0xa2, 0xf6, 0xd4, 0xde, 0xe6, - 0x9b, 0x9f, 0x9d, 0x6f, 0x66, 0xbe, 0x85, 0x2e, 0x17, 0xc9, 0x3a, 0x10, 0x7c, 0xb4, 0x4a, 0x98, - 0x60, 0x68, 0x27, 0xf4, 0x7b, 0x2f, 0x16, 0x54, 0x5c, 0xac, 0xfd, 0x51, 0xc0, 0x96, 0xe3, 0x05, - 0x5b, 0xb0, 0xb1, 0x0a, 0xf9, 0xeb, 0xb9, 0x42, 0x0a, 0x28, 0x4b, 0x97, 0xf4, 0x5e, 0x97, 0xd2, - 0x79, 0x1a, 0x07, 0xe2, 0x82, 0xc6, 0x8b, 0x92, 0x15, 0x51, 0x5f, 0xbf, 0x10, 0xb0, 0x68, 0xec, - 0x93, 0x95, 0x2e, 0x1b, 0x5e, 0x41, 0xfb, 0x94, 0x46, 0xe4, 0x9c, 0x24, 0x9c, 0xb2, 0x18, 0xbd, - 0x84, 0xc6, 0xb5, 0x36, 0x2d, 0xc3, 0x36, 0x9c, 0xf6, 0xd1, 0xde, 0x28, 0x2f, 0x1a, 0x9d, 0x93, - 0x40, 0xb0, 0xc4, 0x35, 0xef, 0x37, 0x83, 0x0a, 0xce, 0xd3, 0xd0, 0x21, 0xd4, 0x43, 0x72, 0x4d, - 0x03, 0x62, 0xed, 0xd8, 0x86, 0xd3, 0xc1, 0x19, 0x42, 0x16, 0x34, 0x68, 0x7c, 0xed, 0x45, 0x34, - 0xb4, 0xaa, 0xb6, 0xe1, 0x34, 0x71, 0x0e, 0x87, 0xa7, 0xd0, 0xce, 0xda, 0xbd, 0xa7, 0x5c, 0xa0, - 0x57, 0xd0, 0xcc, 0xde, 0xe2, 0x96, 0x61, 0x57, 0x9d, 0xf6, 0xd1, 0x93, 0x51, 0xe8, 0x8f, 0x4a, - 0xac, 0xb2, 0x96, 0x45, 0xda, 0x1b, 0xf3, 0xcb, 0xdd, 0xa0, 0x32, 0x7c, 0x30, 0x61, 0x5f, 0x66, - 0x4d, 0xe2, 0x39, 0x3b, 0x4b, 0xd6, 0x71, 0xe0, 0x09, 0x12, 0x22, 0x04, 0x66, 0xec, 0x2d, 0x89, - 0xa2, 0xdf, 0xc2, 0xca, 0x46, 0xcf, 0xc1, 0x14, 0xe9, 0x4a, 0x33, 0xdc, 0x3d, 0x3a, 0x7c, 0x1c, - 0xa9, 0x28, 0x4f, 0x57, 0x04, 0xab, 0x1c, 0x59, 0xcf, 0xe9, 0x2d, 0x51, 0xa4, 0xab, 0x58, 0xd9, - 0xc8, 0x86, 0xf6, 0x8a, 0x24, 0x4b, 0xca, 0x35, 0x4b, 0xd3, 0x36, 0x9c, 0x2e, 0x2e, 0xbb, 0xd0, - 0x53, 0x80, 0x25, 0x0b, 0xe9, 0x9c, 0x92, 0x70, 0xc6, 0xad, 0x9a, 0xaa, 0x6d, 0xe5, 0x9e, 0xa9, - 0x5c, 0x46, 0x48, 0x22, 0x22, 0x48, 0x68, 0xd5, 0xf5, 0x32, 0x32, 0x88, 0x9c, 0xc7, 0x35, 0x35, - 0x64, 0xc4, 0xdd, 0xdd, 0x6e, 0x06, 0x80, 0xbd, 0x9b, 0x89, 0xf6, 0x16, 0x6b, 0x43, 0xcf, 0x60, - 0x37, 0x66, 0xb3, 0x32, 0x8f, 0xa6, 0x7a, 0xaa, 0x1b, 0xb3, 0x4f, 0x25, 0x26, 0xa5, 0x0b, 0xb6, - 0xfe, 0xed, 0x82, 0x3d, 0x68, 0x72, 0x72, 0xb5, 0x26, 0x71, 0x40, 0x2c, 0x50, 0xcc, 0x0b, 0x8c, - 0x06, 0xd0, 0x2e, 0xe6, 0x8a, 0xb9, 0xd5, 0xb6, 0x0d, 0xa7, 0x86, 0x8b, 0x51, 0x3f, 0x70, 0xf4, - 0xb9, 0x94, 0xe0, 0xa7, 0x56, 0xc7, 0x36, 0x1c, 0xd3, 0x7d, 0x2b, 0x1b, 0x7c, 0xdf, 0x0c, 0x8e, - 0xff, 0x43, 0x93, 0xa3, 0xe9, 0x05, 0x4b, 0xc4, 0xe4, 0xe4, 0xf1, 0x75, 0x37, 0x45, 0x63, 0x00, - 0x3f, 0x62, 0xc1, 0xe5, 0x4c, 0x9d, 0xa4, 0x2b, 0xbb, 0xbb, 0x7b, 0xdb, 0xcd, 0xa0, 0x83, 0xbd, - 0x1b, 0x57, 0x06, 0xa6, 0xf4, 0x96, 0xe0, 0x96, 0x9f, 0x9b, 0x72, 0x49, 0x3c, 0x5d, 0x46, 0x34, - 0xbe, 0x9c, 0x09, 0x2f, 0x59, 0x10, 0x61, 0xed, 0x2b, 0x1d, 0x74, 0x33, 0xef, 0x99, 0x72, 0xca, - 0x83, 0x46, 0x2c, 0xf0, 0xa2, 0xd9, 0x3c, 0xf2, 0x16, 0xdc, 0xfa, 0xd5, 0x50, 0x17, 0x05, 0xe5, - 0x3b, 0x95, 0xae, 0x4c, 0x62, 0x5f, 0x0d, 0xa8, 0xbf, 0x63, 0xeb, 0x58, 0x70, 0x74, 0x00, 0xb5, - 0x39, 0x8d, 0x08, 0x57, 0xc2, 0xaa, 0x61, 0x0d, 0xe4, 0x43, 0x21, 0x4d, 0xd4, 0x5a, 0x29, 0xe1, - 0x4a, 0x60, 0x35, 0x5c, 0x76, 0xa9, 0xed, 0xea, 0xde, 0x5c, 0x69, 0xaa, 0x86, 0x0b, 0x5c, 0x96, - 0x85, 0xa9, 0x42, 0x85, 0x2c, 0x0e, 0xa0, 0xe6, 0xa7, 0x82, 0xe4, 0x52, 0xd2, 0xe0, 0x8f, 0x4b, - 0xd5, 0xff, 0xba, 0x54, 0x0f, 0x9a, 0xfa, 0xe7, 0x4d, 0x4e, 0xd4, 0xcc, 0x1d, 0x5c, 0xe0, 0xe1, - 0x47, 0x68, 0xe9, 0x29, 0xa6, 0x44, 0x20, 0x07, 0xea, 0x81, 0x02, 0xd9, 0x6f, 0x03, 0xf9, 0xdb, - 0x74, 0x38, 0x53, 0x46, 0x16, 0x97, 0xf4, 0x82, 0x84, 0xc8, 0x5f, 0xa5, 0x06, 0xab, 0xe2, 0x1c, - 0xba, 0x07, 0xf7, 0x3f, 0xfa, 0x95, 0xfb, 0x6d, 0xdf, 0xf8, 0xb6, 0xed, 0x1b, 0x0f, 0xdb, 0x7e, - 0xe5, 0xee, 0x67, 0xdf, 0xf0, 0xeb, 0xea, 0x96, 0xc7, 0xbf, 0x03, 0x00, 0x00, 0xff, 0xff, 0x9a, - 0x4b, 0x16, 0x44, 0xcd, 0x04, 0x00, 0x00, + // 671 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x53, 0x4d, 0x6b, 0xdb, 0x4c, + 0x10, 0xb6, 0x62, 0xf9, 0x6b, 0x6c, 0xe7, 0x4d, 0x96, 0x10, 0x84, 0xe1, 0xb5, 0x85, 0xa1, 0x20, + 0x0a, 0xb5, 0xdb, 0x84, 0x5e, 0xda, 0x9b, 0x1a, 0x02, 0x86, 0xd2, 0x96, 0x75, 0xc8, 0xa9, 0x60, + 0xf4, 0xb1, 0x76, 0x96, 0xc8, 0x5a, 0x47, 0xbb, 0x4e, 0x50, 0x7e, 0x49, 0x8f, 0xf9, 0x39, 0x39, + 0xf6, 0xdc, 0x83, 0x49, 0xdd, 0x1e, 0xfa, 0x33, 0xca, 0xee, 0x4a, 0x8a, 0x9a, 0x53, 0x7b, 0x9b, + 0x67, 0x3e, 0x76, 0x9e, 0x99, 0x79, 0x16, 0xba, 0x5c, 0x24, 0xeb, 0x40, 0xf0, 0xd1, 0x2a, 0x61, + 0x82, 0xa1, 0x9d, 0xd0, 0xef, 0xbd, 0x58, 0x50, 0x71, 0xb1, 0xf6, 0x47, 0x01, 0x5b, 0x8e, 0x17, + 0x6c, 0xc1, 0xc6, 0x2a, 0xe4, 0xaf, 0xe7, 0x0a, 0x29, 0xa0, 0x2c, 0x5d, 0xd2, 0x7b, 0x5d, 0x4a, + 0xe7, 0x69, 0x1c, 0x88, 0x0b, 0x1a, 0x2f, 0x4a, 0x56, 0x44, 0x7d, 0xfd, 0x42, 0xc0, 0xa2, 0xb1, + 0x4f, 0x56, 0xba, 0x6c, 0x78, 0x05, 0xed, 0x53, 0x1a, 0x91, 0x73, 0x92, 0x70, 0xca, 0x62, 0xf4, + 0x12, 0x1a, 0xd7, 0xda, 0xb4, 0x0c, 0xdb, 0x70, 0xda, 0x47, 0x7b, 0xa3, 0xbc, 0x68, 0x74, 0x4e, + 0x02, 0xc1, 0x12, 0xd7, 0xbc, 0xdf, 0x0c, 0x2a, 0x38, 0x4f, 0x43, 0x87, 0x50, 0x0f, 0xc9, 0x35, + 0x0d, 0x88, 0xb5, 0x63, 0x1b, 0x4e, 0x07, 0x67, 0x08, 0x59, 0xd0, 0xa0, 0xf1, 0xb5, 0x17, 0xd1, + 0xd0, 0xaa, 0xda, 0x86, 0xd3, 0xc4, 0x39, 0x1c, 0x9e, 0x42, 0x3b, 0x6b, 0xf7, 0x9e, 0x72, 0x81, + 0x5e, 0x41, 0x33, 0x7b, 0x8b, 0x5b, 0x86, 0x5d, 0x75, 0xda, 0x47, 0xff, 0x8d, 0x42, 0x7f, 0x54, + 0x62, 0x95, 0xb5, 0x2c, 0xd2, 0xde, 0x98, 0x5f, 0xee, 0x06, 0x95, 0xe1, 0x83, 0x09, 0xfb, 0x32, + 0x6b, 0x12, 0xcf, 0xd9, 0x59, 0xb2, 0x8e, 0x03, 0x4f, 0x90, 0x10, 0x21, 0x30, 0x63, 0x6f, 0x49, + 0x14, 0xfd, 0x16, 0x56, 0x36, 0x7a, 0x0e, 0xa6, 0x48, 0x57, 0x9a, 0xe1, 0xee, 0xd1, 0xe1, 0xe3, + 0x48, 0x45, 0x79, 0xba, 0x22, 0x58, 0xe5, 0xc8, 0x7a, 0x4e, 0x6f, 0x89, 0x22, 0x5d, 0xc5, 0xca, + 0x46, 0x36, 0xb4, 0x57, 0x24, 0x59, 0x52, 0xae, 0x59, 0x9a, 0xb6, 0xe1, 0x74, 0x71, 0xd9, 0x85, + 0xfe, 0x07, 0x58, 0xb2, 0x90, 0xce, 0x29, 0x09, 0x67, 0xdc, 0xaa, 0xa9, 0xda, 0x56, 0xee, 0x99, + 0xca, 0x65, 0x84, 0x24, 0x22, 0x82, 0x84, 0x56, 0x5d, 0x2f, 0x23, 0x83, 0xc8, 0x79, 0x5c, 0x53, + 0x43, 0x46, 0xdc, 0xdd, 0xed, 0x66, 0x00, 0xd8, 0xbb, 0x99, 0x68, 0x6f, 0xb1, 0x36, 0xf4, 0x0c, + 0x76, 0x63, 0x36, 0x2b, 0xf3, 0x68, 0xaa, 0xa7, 0xba, 0x31, 0xfb, 0x54, 0x62, 0x52, 0xba, 0x60, + 0xeb, 0xef, 0x2e, 0xd8, 0x83, 0x26, 0x27, 0x57, 0x6b, 0x12, 0x07, 0xc4, 0x02, 0xc5, 0xbc, 0xc0, + 0x68, 0x00, 0xed, 0x62, 0xae, 0x98, 0x5b, 0x6d, 0xdb, 0x70, 0x6a, 0xb8, 0x18, 0xf5, 0x03, 0x47, + 0x9f, 0x4b, 0x09, 0x7e, 0x6a, 0x75, 0x6c, 0xc3, 0x31, 0xdd, 0xb7, 0xb2, 0xc1, 0xb7, 0xcd, 0xe0, + 0xf8, 0x1f, 0x34, 0x39, 0x9a, 0x5e, 0xb0, 0x44, 0x4c, 0x4e, 0x1e, 0x5f, 0x77, 0x53, 0x34, 0x06, + 0xf0, 0x23, 0x16, 0x5c, 0xce, 0xd4, 0x49, 0xba, 0xb2, 0xbb, 0xbb, 0xb7, 0xdd, 0x0c, 0x3a, 0xd8, + 0xbb, 0x71, 0x65, 0x60, 0x4a, 0x6f, 0x09, 0x6e, 0xf9, 0xb9, 0x29, 0x97, 0xc4, 0xd3, 0x65, 0x44, + 0xe3, 0xcb, 0x99, 0xf0, 0x92, 0x05, 0x11, 0xd6, 0xbe, 0xd2, 0x41, 0x37, 0xf3, 0x9e, 0x29, 0xa7, + 0x3c, 0x68, 0xc4, 0x02, 0x2f, 0x9a, 0xcd, 0x23, 0x6f, 0xc1, 0xad, 0x5f, 0x0d, 0x75, 0x51, 0x50, + 0xbe, 0x53, 0xe9, 0xca, 0x24, 0xf6, 0xd3, 0x80, 0xfa, 0x3b, 0xb6, 0x8e, 0x05, 0x47, 0x07, 0x50, + 0x9b, 0xd3, 0x88, 0x70, 0x25, 0xac, 0x1a, 0xd6, 0x40, 0x3e, 0x14, 0xd2, 0x44, 0xad, 0x95, 0x12, + 0xae, 0x04, 0x56, 0xc3, 0x65, 0x97, 0xda, 0xae, 0xee, 0xcd, 0x95, 0xa6, 0x6a, 0xb8, 0xc0, 0x65, + 0x59, 0x98, 0x2a, 0x54, 0xc8, 0xe2, 0x00, 0x6a, 0x7e, 0x2a, 0x48, 0x2e, 0x25, 0x0d, 0xfe, 0xb8, + 0x54, 0xfd, 0xc9, 0xa5, 0x7a, 0xd0, 0xd4, 0x3f, 0x6f, 0x72, 0xa2, 0x66, 0xee, 0xe0, 0x02, 0xa3, + 0x3e, 0x94, 0x46, 0xb3, 0xd0, 0xd3, 0x61, 0x87, 0x1f, 0xa1, 0xa5, 0xa7, 0x9c, 0x12, 0x81, 0x1c, + 0xa8, 0x07, 0x0a, 0x64, 0xbf, 0x11, 0xe4, 0x6f, 0xd4, 0xe1, 0x4c, 0x39, 0x59, 0x5c, 0xd2, 0x0f, + 0x12, 0x22, 0x7f, 0x9d, 0x1a, 0xbc, 0x8a, 0x73, 0xe8, 0x1e, 0xdc, 0x7f, 0xef, 0x57, 0xee, 0xb7, + 0x7d, 0xe3, 0xeb, 0xb6, 0x6f, 0x3c, 0x6c, 0xfb, 0x95, 0xbb, 0x1f, 0x7d, 0xc3, 0xaf, 0xab, 0x5b, + 0x1f, 0xff, 0x0e, 0x00, 0x00, 0xff, 0xff, 0xc4, 0x4d, 0xd7, 0x14, 0xed, 0x04, 0x00, 0x00, } diff --git a/lib/db/structs.proto b/lib/db/structs.proto index 03309f1cc..9794adb32 100644 --- a/lib/db/structs.proto +++ b/lib/db/structs.proto @@ -46,13 +46,14 @@ message FileInfoTruncated { // For each folder and device we keep one of these to track the current // counts and sequence. We also keep one for the global state of the folder. message Counts { - int32 files = 1; - int32 directories = 2; - int32 symlinks = 3; - int32 deleted = 4; - int64 bytes = 5; - int64 sequence = 6; // zero for the global state - bytes deviceID = 17; // device ID for remote devices, or special values for local/global + int32 files = 1; + int32 directories = 2; + int32 symlinks = 3; + int32 deleted = 4; + int64 bytes = 5; + int64 sequence = 6; // zero for the global state + bytes deviceID = 17; // device ID for remote devices, or special values for local/global + uint32 localFlags = 18; // the local flag for this count bucket } message CountsSet { diff --git a/lib/model/folder.go b/lib/model/folder.go index fd425f598..9716e272e 100644 --- a/lib/model/folder.go +++ b/lib/model/folder.go @@ -27,6 +27,7 @@ var errWatchNotStarted = errors.New("not started") type folder struct { stateTracker config.FolderConfiguration + localFlags uint32 model *Model shortID protocol.ShortID @@ -175,6 +176,8 @@ func (f *folder) BringToFront(string) {} func (f *folder) Override(fs *db.FileSet, updateFn func([]protocol.FileInfo)) {} +func (f *folder) Revert(fs *db.FileSet, updateFn func([]protocol.FileInfo)) {} + func (f *folder) DelayScan(next time.Duration) { f.Delay(next) } @@ -263,7 +266,7 @@ func (f *folder) getHealthError() error { } func (f *folder) scanSubdirs(subDirs []string) error { - if err := f.model.internalScanFolderSubdirs(f.ctx, f.folderID, subDirs); err != nil { + if err := f.model.internalScanFolderSubdirs(f.ctx, f.folderID, subDirs, f.localFlags); err != nil { // Potentially sets the error twice, once in the scanner just // by doing a check, and once here, if the error returned is // the same one as returned by CheckHealth, though diff --git a/lib/model/folder_recvonly.go b/lib/model/folder_recvonly.go new file mode 100644 index 000000000..7638b9437 --- /dev/null +++ b/lib/model/folder_recvonly.go @@ -0,0 +1,210 @@ +// Copyright (C) 2018 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at https://mozilla.org/MPL/2.0/. + +package model + +import ( + "sort" + "time" + + "github.com/syncthing/syncthing/lib/config" + "github.com/syncthing/syncthing/lib/db" + "github.com/syncthing/syncthing/lib/fs" + "github.com/syncthing/syncthing/lib/ignore" + "github.com/syncthing/syncthing/lib/protocol" + "github.com/syncthing/syncthing/lib/versioner" +) + +func init() { + folderFactories[config.FolderTypeReceiveOnly] = newReceiveOnlyFolder +} + +/* +receiveOnlyFolder is a folder that does not propagate local changes outward. +It does this by the following general mechanism (not all of which is +implemted in this file): + +- Local changes are scanned and versioned as usual, but get the + FlagLocalReceiveOnly bit set. + +- When changes are sent to the cluster this bit gets converted to the + Invalid bit (like all other local flags, currently) and also the Version + gets set to the empty version. The reason for clearing the Version is to + ensure that other devices will not consider themselves out of date due to + our change. + +- The database layer accounts sizes per flag bit, so we can know how many + files have been changed locally. We use this to trigger a "Revert" option + on the folder when the amount of locally changed data is nonzero. + +- To revert we take the files which have changed and reset their version + counter down to zero. The next pull will replace our changed version with + the globally latest. As this is a user-initiated operation we do not cause + conflict copies when reverting. + +- When pulling normally (i.e., not in the revert case) with local changes, + normal conflict resolution will apply. Conflict copies will be created, + but not propagated outwards (because receive only, right). + +Implementation wise a receiveOnlyFolder is just a sendReceiveFolder that +sets an extra bit on local changes and has a Revert method. +*/ +type receiveOnlyFolder struct { + *sendReceiveFolder +} + +func newReceiveOnlyFolder(model *Model, cfg config.FolderConfiguration, ver versioner.Versioner, fs fs.Filesystem) service { + sr := newSendReceiveFolder(model, cfg, ver, fs).(*sendReceiveFolder) + sr.localFlags = protocol.FlagLocalReceiveOnly // gets propagated to the scanner, and set on locally changed files + return &receiveOnlyFolder{sr} +} + +func (f *receiveOnlyFolder) Revert(fs *db.FileSet, updateFn func([]protocol.FileInfo)) { + f.setState(FolderScanning) + defer f.setState(FolderIdle) + + // XXX: This *really* should be given to us in the constructor... + f.model.fmut.RLock() + ignores := f.model.folderIgnores[f.folderID] + f.model.fmut.RUnlock() + + delQueue := &deleteQueue{ + handler: f, // for the deleteFile and deleteDir methods + ignores: ignores, + } + + batch := make([]protocol.FileInfo, 0, maxBatchSizeFiles) + batchSizeBytes := 0 + fs.WithHave(protocol.LocalDeviceID, func(intf db.FileIntf) bool { + fi := intf.(protocol.FileInfo) + if !fi.IsReceiveOnlyChanged() { + // We're only interested in files that have changed locally in + // receive only mode. + return true + } + + if len(fi.Version.Counters) == 1 && fi.Version.Counters[0].ID == f.shortID { + // We are the only device mentioned in the version vector so the + // file must originate here. A revert then means to delete it. + // We'll delete files directly, directories get queued and + // handled below. + + handled, err := delQueue.handle(fi) + if err != nil { + l.Infof("Revert: deleting %s: %v\n", fi.Name, err) + return true // continue + } + if !handled { + return true // continue + } + + fi = protocol.FileInfo{ + Name: fi.Name, + Type: fi.Type, + ModifiedS: fi.ModifiedS, + ModifiedNs: fi.ModifiedNs, + ModifiedBy: f.shortID, + Deleted: true, + Version: protocol.Vector{}, // if this file ever resurfaces anywhere we want our delete to be strictly older + } + } else { + // Revert means to throw away our local changes. We reset the + // version to the empty vector, which is strictly older than any + // other existing version. It is not in conflict with anything, + // either, so we will not create a conflict copy of our local + // changes. + fi.Version = protocol.Vector{} + fi.LocalFlags &^= protocol.FlagLocalReceiveOnly + } + + batch = append(batch, fi) + batchSizeBytes += fi.ProtoSize() + + if len(batch) >= maxBatchSizeFiles || batchSizeBytes >= maxBatchSizeBytes { + updateFn(batch) + batch = batch[:0] + batchSizeBytes = 0 + } + return true + }) + if len(batch) > 0 { + updateFn(batch) + } + batch = batch[:0] + batchSizeBytes = 0 + + // Handle any queued directories + deleted, err := delQueue.flush() + if err != nil { + l.Infoln("Revert:", err) + } + now := time.Now() + for _, dir := range deleted { + batch = append(batch, protocol.FileInfo{ + Name: dir, + Type: protocol.FileInfoTypeDirectory, + ModifiedS: now.Unix(), + ModifiedBy: f.shortID, + Deleted: true, + Version: protocol.Vector{}, + }) + } + if len(batch) > 0 { + updateFn(batch) + } + + // We will likely have changed our local index, but that won't trigger a + // pull by itself. Make sure we schedule one so that we start + // downloading files. + f.SchedulePull() +} + +// deleteQueue handles deletes by delegating to a handler and queuing +// directories for last. +type deleteQueue struct { + handler interface { + deleteFile(file protocol.FileInfo) (dbUpdateJob, error) + deleteDir(dir string, ignores *ignore.Matcher, scanChan chan<- string) error + } + ignores *ignore.Matcher + dirs []string +} + +func (q *deleteQueue) handle(fi protocol.FileInfo) (bool, error) { + // Things that are ignored but not marked deletable are not processed. + ign := q.ignores.Match(fi.Name) + if ign.IsIgnored() && !ign.IsDeletable() { + return false, nil + } + + // Directories are queued for later processing. + if fi.IsDirectory() { + q.dirs = append(q.dirs, fi.Name) + return false, nil + } + + // Kill it. + _, err := q.handler.deleteFile(fi) + return true, err +} + +func (q *deleteQueue) flush() ([]string, error) { + // Process directories from the leaves inward. + sort.Sort(sort.Reverse(sort.StringSlice(q.dirs))) + + var firstError error + var deleted []string + + for _, dir := range q.dirs { + if err := q.handler.deleteDir(dir, q.ignores, nil); err == nil { + deleted = append(deleted, dir) + } else if err != nil && firstError == nil { + firstError = err + } + } + + return deleted, firstError +} diff --git a/lib/model/folder_recvonly_test.go b/lib/model/folder_recvonly_test.go new file mode 100644 index 000000000..99591d410 --- /dev/null +++ b/lib/model/folder_recvonly_test.go @@ -0,0 +1,261 @@ +// Copyright (C) 2018 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at https://mozilla.org/MPL/2.0/. + +package model + +import ( + "io/ioutil" + "os" + "testing" + "time" + + "github.com/syncthing/syncthing/lib/config" + "github.com/syncthing/syncthing/lib/db" + "github.com/syncthing/syncthing/lib/fs" + "github.com/syncthing/syncthing/lib/protocol" +) + +func TestRecvOnlyRevertDeletes(t *testing.T) { + // Make sure that we delete extraneous files and directories when we hit + // Revert. + + os.RemoveAll("_recvonly") + defer os.RemoveAll("_recvonly") + + // Create some test data + + os.MkdirAll("_recvonly/.stfolder", 0755) + os.MkdirAll("_recvonly/ignDir", 0755) + os.MkdirAll("_recvonly/unknownDir", 0755) + ioutil.WriteFile("_recvonly/ignDir/ignFile", []byte("hello\n"), 0644) + ioutil.WriteFile("_recvonly/unknownDir/unknownFile", []byte("hello\n"), 0644) + ioutil.WriteFile("_recvonly/.stignore", []byte("ignDir\n"), 0644) + + knownFiles := setupKnownFiles(t, []byte("hello\n")) + + // Get us a model up and running + + m := setupROFolder() + defer m.Stop() + + // Send and index update for the known stuff + + m.Index(device1, "ro", knownFiles) + m.updateLocalsFromScanning("ro", knownFiles) + + size := m.GlobalSize("ro") + if size.Files != 1 || size.Directories != 1 { + t.Fatalf("Global: expected 1 file and 1 directory: %+v", size) + } + + // Start the folder. This will cause a scan, should discover the other stuff in the folder + + m.StartFolder("ro") + m.ScanFolder("ro") + + // We should now have two files and two directories. + + size = m.GlobalSize("ro") + if size.Files != 2 || size.Directories != 2 { + t.Fatalf("Global: expected 2 files and 2 directories: %+v", size) + } + size = m.LocalSize("ro") + if size.Files != 2 || size.Directories != 2 { + t.Fatalf("Local: expected 2 files and 2 directories: %+v", size) + } + size = m.ReceiveOnlyChangedSize("ro") + if size.Files+size.Directories == 0 { + t.Fatalf("ROChanged: expected something: %+v", size) + } + + // Revert should delete the unknown stuff + + m.Revert("ro") + + // These should still exist + for _, p := range []string{"_recvonly/knownDir/knownFile", "_recvonly/ignDir/ignFile"} { + _, err := os.Stat(p) + if err != nil { + t.Error("Unexpected error:", err) + } + } + + // These should have been removed + for _, p := range []string{"_recvonly/unknownDir", "_recvonly/unknownDir/unknownFile"} { + _, err := os.Stat(p) + if !os.IsNotExist(err) { + t.Error("Unexpected existing thing:", p) + } + } + + // We should now have one file and directory again. + + size = m.GlobalSize("ro") + if size.Files != 1 || size.Directories != 1 { + t.Fatalf("Global: expected 1 files and 1 directories: %+v", size) + } + size = m.LocalSize("ro") + if size.Files != 1 || size.Directories != 1 { + t.Fatalf("Local: expected 1 files and 1 directories: %+v", size) + } +} + +func TestRecvOnlyRevertNeeds(t *testing.T) { + // Make sure that a new file gets picked up and considered latest, then + // gets considered old when we hit Revert. + + os.RemoveAll("_recvonly") + defer os.RemoveAll("_recvonly") + + // Create some test data + + os.MkdirAll("_recvonly/.stfolder", 0755) + oldData := []byte("hello\n") + knownFiles := setupKnownFiles(t, oldData) + + // Get us a model up and running + + m := setupROFolder() + defer m.Stop() + + // Send and index update for the known stuff + + m.Index(device1, "ro", knownFiles) + m.updateLocalsFromScanning("ro", knownFiles) + + // Start the folder. This will cause a scan. + + m.StartFolder("ro") + m.ScanFolder("ro") + + // Everything should be in sync. + + size := m.GlobalSize("ro") + if size.Files != 1 || size.Directories != 1 { + t.Fatalf("Global: expected 1 file and 1 directory: %+v", size) + } + size = m.LocalSize("ro") + if size.Files != 1 || size.Directories != 1 { + t.Fatalf("Local: expected 1 file and 1 directory: %+v", size) + } + size = m.NeedSize("ro") + if size.Files+size.Directories > 0 { + t.Fatalf("Need: expected nothing: %+v", size) + } + size = m.ReceiveOnlyChangedSize("ro") + if size.Files+size.Directories > 0 { + t.Fatalf("ROChanged: expected nothing: %+v", size) + } + + // Update the file. + + newData := []byte("totally different data\n") + if err := ioutil.WriteFile("_recvonly/knownDir/knownFile", newData, 0644); err != nil { + t.Fatal(err) + } + + // Rescan. + + if err := m.ScanFolder("ro"); err != nil { + t.Fatal(err) + } + + // We now have a newer file than the rest of the cluster. Global state should reflect this. + + size = m.GlobalSize("ro") + const sizeOfDir = 128 + if size.Files != 1 || size.Bytes != sizeOfDir+int64(len(newData)) { + t.Fatalf("Global: expected the new file to be reflected: %+v", size) + } + size = m.LocalSize("ro") + if size.Files != 1 || size.Bytes != sizeOfDir+int64(len(newData)) { + t.Fatalf("Local: expected the new file to be reflected: %+v", size) + } + size = m.NeedSize("ro") + if size.Files+size.Directories > 0 { + t.Fatalf("Need: expected nothing: %+v", size) + } + size = m.ReceiveOnlyChangedSize("ro") + if size.Files+size.Directories == 0 { + t.Fatalf("ROChanged: expected something: %+v", size) + } + + // We hit the Revert button. The file that was new should become old. + + m.Revert("ro") + + size = m.GlobalSize("ro") + if size.Files != 1 || size.Bytes != sizeOfDir+int64(len(oldData)) { + t.Fatalf("Global: expected the global size to revert: %+v", size) + } + size = m.LocalSize("ro") + if size.Files != 1 || size.Bytes != sizeOfDir+int64(len(newData)) { + t.Fatalf("Local: expected the local size to remain: %+v", size) + } + size = m.NeedSize("ro") + if size.Files != 1 || size.Bytes != int64(len(oldData)) { + t.Fatalf("Local: expected to need the old file data: %+v", size) + } +} + +func setupKnownFiles(t *testing.T, data []byte) []protocol.FileInfo { + if err := os.MkdirAll("_recvonly/knownDir", 0755); err != nil { + t.Fatal(err) + } + if err := ioutil.WriteFile("_recvonly/knownDir/knownFile", data, 0644); err != nil { + t.Fatal(err) + } + + t0 := time.Now().Add(-1 * time.Minute) + if err := os.Chtimes("_recvonly/knownDir/knownFile", t0, t0); err != nil { + t.Fatal(err) + } + + fi, err := os.Stat("_recvonly/knownDir/knownFile") + if err != nil { + t.Fatal(err) + } + knownFiles := []protocol.FileInfo{ + { + Name: "knownDir", + Type: protocol.FileInfoTypeDirectory, + Permissions: 0755, + Version: protocol.Vector{Counters: []protocol.Counter{{ID: 42, Value: 42}}}, + Sequence: 42, + }, + { + Name: "knownDir/knownFile", + Type: protocol.FileInfoTypeFile, + Permissions: 0644, + Size: fi.Size(), + ModifiedS: fi.ModTime().Unix(), + ModifiedNs: int32(fi.ModTime().UnixNano() % 1e9), + Version: protocol.Vector{Counters: []protocol.Counter{{ID: 42, Value: 42}}}, + Sequence: 42, + }, + } + + return knownFiles +} + +func setupROFolder() *Model { + fcfg := config.NewFolderConfiguration(protocol.LocalDeviceID, "ro", "receive only test", fs.FilesystemTypeBasic, "_recvonly") + fcfg.Type = config.FolderTypeReceiveOnly + fcfg.Devices = []config.FolderDeviceConfiguration{{DeviceID: device1}} + + cfg := defaultCfg.Copy() + cfg.Folders = append(cfg.Folders, fcfg) + + wrp := config.Wrap("/dev/null", cfg) + + db := db.OpenMemory() + m := NewModel(wrp, protocol.LocalDeviceID, "syncthing", "dev", db, nil) + + m.ServeBackground() + m.AddFolder(fcfg) + + return m +} diff --git a/lib/model/folder_sendonly.go b/lib/model/folder_sendonly.go index d3a7304ca..bd4a7ac7e 100644 --- a/lib/model/folder_sendonly.go +++ b/lib/model/folder_sendonly.go @@ -76,7 +76,7 @@ func (f *sendOnlyFolder) pull() bool { } file := intf.(protocol.FileInfo) - if !file.IsEquivalent(curFile, f.IgnorePerms, false) { + if !file.IsEquivalentOptional(curFile, f.IgnorePerms, false, 0) { return true } diff --git a/lib/model/folder_sendrecv.go b/lib/model/folder_sendrecv.go index 7635db9ad..998fb99b5 100644 --- a/lib/model/folder_sendrecv.go +++ b/lib/model/folder_sendrecv.go @@ -501,7 +501,11 @@ func (f *sendReceiveFolder) processDeletions(ignores *ignore.Matcher, fileDeleti } l.Debugln(f, "Deleting file", file.Name) - f.deleteFile(file, dbUpdateChan) + if update, err := f.deleteFile(file); err != nil { + f.newError("delete file", file.Name, err) + } else { + dbUpdateChan <- update + } } for i := range dirDeletions { @@ -736,7 +740,7 @@ func (f *sendReceiveFolder) handleDeleteDir(file protocol.FileInfo, ignores *ign } // deleteFile attempts to delete the given file -func (f *sendReceiveFolder) deleteFile(file protocol.FileInfo, dbUpdateChan chan<- dbUpdateJob) { +func (f *sendReceiveFolder) deleteFile(file protocol.FileInfo) (dbUpdateJob, error) { // Used in the defer closure below, updated by the function body. Take // care not declare another err. var err error @@ -775,16 +779,18 @@ func (f *sendReceiveFolder) deleteFile(file protocol.FileInfo, dbUpdateChan chan if err == nil || fs.IsNotExist(err) { // It was removed or it doesn't exist to start with - dbUpdateChan <- dbUpdateJob{file, dbUpdateDeleteFile} - } else if _, serr := f.fs.Lstat(file.Name); serr != nil && !fs.IsPermission(serr) { + return dbUpdateJob{file, dbUpdateDeleteFile}, nil + } + + if _, serr := f.fs.Lstat(file.Name); serr != nil && !fs.IsPermission(serr) { // We get an error just looking at the file, and it's not a permission // problem. Lets assume the error is in fact some variant of "file // does not exist" (possibly expressed as some parent being a file and // not a directory etc) and that the delete is handled. - dbUpdateChan <- dbUpdateJob{file, dbUpdateDeleteFile} - } else { - f.newError("delete file", file.Name, err) + return dbUpdateJob{file, dbUpdateDeleteFile}, nil } + + return dbUpdateJob{}, err } // renameFile attempts to rename an existing file to a destination @@ -1778,10 +1784,14 @@ func (f *sendReceiveFolder) deleteDir(dir string, ignores *ignore.Matcher, scanC } else if ignores != nil && ignores.Match(fullDirFile).IsIgnored() { hasIgnored = true } else if cf, ok := f.model.CurrentFolderFile(f.ID, fullDirFile); !ok || cf.IsDeleted() || cf.IsInvalid() { - // Something appeared in the dir that we either are not - // aware of at all, that we think should be deleted or that - // is invalid, but not currently ignored -> schedule scan - scanChan <- fullDirFile + // Something appeared in the dir that we either are not aware of + // at all, that we think should be deleted or that is invalid, + // but not currently ignored -> schedule scan. The scanChan + // might be nil, in which case we trust the scanning to be + // handled later as a result of our error return. + if scanChan != nil { + scanChan <- fullDirFile + } hasToBeScanned = true } else { // Dir contains file that is valid according to db and diff --git a/lib/model/folder_sendrecv_test.go b/lib/model/folder_sendrecv_test.go index 1af9fb3f7..b3b007836 100644 --- a/lib/model/folder_sendrecv_test.go +++ b/lib/model/folder_sendrecv_test.go @@ -74,12 +74,12 @@ func setUpFile(filename string, blockNumbers []int) protocol.FileInfo { } } -func setUpModel(file protocol.FileInfo) *Model { +func setUpModel(files ...protocol.FileInfo) *Model { db := db.OpenMemory() model := NewModel(defaultCfgWrapper, protocol.LocalDeviceID, "syncthing", "dev", db, nil) model.AddFolder(defaultFolderConfig) // Update index - model.updateLocalsFromScanning("default", []protocol.FileInfo{file}) + model.updateLocalsFromScanning("default", files) return model } diff --git a/lib/model/model.go b/lib/model/model.go index 6ce39b1c8..e0b06ef18 100644 --- a/lib/model/model.go +++ b/lib/model/model.go @@ -57,6 +57,7 @@ const ( type service interface { BringToFront(string) Override(*db.FileSet, func([]protocol.FileInfo)) + Revert(*db.FileSet, func([]protocol.FileInfo)) DelayScan(d time.Duration) IgnoresUpdated() // ignore matcher was updated notification SchedulePull() // something relevant changed, we should try a pull @@ -690,6 +691,18 @@ func (m *Model) LocalSize(folder string) db.Counts { return db.Counts{} } +// ReceiveOnlyChangedSize returns the number of files, deleted files and +// total bytes for all files that have changed locally in a receieve only +// folder. +func (m *Model) ReceiveOnlyChangedSize(folder string) db.Counts { + m.fmut.RLock() + defer m.fmut.RUnlock() + if rf, ok := m.folderFiles[folder]; ok { + return rf.ReceiveOnlyChangedSize() + } + return db.Counts{} +} + // NeedSize returns the number and total size of currently needed files. func (m *Model) NeedSize(folder string) db.Counts { m.fmut.RLock() @@ -1747,6 +1760,12 @@ func sendIndexTo(prevSequence int64, conn protocol.Connection, folder string, fs // Mark the file as invalid if any of the local bad stuff flags are set. f.RawInvalid = f.IsInvalid() + // If the file is marked LocalReceive (i.e., changed locally on a + // receive only folder) we do not want it to ever become the + // globally best version, invalid or not. + if f.IsReceiveOnlyChanged() { + f.Version = protocol.Vector{} + } f.LocalFlags = 0 // never sent externally if dropSymlinks && f.IsSymlink() { @@ -1940,7 +1959,7 @@ func (m *Model) ScanFolderSubdirs(folder string, subs []string) error { return runner.Scan(subs) } -func (m *Model) internalScanFolderSubdirs(ctx context.Context, folder string, subDirs []string) error { +func (m *Model) internalScanFolderSubdirs(ctx context.Context, folder string, subDirs []string, localFlags uint32) error { m.fmut.RLock() if err := m.checkFolderRunningLocked(folder); err != nil { m.fmut.RUnlock() @@ -2010,6 +2029,7 @@ func (m *Model) internalScanFolderSubdirs(ctx context.Context, folder string, su ShortID: m.shortID, ProgressTickIntervalS: folderCfg.ScanProgressIntervalS, UseLargeBlocks: folderCfg.UseLargeBlocks, + LocalFlags: localFlags, }) if err := runner.CheckHealth(); err != nil { @@ -2106,6 +2126,7 @@ func (m *Model) internalScanFolderSubdirs(ctx context.Context, folder string, su ModifiedBy: m.id.Short(), Deleted: true, Version: f.Version.Update(m.shortID), + LocalFlags: localFlags, } // We do not want to override the global version // with the deleted file. Keeping only our local @@ -2289,6 +2310,24 @@ func (m *Model) Override(folder string) { }) } +func (m *Model) Revert(folder string) { + // Grab the runner and the file set. + + m.fmut.RLock() + fs, fsOK := m.folderFiles[folder] + runner, runnerOK := m.folderRunners[folder] + m.fmut.RUnlock() + if !fsOK || !runnerOK { + return + } + + // Run the revert, taking updates as if they came from scanning. + + runner.Revert(fs, func(files []protocol.FileInfo) { + m.updateLocalsFromScanning(folder, files) + }) +} + // CurrentSequence returns the change version for the given folder. // This is guaranteed to increment if the contents of the local folder has // changed. diff --git a/lib/protocol/bep_extensions.go b/lib/protocol/bep_extensions.go index 16c64a058..11cfc7d83 100644 --- a/lib/protocol/bep_extensions.go +++ b/lib/protocol/bep_extensions.go @@ -49,6 +49,10 @@ func (f FileInfo) IsInvalid() bool { return f.RawInvalid || f.LocalFlags&LocalInvalidFlags != 0 } +func (f FileInfo) IsUnsupported() bool { + return f.LocalFlags&FlagLocalUnsupported != 0 +} + func (f FileInfo) IsIgnored() bool { return f.LocalFlags&FlagLocalIgnored != 0 } @@ -57,6 +61,10 @@ func (f FileInfo) MustRescan() bool { return f.LocalFlags&FlagLocalMustRescan != 0 } +func (f FileInfo) IsReceiveOnlyChanged() bool { + return f.LocalFlags&FlagLocalReceiveOnly != 0 +} + func (f FileInfo) IsDirectory() bool { return f.Type == FileInfoTypeDirectory } @@ -99,6 +107,10 @@ func (f FileInfo) FileName() string { return f.Name } +func (f FileInfo) FileLocalFlags() uint32 { + return f.LocalFlags +} + func (f FileInfo) ModTime() time.Time { return time.Unix(f.ModifiedS, int64(f.ModifiedNs)) } @@ -114,7 +126,7 @@ func (f FileInfo) FileVersion() Vector { // WinsConflict returns true if "f" is the one to choose when it is in // conflict with "other". func (f FileInfo) WinsConflict(other FileInfo) bool { - // If only one of the files is invalid, that one loses + // If only one of the files is invalid, that one loses. if f.IsInvalid() != other.IsInvalid() { return !f.IsInvalid() } @@ -145,7 +157,15 @@ func (f FileInfo) IsEmpty() bool { return f.Version.Counters == nil } -// IsEquivalent checks that the two file infos represent the same actual file content, +func (f FileInfo) IsEquivalent(other FileInfo) bool { + return f.isEquivalent(other, false, false, 0) +} + +func (f FileInfo) IsEquivalentOptional(other FileInfo, ignorePerms bool, ignoreBlocks bool, ignoreFlags uint32) bool { + return f.isEquivalent(other, ignorePerms, ignoreBlocks, ignoreFlags) +} + +// isEquivalent checks that the two file infos represent the same actual file content, // i.e. it does purposely not check only selected (see below) struct members. // Permissions (config) and blocks (scanning) can be excluded from the comparison. // Any file info is not "equivalent", if it has different @@ -160,7 +180,7 @@ func (f FileInfo) IsEmpty() bool { // A symlink is not "equivalent", if it has different // - target // A directory does not have anything specific to check. -func (f FileInfo) IsEquivalent(other FileInfo, ignorePerms bool, ignoreBlocks bool) bool { +func (f FileInfo) isEquivalent(other FileInfo, ignorePerms bool, ignoreBlocks bool, ignoreFlags uint32) bool { if f.MustRescan() || other.MustRescan() { // These are per definition not equivalent because they don't // represent a valid state, even if both happen to have the @@ -168,6 +188,10 @@ func (f FileInfo) IsEquivalent(other FileInfo, ignorePerms bool, ignoreBlocks bo return false } + // Mask out the ignored local flags before checking IsInvalid() below + f.LocalFlags &^= ignoreFlags + other.LocalFlags &^= ignoreFlags + if f.Name != other.Name || f.Type != other.Type || f.Deleted != other.Deleted || f.IsInvalid() != other.IsInvalid() { return false } diff --git a/lib/protocol/deviceid.go b/lib/protocol/deviceid.go index c2a30b286..015080387 100644 --- a/lib/protocol/deviceid.go +++ b/lib/protocol/deviceid.go @@ -19,10 +19,18 @@ type DeviceID [DeviceIDLength]byte type ShortID uint64 var ( - LocalDeviceID = DeviceID{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff} - EmptyDeviceID = DeviceID{ /* all zeroes */ } + LocalDeviceID = repeatedDeviceID(0xff) + GlobalDeviceID = repeatedDeviceID(0xf8) + EmptyDeviceID = DeviceID{ /* all zeroes */ } ) +func repeatedDeviceID(v byte) (d DeviceID) { + for i := range d { + d[i] = v + } + return +} + // NewDeviceID generates a new device ID from the raw bytes of a certificate func NewDeviceID(rawCert []byte) DeviceID { var n DeviceID diff --git a/lib/protocol/protocol.go b/lib/protocol/protocol.go index cdd20dd51..9db4316f3 100644 --- a/lib/protocol/protocol.go +++ b/lib/protocol/protocol.go @@ -94,14 +94,15 @@ const ( FlagLocalUnsupported = 1 << 0 // The kind is unsupported, e.g. symlinks on Windows FlagLocalIgnored = 1 << 1 // Matches local ignore patterns FlagLocalMustRescan = 1 << 2 // Doesn't match content on disk, must be rechecked fully + FlagLocalReceiveOnly = 1 << 3 // Change detected on receive only folder // Flags that should result in the Invalid bit on outgoing updates - LocalInvalidFlags = FlagLocalUnsupported | FlagLocalIgnored | FlagLocalMustRescan + LocalInvalidFlags = FlagLocalUnsupported | FlagLocalIgnored | FlagLocalMustRescan | FlagLocalReceiveOnly // Flags that should result in a file being in conflict with its // successor, due to us not having an up to date picture of its state on // disk. - LocalConflictFlags = FlagLocalUnsupported | FlagLocalIgnored + LocalConflictFlags = FlagLocalUnsupported | FlagLocalIgnored | FlagLocalReceiveOnly ) var ( diff --git a/lib/protocol/protocol_test.go b/lib/protocol/protocol_test.go index 1ed48ab3c..2108a9733 100644 --- a/lib/protocol/protocol_test.go +++ b/lib/protocol/protocol_test.go @@ -423,6 +423,7 @@ func TestIsEquivalent(t *testing.T) { b FileInfo ignPerms *bool // nil means should not matter, we'll test both variants ignBlocks *bool + ignFlags uint32 eq bool } cases := []testCase{ @@ -491,6 +492,17 @@ func TestIsEquivalent(t *testing.T) { b: FileInfo{LocalFlags: FlagLocalUnsupported}, eq: true, }, + { + a: FileInfo{LocalFlags: 0}, + b: FileInfo{LocalFlags: FlagLocalReceiveOnly}, + eq: false, + }, + { + a: FileInfo{LocalFlags: 0}, + b: FileInfo{LocalFlags: FlagLocalReceiveOnly}, + ignFlags: FlagLocalReceiveOnly, + eq: true, + }, // Difference in blocks is not OK { @@ -588,10 +600,10 @@ func TestIsEquivalent(t *testing.T) { continue } - if res := tc.a.IsEquivalent(tc.b, ignPerms, ignBlocks); res != tc.eq { + if res := tc.a.isEquivalent(tc.b, ignPerms, ignBlocks, tc.ignFlags); res != tc.eq { t.Errorf("Case %d:\na: %v\nb: %v\na.IsEquivalent(b, %v, %v) => %v, expected %v", i, tc.a, tc.b, ignPerms, ignBlocks, res, tc.eq) } - if res := tc.b.IsEquivalent(tc.a, ignPerms, ignBlocks); res != tc.eq { + if res := tc.b.isEquivalent(tc.a, ignPerms, ignBlocks, tc.ignFlags); res != tc.eq { t.Errorf("Case %d:\na: %v\nb: %v\nb.IsEquivalent(a, %v, %v) => %v, expected %v", i, tc.a, tc.b, ignPerms, ignBlocks, res, tc.eq) } } diff --git a/lib/scanner/walk.go b/lib/scanner/walk.go index 37e2af4f7..76a55cb4a 100644 --- a/lib/scanner/walk.go +++ b/lib/scanner/walk.go @@ -68,6 +68,8 @@ type Config struct { ProgressTickIntervalS int // Whether to use large blocks for large files or the old standard of 128KiB for everything. UseLargeBlocks bool + // Local flags to set on scanned files + LocalFlags uint32 } type CurrentFiler interface { @@ -367,10 +369,11 @@ func (w *walker) walkRegular(ctx context.Context, relPath string, info fs.FileIn ModifiedBy: w.ShortID, Size: info.Size(), RawBlockSize: int32(blockSize), + LocalFlags: w.LocalFlags, } if hasCurFile { - if curFile.IsEquivalent(f, w.IgnorePerms, true) { + if curFile.IsEquivalentOptional(f, w.IgnorePerms, true, w.LocalFlags) { return nil } if curFile.ShouldConflict() { @@ -407,10 +410,11 @@ func (w *walker) walkDir(ctx context.Context, relPath string, info fs.FileInfo, ModifiedS: info.ModTime().Unix(), ModifiedNs: int32(info.ModTime().Nanosecond()), ModifiedBy: w.ShortID, + LocalFlags: w.LocalFlags, } if ok { - if cf.IsEquivalent(f, w.IgnorePerms, true) { + if cf.IsEquivalentOptional(f, w.IgnorePerms, true, w.LocalFlags) { return nil } if cf.ShouldConflict() { @@ -463,10 +467,11 @@ func (w *walker) walkSymlink(ctx context.Context, relPath string, dchan chan pro NoPermissions: true, // Symlinks don't have permissions of their own SymlinkTarget: target, ModifiedBy: w.ShortID, + LocalFlags: w.LocalFlags, } if ok { - if cf.IsEquivalent(f, w.IgnorePerms, true) { + if cf.IsEquivalentOptional(f, w.IgnorePerms, true, w.LocalFlags) { return nil } if cf.ShouldConflict() { diff --git a/lib/scanner/walk_test.go b/lib/scanner/walk_test.go index c4d70e79f..447d73655 100644 --- a/lib/scanner/walk_test.go +++ b/lib/scanner/walk_test.go @@ -221,8 +221,8 @@ func TestNormalization(t *testing.T) { // make sure it all gets done. In production, things will be correct // eventually... - walkDir(testFs, "normalization", nil, nil) - tmp := walkDir(testFs, "normalization", nil, nil) + walkDir(testFs, "normalization", nil, nil, 0) + tmp := walkDir(testFs, "normalization", nil, nil, 0) files := fileList(tmp).testfiles() @@ -267,7 +267,7 @@ func TestWalkSymlinkUnix(t *testing.T) { fs := fs.NewFilesystem(fs.FilesystemTypeBasic, "_symlinks") for _, path := range []string{".", "link"} { // Scan it - files := walkDir(fs, path, nil, nil) + files := walkDir(fs, path, nil, nil, 0) // Verify that we got one symlink and with the correct attributes if len(files) != 1 { @@ -300,7 +300,7 @@ func TestWalkSymlinkWindows(t *testing.T) { for _, path := range []string{".", "link"} { // Scan it - files := walkDir(fs, path, nil, nil) + files := walkDir(fs, path, nil, nil, 0) // Verify that we got zero symlinks if len(files) != 0 { @@ -329,7 +329,7 @@ func TestWalkRootSymlink(t *testing.T) { } // Scan it - files := walkDir(fs.NewFilesystem(fs.FilesystemTypeBasic, link), ".", nil, nil) + files := walkDir(fs.NewFilesystem(fs.FilesystemTypeBasic, link), ".", nil, nil, 0) // Verify that we got two files if len(files) != 2 { @@ -353,7 +353,7 @@ func TestBlocksizeHysteresis(t *testing.T) { current := make(fakeCurrentFiler) runTest := func(expectedBlockSize int) { - files := walkDir(sf, ".", current, nil) + files := walkDir(sf, ".", current, nil, 0) if len(files) != 1 { t.Fatalf("expected one file, not %d", len(files)) } @@ -407,7 +407,57 @@ func TestBlocksizeHysteresis(t *testing.T) { runTest(512 << 10) } -func walkDir(fs fs.Filesystem, dir string, cfiler CurrentFiler, matcher *ignore.Matcher) []protocol.FileInfo { +func TestWalkReceiveOnly(t *testing.T) { + sf := fs.NewWalkFilesystem(&singleFileFS{ + name: "testfile.dat", + filesize: 1024, + }) + + current := make(fakeCurrentFiler) + + // Initial scan, no files in the CurrentFiler. Should pick up the file and + // set the ReceiveOnly flag on it, because that's the flag we give the + // walker to set. + + files := walkDir(sf, ".", current, nil, protocol.FlagLocalReceiveOnly) + if len(files) != 1 { + t.Fatal("Should have scanned one file") + } + + if files[0].LocalFlags != protocol.FlagLocalReceiveOnly { + t.Fatal("Should have set the ReceiveOnly flag") + } + + // Update the CurrentFiler and scan again. It should not return + // anything, because the file has not changed. This verifies that the + // ReceiveOnly flag is properly ignored and doesn't trigger a rescan + // every time. + + cur := files[0] + current[cur.Name] = cur + + files = walkDir(sf, ".", current, nil, protocol.FlagLocalReceiveOnly) + if len(files) != 0 { + t.Fatal("Should not have scanned anything") + } + + // Now pretend the file was previously ignored instead. We should pick up + // the difference in flags and set just the LocalReceive flags. + + cur.LocalFlags = protocol.FlagLocalIgnored + current[cur.Name] = cur + + files = walkDir(sf, ".", current, nil, protocol.FlagLocalReceiveOnly) + if len(files) != 1 { + t.Fatal("Should have scanned one file") + } + + if files[0].LocalFlags != protocol.FlagLocalReceiveOnly { + t.Fatal("Should have set the ReceiveOnly flag") + } +} + +func walkDir(fs fs.Filesystem, dir string, cfiler CurrentFiler, matcher *ignore.Matcher, localFlags uint32) []protocol.FileInfo { fchan := Walk(context.TODO(), Config{ Filesystem: fs, Subs: []string{dir}, @@ -416,6 +466,7 @@ func walkDir(fs fs.Filesystem, dir string, cfiler CurrentFiler, matcher *ignore. UseLargeBlocks: true, CurrentFiler: cfiler, Matcher: matcher, + LocalFlags: localFlags, }) var tmp []protocol.FileInfo @@ -579,7 +630,7 @@ func TestIssue4799(t *testing.T) { } fd.Close() - files := walkDir(fs, "/foo", nil, nil) + files := walkDir(fs, "/foo", nil, nil, 0) if len(files) != 1 || files[0].Name != "foo" { t.Error(`Received unexpected file infos when walking "/foo"`, files) } @@ -597,7 +648,7 @@ func TestRecurseInclude(t *testing.T) { t.Fatal(err) } - files := walkDir(testFs, ".", nil, ignores) + files := walkDir(testFs, ".", nil, ignores, 0) expected := []string{ filepath.Join("dir1"),