From 445a82f1205890dceb366a4d0e21e9f920c4d5f5 Mon Sep 17 00:00:00 2001 From: Simon Frei Date: Sun, 27 Jun 2021 08:48:54 +0200 Subject: [PATCH] lib/model: Compare all items with global on scan (fixes #7740) (#7791) --- lib/model/folder.go | 168 ++++++++++++++++++------------------ lib/model/folder_recvenc.go | 4 +- 2 files changed, 88 insertions(+), 84 deletions(-) diff --git a/lib/model/folder.go b/lib/model/folder.go index 159f92f67..5a88f9bf1 100644 --- a/lib/model/folder.go +++ b/lib/model/folder.go @@ -473,16 +473,7 @@ func (f *folder) scanSubdirs(subDirs []string) error { f.setState(FolderScanning) f.clearScanErrors(subDirs) - batch := db.NewFileInfoBatch(func(fs []protocol.FileInfo) error { - if err := f.getHealthErrorWithoutIgnores(); err != nil { - l.Debugf("Stopping scan of folder %s due to: %s", f.Description(), err) - return err - } - f.updateLocalsFromScanning(fs) - return nil - }) - - batchAppend := f.scanSubdirsBatchAppendFunc(batch) + batch := f.newScanBatch() // Schedule a pull after scanning, but only if we actually detected any // changes. @@ -494,7 +485,7 @@ func (f *folder) scanSubdirs(subDirs []string) error { } }() - changesHere, err := f.scanSubdirsChangedAndNew(subDirs, batch, batchAppend) + changesHere, err := f.scanSubdirsChangedAndNew(subDirs, batch) changes += changesHere if err != nil { return err @@ -513,7 +504,7 @@ func (f *folder) scanSubdirs(subDirs []string) error { // Do a scan of the database for each prefix, to check for deleted and // ignored files. - changesHere, err = f.scanSubdirsDeletedAndIgnored(subDirs, batch, batchAppend) + changesHere, err = f.scanSubdirsDeletedAndIgnored(subDirs, batch) changes += changesHere if err != nil { return err @@ -527,58 +518,57 @@ func (f *folder) scanSubdirs(subDirs []string) error { return nil } -type batchAppendFunc func(protocol.FileInfo, *db.Snapshot) bool - -func (f *folder) scanSubdirsBatchAppendFunc(batch *db.FileInfoBatch) batchAppendFunc { - // Resolve items which are identical with the global state. - switch f.Type { - case config.FolderTypeReceiveOnly: - return func(fi protocol.FileInfo, snap *db.Snapshot) bool { - switch gf, ok := snap.GetGlobal(fi.Name); { - case !ok: - case gf.IsEquivalentOptional(fi, f.modTimeWindow, false, false, protocol.FlagLocalReceiveOnly): - // What we have locally is equivalent to the global file. - fi.Version = gf.Version - l.Debugf("%v scanning: Merging identical locally changed item with global", f, fi) - fallthrough - case fi.IsDeleted() && (gf.IsReceiveOnlyChanged() || gf.IsDeleted()): - // Our item is deleted and the global item is our own - // receive only file or deleted too. In the former - // case we can't delete file infos, so we just - // pretend it is a normal deleted file (nobody - // cares about that). - l.Debugf("%v scanning: Marking item as not locally changed", f, fi) - fi.LocalFlags &^= protocol.FlagLocalReceiveOnly - } - batch.Append(fi) - return true - } - case config.FolderTypeReceiveEncrypted: - return func(fi protocol.FileInfo, _ *db.Snapshot) bool { - // This is a "virtual" parent directory of encrypted files. - // We don't track it, but check if anything still exists - // within and delete it otherwise. - if fi.IsDirectory() && protocol.IsEncryptedParent(fs.PathComponents(fi.Name)) { - if names, err := f.mtimefs.DirNames(fi.Name); err == nil && len(names) == 0 { - f.mtimefs.Remove(fi.Name) - } - return false - } - // Any local change must not be sent as index entry to - // remotes and show up as an error in the UI. - fi.LocalFlags = protocol.FlagLocalReceiveOnly - batch.Append(fi) - return true - } - default: - return func(fi protocol.FileInfo, _ *db.Snapshot) bool { - batch.Append(fi) - return true - } - } +type scanBatch struct { + *db.FileInfoBatch + f *folder } -func (f *folder) scanSubdirsChangedAndNew(subDirs []string, batch *db.FileInfoBatch, batchAppend batchAppendFunc) (int, error) { +func (f *folder) newScanBatch() *scanBatch { + b := &scanBatch{ + f: f, + } + b.FileInfoBatch = db.NewFileInfoBatch(func(fs []protocol.FileInfo) error { + if err := b.f.getHealthErrorWithoutIgnores(); err != nil { + l.Debugf("Stopping scan of folder %s due to: %s", b.f.Description(), err) + return err + } + b.f.updateLocalsFromScanning(fs) + return nil + }) + return b +} + +// Append adds the fileinfo to the batch for updating, and does a few checks. +// It returns false if the checks result in the file not going to be updated or removed. +func (b *scanBatch) Append(fi protocol.FileInfo, snap *db.Snapshot) bool { + // Check for a "virtual" parent directory of encrypted files. We don't track + // it, but check if anything still exists within and delete it otherwise. + if b.f.Type == config.FolderTypeReceiveEncrypted && fi.IsDirectory() && protocol.IsEncryptedParent(fs.PathComponents(fi.Name)) { + if names, err := b.f.mtimefs.DirNames(fi.Name); err == nil && len(names) == 0 { + b.f.mtimefs.Remove(fi.Name) + } + return false + } + // Resolve receive-only items which are identical with the global state or + // the global item is our own receive-only item. + // Except if they are in a receive-encrypted folder and are locally added. + // Those must never be sent in index updates and thus must retain the flag. + switch gf, ok := snap.GetGlobal(fi.Name); { + case !ok: + case gf.IsReceiveOnlyChanged(): + if b.f.Type == config.FolderTypeReceiveOnly && fi.Deleted { + l.Debugf("%v scanning: Marking deleted item as not locally changed", b.f, fi) + fi.LocalFlags &^= protocol.FlagLocalReceiveOnly + } + case gf.IsEquivalentOptional(fi, b.f.modTimeWindow, false, false, protocol.FlagLocalReceiveOnly): + l.Debugf("%v scanning: Replacing scanned file info with global as it's equivalent", b.f, fi) + fi = gf + } + b.FileInfoBatch.Append(fi) + return true +} + +func (f *folder) scanSubdirsChangedAndNew(subDirs []string, batch *scanBatch) (int, error) { changes := 0 snap, err := f.dbSnapshot() if err != nil { @@ -630,7 +620,7 @@ func (f *folder) scanSubdirsChangedAndNew(subDirs []string, batch *db.FileInfoBa return changes, err } - if batchAppend(res.File, snap) { + if batch.Append(res.File, snap) { changes++ } @@ -638,7 +628,7 @@ func (f *folder) scanSubdirsChangedAndNew(subDirs []string, batch *db.FileInfoBa case config.FolderTypeReceiveOnly, config.FolderTypeReceiveEncrypted: default: if nf, ok := f.findRename(snap, res.File, alreadyUsedOrExisting); ok { - if batchAppend(nf, snap) { + if batch.Append(nf, snap) { changes++ } } @@ -648,7 +638,7 @@ func (f *folder) scanSubdirsChangedAndNew(subDirs []string, batch *db.FileInfoBa return changes, nil } -func (f *folder) scanSubdirsDeletedAndIgnored(subDirs []string, batch *db.FileInfoBatch, batchAppend batchAppendFunc) (int, error) { +func (f *folder) scanSubdirsDeletedAndIgnored(subDirs []string, batch *scanBatch) (int, error) { var toIgnore []db.FileInfoTruncated ignoredParent := "" changes := 0 @@ -679,7 +669,7 @@ func (f *folder) scanSubdirsDeletedAndIgnored(subDirs []string, batch *db.FileIn for _, file := range toIgnore { l.Debugln("marking file as ignored", file) nf := file.ConvertToIgnoredFileInfo() - if batchAppend(nf, snap) { + if batch.Append(nf, snap) { changes++ } if err := batch.FlushIfFull(); err != nil { @@ -709,7 +699,7 @@ func (f *folder) scanSubdirsDeletedAndIgnored(subDirs []string, batch *db.FileIn l.Debugln("marking file as ignored", file) nf := file.ConvertToIgnoredFileInfo() - if batchAppend(nf, snap) { + if batch.Append(nf, snap) { changes++ } @@ -739,23 +729,35 @@ func (f *folder) scanSubdirsDeletedAndIgnored(subDirs []string, batch *db.FileIn nf.Version = protocol.Vector{} } l.Debugln("marking file as deleted", nf) - if batchAppend(nf, snap) { + if batch.Append(nf, snap) { changes++ } - case file.IsDeleted() && file.IsReceiveOnlyChanged() && f.Type == config.FolderTypeReceiveOnly && len(snap.Availability(file.Name)) == 0: - file.Version = protocol.Vector{} - file.LocalFlags &^= protocol.FlagLocalReceiveOnly - l.Debugln("marking deleted item that doesn't exist anywhere as not receive-only", file) - if batchAppend(file.ConvertDeletedToFileInfo(), snap) { - changes++ - } - case file.IsDeleted() && file.IsReceiveOnlyChanged() && f.Type != config.FolderTypeReceiveOnly: - // No need to bump the version for a file that was and is - // deleted and just the folder type/local flags changed. - file.LocalFlags &^= protocol.FlagLocalReceiveOnly - l.Debugln("removing receive-only flag on deleted item", file) - if batchAppend(file.ConvertDeletedToFileInfo(), snap) { - changes++ + case file.IsDeleted() && file.IsReceiveOnlyChanged(): + switch f.Type { + case config.FolderTypeReceiveOnly, config.FolderTypeReceiveEncrypted: + if gf, _ := snap.GetGlobal(file.Name); gf.IsDeleted() { + // Our item is deleted and the global item is deleted too. We just + // pretend it is a normal deleted file (nobody cares about that). + // Except if this is a receive-encrypted folder and it + // is a locally added file. Those must never be sent + // in index updates and thus must retain the flag. + if f.Type == config.FolderTypeReceiveEncrypted && gf.IsReceiveOnlyChanged() { + return true + } + l.Debugf("%v scanning: Marking globally deleted item as not locally changed: %v", f, file.Name) + file.LocalFlags &^= protocol.FlagLocalReceiveOnly + if batch.Append(file.ConvertDeletedToFileInfo(), snap) { + changes++ + } + } + default: + // No need to bump the version for a file that was and is + // deleted and just the folder type/local flags changed. + file.LocalFlags &^= protocol.FlagLocalReceiveOnly + l.Debugln("removing receive-only flag on deleted item", file) + if batch.Append(file.ConvertDeletedToFileInfo(), snap) { + changes++ + } } } @@ -772,7 +774,7 @@ func (f *folder) scanSubdirsDeletedAndIgnored(subDirs []string, batch *db.FileIn for _, file := range toIgnore { l.Debugln("marking file as ignored", f) nf := file.ConvertToIgnoredFileInfo() - if batchAppend(nf, snap) { + if batch.Append(nf, snap) { changes++ } if iterError = batch.FlushIfFull(); iterError != nil { diff --git a/lib/model/folder_recvenc.go b/lib/model/folder_recvenc.go index 2ddb6dde7..2c95a54f2 100644 --- a/lib/model/folder_recvenc.go +++ b/lib/model/folder_recvenc.go @@ -29,7 +29,9 @@ type receiveEncryptedFolder struct { } func newReceiveEncryptedFolder(model *model, fset *db.FileSet, ignores *ignore.Matcher, cfg config.FolderConfiguration, ver versioner.Versioner, evLogger events.Logger, ioLimiter *util.Semaphore) service { - return &receiveEncryptedFolder{newSendReceiveFolder(model, fset, ignores, cfg, ver, evLogger, ioLimiter).(*sendReceiveFolder)} + f := &receiveEncryptedFolder{newSendReceiveFolder(model, fset, ignores, cfg, ver, evLogger, ioLimiter).(*sendReceiveFolder)} + f.localFlags = protocol.FlagLocalReceiveOnly // gets propagated to the scanner, and set on locally changed files + return f } func (f *receiveEncryptedFolder) Revert() {