lib/model: Compare all items with global on scan (fixes #7740) (#7791)

lib/model/folder.go

@@ -473,16 +473,7 @@ func (f *folder) scanSubdirs(subDirs []string) error {
 	f.setState(FolderScanning)
 	f.clearScanErrors(subDirs)
 
-	batch := db.NewFileInfoBatch(func(fs []protocol.FileInfo) error {
+	batch := f.newScanBatch()
-		if err := f.getHealthErrorWithoutIgnores(); err != nil {
-			l.Debugf("Stopping scan of folder %s due to: %s", f.Description(), err)
-			return err
-		}
-		f.updateLocalsFromScanning(fs)
-		return nil
-	})
-
-	batchAppend := f.scanSubdirsBatchAppendFunc(batch)
 
 	// Schedule a pull after scanning, but only if we actually detected any
 	// changes.
@@ -494,7 +485,7 @@ func (f *folder) scanSubdirs(subDirs []string) error {
 		}
 	}()
 
-	changesHere, err := f.scanSubdirsChangedAndNew(subDirs, batch, batchAppend)
+	changesHere, err := f.scanSubdirsChangedAndNew(subDirs, batch)
 	changes += changesHere
 	if err != nil {
 		return err
@@ -513,7 +504,7 @@ func (f *folder) scanSubdirs(subDirs []string) error {
 	// Do a scan of the database for each prefix, to check for deleted and
 	// ignored files.
 
-	changesHere, err = f.scanSubdirsDeletedAndIgnored(subDirs, batch, batchAppend)
+	changesHere, err = f.scanSubdirsDeletedAndIgnored(subDirs, batch)
 	changes += changesHere
 	if err != nil {
 		return err
@@ -527,58 +518,57 @@ func (f *folder) scanSubdirs(subDirs []string) error {
 	return nil
 }
 
-type batchAppendFunc func(protocol.FileInfo, *db.Snapshot) bool
+type scanBatch struct {
+	*db.FileInfoBatch
+	f *folder
+}
 
-func (f *folder) scanSubdirsBatchAppendFunc(batch *db.FileInfoBatch) batchAppendFunc {
+func (f *folder) newScanBatch() *scanBatch {
-	// Resolve items which are identical with the global state.
+	b := &scanBatch{
-	switch f.Type {
+		f: f,
-	case config.FolderTypeReceiveOnly:
-		return func(fi protocol.FileInfo, snap *db.Snapshot) bool {
-			switch gf, ok := snap.GetGlobal(fi.Name); {
-			case !ok:
-			case gf.IsEquivalentOptional(fi, f.modTimeWindow, false, false, protocol.FlagLocalReceiveOnly):
-				// What we have locally is equivalent to the global file.
-				fi.Version = gf.Version
-				l.Debugf("%v scanning: Merging identical locally changed item with global", f, fi)
-				fallthrough
-			case fi.IsDeleted() && (gf.IsReceiveOnlyChanged() || gf.IsDeleted()):
-				// Our item is deleted and the global item is our own
-				// receive only file or deleted too. In the former
-				// case we can't delete file infos, so we just
-				// pretend it is a normal deleted file (nobody
-				// cares about that).
-				l.Debugf("%v scanning: Marking item as not locally changed", f, fi)
-				fi.LocalFlags &^= protocol.FlagLocalReceiveOnly
 	}
-			batch.Append(fi)
+	b.FileInfoBatch = db.NewFileInfoBatch(func(fs []protocol.FileInfo) error {
-			return true
+		if err := b.f.getHealthErrorWithoutIgnores(); err != nil {
+			l.Debugf("Stopping scan of folder %s due to: %s", b.f.Description(), err)
+			return err
 		}
-	case config.FolderTypeReceiveEncrypted:
+		b.f.updateLocalsFromScanning(fs)
-		return func(fi protocol.FileInfo, _ *db.Snapshot) bool {
+		return nil
-			// This is a "virtual" parent directory of encrypted files.
+	})
-			// We don't track it, but check if anything still exists
+	return b
-			// within and delete it otherwise.
+}
-			if fi.IsDirectory() && protocol.IsEncryptedParent(fs.PathComponents(fi.Name)) {
+
-				if names, err := f.mtimefs.DirNames(fi.Name); err == nil && len(names) == 0 {
+// Append adds the fileinfo to the batch for updating, and does a few checks.
-					f.mtimefs.Remove(fi.Name)
+// It returns false if the checks result in the file not going to be updated or removed.
+func (b *scanBatch) Append(fi protocol.FileInfo, snap *db.Snapshot) bool {
+	// Check for a "virtual" parent directory of encrypted files. We don't track
+	// it, but check if anything still exists within and delete it otherwise.
+	if b.f.Type == config.FolderTypeReceiveEncrypted && fi.IsDirectory() && protocol.IsEncryptedParent(fs.PathComponents(fi.Name)) {
+		if names, err := b.f.mtimefs.DirNames(fi.Name); err == nil && len(names) == 0 {
+			b.f.mtimefs.Remove(fi.Name)
 		}
 		return false
 	}
-			// Any local change must not be sent as index entry to
+	// Resolve receive-only items which are identical with the global state or
-			// remotes and show up as an error in the UI.
+	// the global item is our own receive-only item.
-			fi.LocalFlags = protocol.FlagLocalReceiveOnly
+	// Except if they are in a receive-encrypted folder and are locally added.
-			batch.Append(fi)
+	// Those must never be sent in index updates and thus must retain the flag.
+	switch gf, ok := snap.GetGlobal(fi.Name); {
+	case !ok:
+	case gf.IsReceiveOnlyChanged():
+		if b.f.Type == config.FolderTypeReceiveOnly && fi.Deleted {
+			l.Debugf("%v scanning: Marking deleted item as not locally changed", b.f, fi)
+			fi.LocalFlags &^= protocol.FlagLocalReceiveOnly
+		}
+	case gf.IsEquivalentOptional(fi, b.f.modTimeWindow, false, false, protocol.FlagLocalReceiveOnly):
+		l.Debugf("%v scanning: Replacing scanned file info with global as it's equivalent", b.f, fi)
+		fi = gf
+	}
+	b.FileInfoBatch.Append(fi)
 	return true
 }
-	default:
-		return func(fi protocol.FileInfo, _ *db.Snapshot) bool {
-			batch.Append(fi)
-			return true
-		}
-	}
-}
 
-func (f *folder) scanSubdirsChangedAndNew(subDirs []string, batch *db.FileInfoBatch, batchAppend batchAppendFunc) (int, error) {
+func (f *folder) scanSubdirsChangedAndNew(subDirs []string, batch *scanBatch) (int, error) {
 	changes := 0
 	snap, err := f.dbSnapshot()
 	if err != nil {
@@ -630,7 +620,7 @@ func (f *folder) scanSubdirsChangedAndNew(subDirs []string, batch *db.FileInfoBa
 			return changes, err
 		}
 
-		if batchAppend(res.File, snap) {
+		if batch.Append(res.File, snap) {
 			changes++
 		}
 
@@ -638,7 +628,7 @@ func (f *folder) scanSubdirsChangedAndNew(subDirs []string, batch *db.FileInfoBa
 		case config.FolderTypeReceiveOnly, config.FolderTypeReceiveEncrypted:
 		default:
 			if nf, ok := f.findRename(snap, res.File, alreadyUsedOrExisting); ok {
-				if batchAppend(nf, snap) {
+				if batch.Append(nf, snap) {
 					changes++
 				}
 			}
@@ -648,7 +638,7 @@ func (f *folder) scanSubdirsChangedAndNew(subDirs []string, batch *db.FileInfoBa
 	return changes, nil
 }
 
-func (f *folder) scanSubdirsDeletedAndIgnored(subDirs []string, batch *db.FileInfoBatch, batchAppend batchAppendFunc) (int, error) {
+func (f *folder) scanSubdirsDeletedAndIgnored(subDirs []string, batch *scanBatch) (int, error) {
 	var toIgnore []db.FileInfoTruncated
 	ignoredParent := ""
 	changes := 0
@@ -679,7 +669,7 @@ func (f *folder) scanSubdirsDeletedAndIgnored(subDirs []string, batch *db.FileIn
 				for _, file := range toIgnore {
 					l.Debugln("marking file as ignored", file)
 					nf := file.ConvertToIgnoredFileInfo()
-					if batchAppend(nf, snap) {
+					if batch.Append(nf, snap) {
 						changes++
 					}
 					if err := batch.FlushIfFull(); err != nil {
@@ -709,7 +699,7 @@ func (f *folder) scanSubdirsDeletedAndIgnored(subDirs []string, batch *db.FileIn
 
 				l.Debugln("marking file as ignored", file)
 				nf := file.ConvertToIgnoredFileInfo()
-				if batchAppend(nf, snap) {
+				if batch.Append(nf, snap) {
 					changes++
 				}
 
@@ -739,25 +729,37 @@ func (f *folder) scanSubdirsDeletedAndIgnored(subDirs []string, batch *db.FileIn
 					nf.Version = protocol.Vector{}
 				}
 				l.Debugln("marking file as deleted", nf)
-				if batchAppend(nf, snap) {
+				if batch.Append(nf, snap) {
 					changes++
 				}
-			case file.IsDeleted() && file.IsReceiveOnlyChanged() && f.Type == config.FolderTypeReceiveOnly && len(snap.Availability(file.Name)) == 0:
+			case file.IsDeleted() && file.IsReceiveOnlyChanged():
-				file.Version = protocol.Vector{}
+				switch f.Type {
+				case config.FolderTypeReceiveOnly, config.FolderTypeReceiveEncrypted:
+					if gf, _ := snap.GetGlobal(file.Name); gf.IsDeleted() {
+						// Our item is deleted and the global item is deleted too. We just
+						// pretend it is a normal deleted file (nobody cares about that).
+						// Except if this is a receive-encrypted folder and it
+						// is a locally added file. Those must never be sent
+						// in index updates and thus must retain the flag.
+						if f.Type == config.FolderTypeReceiveEncrypted && gf.IsReceiveOnlyChanged() {
+							return true
+						}
+						l.Debugf("%v scanning: Marking globally deleted item as not locally changed: %v", f, file.Name)
 						file.LocalFlags &^= protocol.FlagLocalReceiveOnly
-				l.Debugln("marking deleted item that doesn't exist anywhere as not receive-only", file)
+						if batch.Append(file.ConvertDeletedToFileInfo(), snap) {
-				if batchAppend(file.ConvertDeletedToFileInfo(), snap) {
 							changes++
 						}
-			case file.IsDeleted() && file.IsReceiveOnlyChanged() && f.Type != config.FolderTypeReceiveOnly:
+					}
+				default:
 					// No need to bump the version for a file that was and is
 					// deleted and just the folder type/local flags changed.
 					file.LocalFlags &^= protocol.FlagLocalReceiveOnly
 					l.Debugln("removing receive-only flag on deleted item", file)
-				if batchAppend(file.ConvertDeletedToFileInfo(), snap) {
+					if batch.Append(file.ConvertDeletedToFileInfo(), snap) {
 						changes++
 					}
 				}
+			}
 
 			return true
 		})
@@ -772,7 +774,7 @@ func (f *folder) scanSubdirsDeletedAndIgnored(subDirs []string, batch *db.FileIn
 			for _, file := range toIgnore {
 				l.Debugln("marking file as ignored", f)
 				nf := file.ConvertToIgnoredFileInfo()
-				if batchAppend(nf, snap) {
+				if batch.Append(nf, snap) {
 					changes++
 				}
 				if iterError = batch.FlushIfFull(); iterError != nil {

lib/model/folder_recvenc.go

@@ -29,7 +29,9 @@ type receiveEncryptedFolder struct {
 }
 
 func newReceiveEncryptedFolder(model *model, fset *db.FileSet, ignores *ignore.Matcher, cfg config.FolderConfiguration, ver versioner.Versioner, evLogger events.Logger, ioLimiter *util.Semaphore) service {
-	return &receiveEncryptedFolder{newSendReceiveFolder(model, fset, ignores, cfg, ver, evLogger, ioLimiter).(*sendReceiveFolder)}
+	f := &receiveEncryptedFolder{newSendReceiveFolder(model, fset, ignores, cfg, ver, evLogger, ioLimiter).(*sendReceiveFolder)}
+	f.localFlags = protocol.FlagLocalReceiveOnly // gets propagated to the scanner, and set on locally changed files
+	return f
 }
 
 func (f *receiveEncryptedFolder) Revert() {