lib/fs, lib/model: Rewrite RecvOnly tests (#6318)

During some other work I discovered these tests weren't great, so I've
rewritten them to be a little better. The real changes here are:

- Don't play games with not starting the folder and such, and don't
  construct a fake folder instance -- just use the one the model has. The
  folder starts and scans but the folder contents are empty at this point
  so that's fine.

- Use a fakefs instead of a temp dir.

- To support the above, implement a fakefs option `?content=true` to
  make the fakefs actually retain written content. Use sparingly,
  obviously, but it means the fakefs can usually be used instead of an
  on disk real directory.

lib/fs/fakefs.go

@@ -52,9 +52,10 @@ const randomBlockShift = 14 // 128k
 // - Two fakefs:s pointing at the same root path see the same files.
 //
 type fakefs struct {
-	mut    sync.Mutex
-	root   *fakeEntry
-	insens bool
+	mut         sync.Mutex
+	root        *fakeEntry
+	insens      bool
+	withContent bool
 }
 
 var (
@@ -93,9 +94,9 @@ func newFakeFilesystem(root string) *fakefs {
 	sizeavg, _ := strconv.Atoi(params.Get("sizeavg"))
 	seed, _ := strconv.Atoi(params.Get("seed"))
 
-	if params.Get("insens") == "true" {
-		fs.insens = true
-	}
+	fs.insens = params.Get("insens") == "true"
+	fs.withContent = params.Get("content") == "true"
+
 	if sizeavg == 0 {
 		sizeavg = 1 << 20
 	}
@@ -151,6 +152,7 @@ type fakeEntry struct {
 	gid       int
 	mtime     time.Time
 	children  map[string]*fakeEntry
+	content   []byte
 }
 
 func (fs *fakefs) entryForName(name string) *fakeEntry {
@@ -227,6 +229,10 @@ func (fs *fakefs) create(name string) (*fakeEntry, error) {
 		entry.size = 0
 		entry.mtime = time.Now()
 		entry.mode = 0666
+		entry.content = nil
+		if fs.withContent {
+			entry.content = make([]byte, 0)
+		}
 		return entry, nil
 	}
 
@@ -246,6 +252,10 @@ func (fs *fakefs) create(name string) (*fakeEntry, error) {
 		base = UnicodeLowercase(base)
 	}
 
+	if fs.withContent {
+		new.content = make([]byte, 0)
+	}
+
 	entry.children[base] = new
 	return new, nil
 }
@@ -417,6 +427,9 @@ func (fs *fakefs) OpenFile(name string, flags int, mode FileMode) (File, error)
 		mode:  mode,
 		mtime: time.Now(),
 	}
+	if fs.withContent {
+		newEntry.content = make([]byte, 0)
+	}
 
 	entry.children[key] = newEntry
 	return &fakeFile{fakeEntry: newEntry}, nil
@@ -660,6 +673,12 @@ func (f *fakeFile) readShortAt(p []byte, offs int64) (int, error) {
 		return 0, io.EOF
 	}
 
+	if f.content != nil {
+		n := copy(p, f.content[int(offs):])
+		f.offset = offs + int64(n)
+		return n, nil
+	}
+
 	// Lazily calculate our main seed, a simple 64 bit FNV hash our file
 	// name.
 	if f.seed == 0 {
@@ -746,6 +765,15 @@ func (f *fakeFile) WriteAt(p []byte, off int64) (int, error) {
 		return 0, errors.New("is a directory")
 	}
 
+	if f.content != nil {
+		if len(f.content) < int(off)+len(p) {
+			newc := make([]byte, int(off)+len(p))
+			copy(newc, f.content)
+			f.content = newc
+		}
+		copy(f.content[int(off):], p)
+	}
+
 	f.rng = nil
 	f.offset = off + int64(len(p))
 	if f.offset > f.size {
@@ -765,6 +793,9 @@ func (f *fakeFile) Truncate(size int64) error {
 	f.mut.Lock()
 	defer f.mut.Unlock()
 
+	if f.content != nil {
+		f.content = f.content[:int(size)]
+	}
 	f.rng = nil
 	f.size = size
 	if f.offset > size {

lib/fs/fakefs_test.go

@@ -896,6 +896,35 @@ func testFakeFSCreateInsens(t *testing.T, fs Filesystem) {
 	assertDir(t, fs, "/", []string{"FOO"})
 }
 
+func TestReadWriteContent(t *testing.T) {
+	fs := newFakeFilesystem("foo?content=true")
+	fd, err := fs.Create("file")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if _, err := fd.Write([]byte("foo")); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := fd.WriteAt([]byte("bar"), 5); err != nil {
+		t.Fatal(err)
+	}
+	expected := []byte("foo\x00\x00bar")
+
+	buf := make([]byte, len(expected)-1)
+	n, err := fd.ReadAt(buf, 1) // note offset one byte
+	if err != nil {
+		t.Fatal(err)
+	}
+	if n != len(expected)-1 {
+		t.Fatal("wrong number of bytes read")
+	}
+	if !bytes.Equal(buf[:n], expected[1:]) {
+		fmt.Printf("%d %q\n", n, buf[:n])
+		t.Error("wrong data in file")
+	}
+}
+
 func cleanup(fs Filesystem) error {
 	filenames, _ := fs.DirNames("/")
 	for _, filename := range filenames {

lib/model/folder_recvonly_test.go

@@ -9,8 +9,6 @@ package model
 import (
 	"bytes"
 	"context"
-	"io/ioutil"
-	"path/filepath"
 	"testing"
 	"time"
 
@@ -28,18 +26,18 @@ func TestRecvOnlyRevertDeletes(t *testing.T) {
 
 	// Get us a model up and running
 
-	m, f := setupROFolder()
+	m, f := setupROFolder(t)
 	ffs := f.Filesystem()
-	defer cleanupModelAndRemoveDir(m, ffs.URI())
+	defer cleanupModel(m)
 
 	// Create some test data
 
 	for _, dir := range []string{".stfolder", "ignDir", "unknownDir"} {
 		must(t, ffs.MkdirAll(dir, 0755))
 	}
-	must(t, ioutil.WriteFile(filepath.Join(ffs.URI(), "ignDir/ignFile"), []byte("hello\n"), 0644))
-	must(t, ioutil.WriteFile(filepath.Join(ffs.URI(), "unknownDir/unknownFile"), []byte("hello\n"), 0644))
-	must(t, ioutil.WriteFile(filepath.Join(ffs.URI(), ".stignore"), []byte("ignDir\n"), 0644))
+	must(t, writeFile(ffs, "ignDir/ignFile", []byte("hello\n"), 0644))
+	must(t, writeFile(ffs, "unknownDir/unknownFile", []byte("hello\n"), 0644))
+	must(t, writeFile(ffs, ".stignore", []byte("ignDir\n"), 0644))
 
 	knownFiles := setupKnownFiles(t, ffs, []byte("hello\n"))
 
@@ -57,10 +55,9 @@ func TestRecvOnlyRevertDeletes(t *testing.T) {
 		t.Fatalf("Global: expected 1 file and 1 directory: %+v", size)
 	}
 
-	// Start the folder. This will cause a scan, should discover the other stuff in the folder
+	// Scan, should discover the other stuff in the folder
 
-	m.startFolder("ro")
-	m.ScanFolder("ro")
+	must(t, m.ScanFolder("ro"))
 
 	// We should now have two files and two directories.
 
@@ -113,9 +110,9 @@ func TestRecvOnlyRevertNeeds(t *testing.T) {
 
 	// Get us a model up and running
 
-	m, f := setupROFolder()
+	m, f := setupROFolder(t)
 	ffs := f.Filesystem()
-	defer cleanupModelAndRemoveDir(m, ffs.URI())
+	defer cleanupModel(m)
 
 	// Create some test data
 
@@ -128,10 +125,9 @@ func TestRecvOnlyRevertNeeds(t *testing.T) {
 	m.Index(device1, "ro", knownFiles)
 	f.updateLocalsFromScanning(knownFiles)
 
-	// Start the folder. This will cause a scan.
+	// Scan the folder.
 
-	m.startFolder("ro")
-	m.ScanFolder("ro")
+	must(t, m.ScanFolder("ro"))
 
 	// Everything should be in sync.
 
@@ -155,7 +151,7 @@ func TestRecvOnlyRevertNeeds(t *testing.T) {
 	// Update the file.
 
 	newData := []byte("totally different data\n")
-	must(t, ioutil.WriteFile(filepath.Join(ffs.URI(), "knownDir/knownFile"), newData, 0644))
+	must(t, writeFile(ffs, "knownDir/knownFile", newData, 0644))
 
 	// Rescan.
 
@@ -200,13 +196,11 @@ func TestRecvOnlyRevertNeeds(t *testing.T) {
 }
 
 func TestRecvOnlyUndoChanges(t *testing.T) {
-	testOs := &fatalOs{t}
-
 	// Get us a model up and running
 
-	m, f := setupROFolder()
+	m, f := setupROFolder(t)
 	ffs := f.Filesystem()
-	defer cleanupModelAndRemoveDir(m, ffs.URI())
+	defer cleanupModel(m)
 
 	// Create some test data
 
@@ -214,20 +208,14 @@ func TestRecvOnlyUndoChanges(t *testing.T) {
 	oldData := []byte("hello\n")
 	knownFiles := setupKnownFiles(t, ffs, oldData)
 
-	m.fmut.Lock()
-	fset := m.folderFiles["ro"]
-	m.fmut.Unlock()
-	folderFs := fset.MtimeFS()
-
-	// Send and index update for the known stuff
+	// Send an index update for the known stuff
 
 	m.Index(device1, "ro", knownFiles)
 	f.updateLocalsFromScanning(knownFiles)
 
-	// Start the folder. This will cause a scan.
+	// Scan the folder.
 
-	m.startFolder("ro")
-	m.ScanFolder("ro")
+	must(t, m.ScanFolder("ro"))
 
 	// Everything should be in sync.
 
@@ -250,12 +238,11 @@ func TestRecvOnlyUndoChanges(t *testing.T) {
 
 	// Create a file and modify another
 
-	file := filepath.Join(ffs.URI(), "foo")
-	must(t, ioutil.WriteFile(file, []byte("hello\n"), 0644))
+	const file = "foo"
+	must(t, writeFile(ffs, file, []byte("hello\n"), 0644))
+	must(t, writeFile(ffs, "knownDir/knownFile", []byte("bye\n"), 0644))
 
-	must(t, ioutil.WriteFile(filepath.Join(ffs.URI(), "knownDir/knownFile"), []byte("bye\n"), 0644))
-
-	m.ScanFolder("ro")
+	must(t, m.ScanFolder("ro"))
 
 	size = receiveOnlyChangedSize(t, m, "ro")
 	if size.Files != 2 {
@@ -264,11 +251,11 @@ func TestRecvOnlyUndoChanges(t *testing.T) {
 
 	// Remove the file again and undo the modification
 
-	testOs.Remove(file)
-	must(t, ioutil.WriteFile(filepath.Join(ffs.URI(), "knownDir/knownFile"), oldData, 0644))
-	folderFs.Chtimes("knownDir/knownFile", knownFiles[1].ModTime(), knownFiles[1].ModTime())
+	must(t, ffs.Remove(file))
+	must(t, writeFile(ffs, "knownDir/knownFile", oldData, 0644))
+	must(t, ffs.Chtimes("knownDir/knownFile", knownFiles[1].ModTime(), knownFiles[1].ModTime()))
 
-	m.ScanFolder("ro")
+	must(t, m.ScanFolder("ro"))
 
 	size = receiveOnlyChangedSize(t, m, "ro")
 	if size.Files+size.Directories+size.Deleted != 0 {
@@ -280,7 +267,7 @@ func setupKnownFiles(t *testing.T, ffs fs.Filesystem, data []byte) []protocol.Fi
 	t.Helper()
 
 	must(t, ffs.MkdirAll("knownDir", 0755))
-	must(t, ioutil.WriteFile(filepath.Join(ffs.URI(), "knownDir/knownFile"), data, 0644))
+	must(t, writeFile(ffs, "knownDir/knownFile", data, 0644))
 
 	t0 := time.Now().Add(-1 * time.Minute)
 	must(t, ffs.Chtimes("knownDir/knownFile", t0, t0))
@@ -314,30 +301,38 @@ func setupKnownFiles(t *testing.T, ffs fs.Filesystem, data []byte) []protocol.Fi
 	return knownFiles
 }
 
-func setupROFolder() (*model, *sendOnlyFolder) {
+func setupROFolder(t *testing.T) (*model, *receiveOnlyFolder) {
+	t.Helper()
+
 	w := createTmpWrapper(defaultCfg)
-	fcfg := testFolderConfigTmp()
+	fcfg := testFolderConfigFake()
 	fcfg.ID = "ro"
+	fcfg.Label = "ro"
 	fcfg.Type = config.FolderTypeReceiveOnly
 	w.SetFolder(fcfg)
 
 	m := newModel(w, myID, "syncthing", "dev", db.NewLowlevel(backend.OpenMemory()), nil)
-
 	m.ServeBackground()
-
-	// Folder should only be added, not started.
-	m.removeFolder(fcfg)
-	m.addFolder(fcfg)
+	must(t, m.ScanFolder("ro"))
 
 	m.fmut.RLock()
-	f := &sendOnlyFolder{
-		folder: folder{
-			stateTracker:        newStateTracker(fcfg.ID, m.evLogger),
-			fset:                m.folderFiles[fcfg.ID],
-			FolderConfiguration: fcfg,
-		},
-	}
-	m.fmut.RUnlock()
+	defer m.fmut.RUnlock()
+	f := m.folderRunners["ro"].(*receiveOnlyFolder)
 
 	return m, f
 }
+
+func writeFile(fs fs.Filesystem, filename string, data []byte, perm fs.FileMode) error {
+	fd, err := fs.Create(filename)
+	if err != nil {
+		return err
+	}
+	_, err = fd.Write(data)
+	if err != nil {
+		return err
+	}
+	if err := fd.Close(); err != nil {
+		return err
+	}
+	return fs.Chmod(filename, perm)
+}

lib/model/testutils_test.go

@@ -18,6 +18,7 @@ import (
 	"github.com/syncthing/syncthing/lib/events"
 	"github.com/syncthing/syncthing/lib/fs"
 	"github.com/syncthing/syncthing/lib/protocol"
+	"github.com/syncthing/syncthing/lib/rand"
 )
 
 var (
@@ -86,6 +87,13 @@ func testFolderConfig(path string) config.FolderConfiguration {
 	return cfg
 }
 
+func testFolderConfigFake() config.FolderConfiguration {
+	cfg := config.NewFolderConfiguration(myID, "default", "default", fs.FilesystemTypeFake, rand.String(32)+"?content=true")
+	cfg.FSWatcherEnabled = false
+	cfg.Devices = append(cfg.Devices, config.FolderDeviceConfiguration{DeviceID: device1})
+	return cfg
+}
+
 func setupModelWithConnection() (*model, *fakeConnection, config.FolderConfiguration) {
 	w, fcfg := tmpDefaultWrapper()
 	m, fc := setupModelWithConnectionFromWrapper(w)