Merge pull request #2106 from Zillode/scan-deletes

Reduce scanning effort

internal/model/model.go

@@ -1227,13 +1227,14 @@ func (m *Model) internalScanFolderSubs(folder string, subs []string) error {
 nextSub:
 	for _, sub := range subs {
 		for sub != "" {
-			if _, ok = fs.Get(protocol.LocalDeviceID, sub); ok {
+			parent := filepath.Dir(sub)
+			if parent == "." || parent == string(filepath.Separator) {
+				parent = ""
+			}
+			if _, ok = fs.Get(protocol.LocalDeviceID, parent); ok {
 				break
 			}
-			sub = filepath.Dir(sub)
-			if sub == "." || sub == string(filepath.Separator) {
-				sub = ""
-			}
+			sub = parent
 		}
 		for _, us := range unifySubs {
 			if strings.HasPrefix(sub, us) {

internal/rc/rc.go

@@ -17,9 +17,11 @@ import (
 	"io/ioutil"
 	"log"
 	"net/http"
+	"net/url"
 	"os"
 	"os/exec"
 	"path/filepath"
+	"strconv"
 	stdsync "sync"
 	"time"
 
@@ -233,6 +235,21 @@ func (p *Process) RescanDelay(folder string, delaySeconds int) error {
 	return err
 }
 
+func (p *Process) RescanSub(folder string, sub string, delaySeconds int) error {
+	return p.RescanSubs(folder, []string{sub}, delaySeconds)
+}
+
+func (p *Process) RescanSubs(folder string, subs []string, delaySeconds int) error {
+	data := url.Values{}
+	data.Set("folder", folder)
+	for _, sub := range subs {
+		data.Add("sub", sub)
+	}
+	data.Set("next", strconv.Itoa(delaySeconds))
+	_, err := p.Post("/rest/db/scan?"+data.Encode(), nil)
+	return err
+}
+
 func (p *Process) ConfigInSync() (bool, error) {
 	bs, err := p.Get("/rest/system/config/insync")
 	if err != nil {

test/scan_test.go

@@ -0,0 +1,188 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+// +build integration
+
+package integration
+
+import (
+	"log"
+	"os"
+	"testing"
+
+	"github.com/syncthing/syncthing/internal/rc"
+)
+
+func TestSubScan(t *testing.T) {
+	log.Println("Cleaning...")
+	err := removeAll("s1", "s2", "h1/index*", "h2/index*")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	log.Println("Generating files...")
+	err = generateFiles("s1", 10, 10, "../LICENSE")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// 1. Scan a single file in a known directory "file1.txt"
+	// 2. Scan a single file in an unknown directory "filetest/file1.txt"
+	// 3. Scan a single file in a deep unknown directory "filetest/1/2/3/4/5/6/7/file1.txt"
+	// 4. Scan a directory in a deep unknown directory "dirtest/1/2/3/4/5/6/7"
+	// 5. Scan a deleted file in a known directory "filetest/file1.txt"
+	// 6. Scan a deleted file in a deep unknown directory "rmdirtest/1/2/3/4/5/6/7"
+	// 7. 'Accidentally' forget to scan 1 of the 2 files in a known directory
+
+	// Verify that the files and directories sync to the other side
+	sender := startInstance(t, 1)
+	defer checkedStop(t, sender)
+	receiver := startInstance(t, 2)
+	defer checkedStop(t, receiver)
+	log.Println("Syncing...")
+	rc.AwaitSync("default", sender, receiver)
+
+	// Delay scans for the moment
+	if err := sender.RescanDelay("default", 86400); err != nil {
+		t.Fatal(err)
+	}
+
+	log.Println("Comparing directories...")
+	err = compareDirectories("s1", "s2")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// 1
+	log.Println("Creating new file...")
+	if fd, err := os.Create("s1/file1.txt"); err != nil {
+		t.Fatal(err)
+	} else {
+		fd.Close()
+	}
+	if err := sender.RescanSub("default", "file1.txt", 86400); err != nil {
+		t.Fatal(err)
+	}
+	log.Println("Syncing...")
+	rc.AwaitSync("default", sender, receiver)
+	log.Println("Comparing directories...")
+	err = compareDirectories("s1", "s2")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// 2
+	log.Println("Creating a file in an unknown directory")
+	os.MkdirAll("s1/filetest", 0755)
+	if fd, err := os.Create("s1/filetest/file1.txt"); err != nil {
+		t.Fatal(err)
+	} else {
+		fd.Close()
+	}
+	if err := sender.RescanSub("default", "filetest/file1.txt", 86400); err != nil {
+		t.Fatal(err)
+	}
+	log.Println("Syncing...")
+	rc.AwaitSync("default", sender, receiver)
+	log.Println("Comparing directories...")
+	err = compareDirectories("s1", "s2")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// 3
+	log.Println("Creating a file in an unknown deep directory")
+	os.MkdirAll("s1/filetest/1/2/3/4/5/6/7", 0755)
+	if fd, err := os.Create("s1/filetest/1/2/3/4/5/6/7/file1.txt"); err != nil {
+		t.Fatal(err)
+	} else {
+		fd.Close()
+	}
+	if err := sender.RescanSub("default", "filetest/1/2/3/4/5/6/7/file1.txt", 86400); err != nil {
+		t.Fatal(err)
+	}
+	log.Println("Syncing...")
+	rc.AwaitSync("default", sender, receiver)
+	log.Println("Comparing directories...")
+	err = compareDirectories("s1", "s2")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// 4
+	log.Println("Creating a directory in an unknown directory")
+	err = os.MkdirAll("s1/dirtest/1/2/3/4/5/6/7", 0755)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := sender.RescanSub("default", "dirtest/1/2/3/4/5/6/7", 86400); err != nil {
+		t.Fatal(err)
+	}
+	log.Println("Syncing...")
+	rc.AwaitSync("default", sender, receiver)
+	log.Println("Comparing directories...")
+	err = compareDirectories("s1", "s2")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// 5
+	log.Println("Scan a deleted file in a known directory")
+	if err := os.Remove("s1/filetest/file1.txt"); err != nil {
+		t.Fatal(err)
+	}
+	if err := sender.RescanSub("default", "filetest/file1.txt", 86400); err != nil {
+		t.Fatal(err)
+	}
+	log.Println("Syncing...")
+	rc.AwaitSync("default", sender, receiver)
+	log.Println("Comparing directories...")
+	err = compareDirectories("s1", "s2")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// 6
+	log.Println("Scan a deleted file in an unknown directory")
+	if err := sender.RescanSub("default", "rmdirtest/1/2/3/4/5/6/7", 86400); err != nil {
+		t.Fatal(err)
+	}
+	log.Println("Syncing...")
+	rc.AwaitSync("default", sender, receiver)
+	log.Println("Comparing directories...")
+	err = compareDirectories("s1", "s2")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// 7
+	log.Println("'Accidentally' forget to scan 1 of the 2 files")
+	if fd, err := os.Create("s1/filetest/1/2/3/4/5/6/7/file2.txt"); err != nil {
+		t.Fatal(err)
+	} else {
+		fd.Close()
+	}
+	if fd, err := os.Create("s1/filetest/1/2/3/4/5/6/7/file3.txt"); err != nil {
+		t.Fatal(err)
+	} else {
+		fd.Close()
+	}
+	if err := sender.RescanSub("default", "filetest/1/2/3/4/5/6/7/file2.txt", 86400); err != nil {
+		t.Fatal(err)
+	}
+	log.Println("Syncing...")
+	rc.AwaitSync("default", sender, receiver)
+	log.Println("Comparing directories...")
+	err = compareDirectories("s1", "s2")
+	if err == nil {
+		t.Fatal("filetest/1/2/3/4/5/6/7/file3.txt should not be synced")
+	}
+	os.Remove("s1/filetest/1/2/3/4/5/6/7/file3.txt")
+	err = compareDirectories("s1", "s2")
+	if err != nil {
+		t.Fatal(err)
+	}
+}