Merge pull request #833 from AudriusButkevicius/marker

Add folder marker (fixes #762)

cmd/syncthing/main.go

@@ -575,17 +575,11 @@ func setupGUI(cfg *config.ConfigWrapper, m *model.Model) {
 }
 
 func sanityCheckFolders(cfg *config.ConfigWrapper, m *model.Model) {
-	var err error
-
 nextFolder:
 	for id, folder := range cfg.Folders() {
 		if folder.Invalid != "" {
 			continue
 		}
-		folder.Path, err = osutil.ExpandTilde(folder.Path)
-		if err != nil {
-			l.Fatalln("home:", err)
-		}
 		m.AddFolder(folder)
 
 		fi, err := os.Stat(folder.Path)
@@ -598,11 +592,25 @@ nextFolder:
 				l.Warnf("Stopping folder %q - path does not exist, but has files in index", folder.ID)
 				cfg.InvalidateFolder(id, "folder path missing")
 				continue nextFolder
+			} else if !folder.HasMarker() {
+				l.Warnf("Stopping folder %q - path exists, but folder marker missing, check for mount issues", folder.ID)
+				cfg.InvalidateFolder(id, "folder marker missing")
+				continue nextFolder
 			}
 		} else if os.IsNotExist(err) {
 			// If we don't have any files in the index, and the directory
 			// doesn't exist, try creating it.
 			err = os.MkdirAll(folder.Path, 0700)
+			if err != nil {
+				l.Warnf("Stopping folder %q - %v", err)
+				cfg.InvalidateFolder(id, err.Error())
+				continue nextFolder
+			}
+			err = folder.CreateMarker()
+		} else if !folder.HasMarker() {
+			// If we don't have any files in the index, and the path does exist
+			// but the marker is not there, create it.
+			err = folder.CreateMarker()
 		}
 
 		if err != nil {

cmd/syncthing/main_test.go

@@ -13,6 +13,107 @@
 // You should have received a copy of the GNU General Public License along
 // with this program. If not, see <http://www.gnu.org/licenses/>.
 
-package main_test
+package main
 
-// Empty test file to generate 0% coverage rather than no coverage
+import (
+	"os"
+	"testing"
+
+	"github.com/syncthing/syncthing/internal/config"
+	"github.com/syncthing/syncthing/internal/files"
+	"github.com/syncthing/syncthing/internal/model"
+	"github.com/syncthing/syncthing/internal/protocol"
+
+	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/storage"
+)
+
+func TestSanityCheck(t *testing.T) {
+	fcfg := config.FolderConfiguration{
+		ID:   "folder",
+		Path: "testdata/testfolder",
+	}
+	cfg := config.Wrap("/tmp/test", config.Configuration{
+		Folders: []config.FolderConfiguration{fcfg},
+	})
+
+	for _, file := range []string{".stfolder", "testfolder", "testfolder/.stfolder"} {
+		_, err := os.Stat("testdata/" + file)
+		if err == nil {
+			t.Error("Found unexpected file")
+		}
+	}
+
+	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
+
+	// Case 1 - new folder, directory and marker created
+
+	m := model.NewModel(cfg, "device", "syncthing", "dev", db)
+	sanityCheckFolders(cfg, m)
+
+	if cfg.Folders()["folder"].Invalid != "" {
+		t.Error("Unexpected error", cfg.Folders()["folder"].Invalid)
+	}
+
+	s, err := os.Stat("testdata/testfolder")
+	if err != nil || !s.IsDir() {
+		t.Error(err)
+	}
+
+	_, err = os.Stat("testdata/testfolder/.stfolder")
+	if err != nil {
+		t.Error(err)
+	}
+
+	os.Remove("testdata/testfolder/.stfolder")
+	os.Remove("testdata/testfolder/")
+
+	// Case 2 - new folder, marker created
+
+	fcfg.Path = "testdata/"
+	cfg = config.Wrap("/tmp/test", config.Configuration{
+		Folders: []config.FolderConfiguration{fcfg},
+	})
+
+	m = model.NewModel(cfg, "device", "syncthing", "dev", db)
+	sanityCheckFolders(cfg, m)
+
+	if cfg.Folders()["folder"].Invalid != "" {
+		t.Error("Unexpected error", cfg.Folders()["folder"].Invalid)
+	}
+
+	_, err = os.Stat("testdata/.stfolder")
+	if err != nil {
+		t.Error(err)
+	}
+
+	os.Remove("testdata/.stfolder")
+
+	// Case 3 - marker missing
+
+	set := files.NewSet("folder", db)
+	set.Update(protocol.LocalDeviceID, []protocol.FileInfo{
+		{Name: "dummyfile"},
+	})
+
+	m = model.NewModel(cfg, "device", "syncthing", "dev", db)
+	sanityCheckFolders(cfg, m)
+
+	if cfg.Folders()["folder"].Invalid != "folder marker missing" {
+		t.Error("Incorrect error")
+	}
+
+	// Case 4 - path missing
+
+	fcfg.Path = "testdata/testfolder"
+	cfg = config.Wrap("/tmp/test", config.Configuration{
+		Folders: []config.FolderConfiguration{fcfg},
+	})
+
+	m = model.NewModel(cfg, "device", "syncthing", "dev", db)
+	sanityCheckFolders(cfg, m)
+
+	if cfg.Folders()["folder"].Invalid != "folder path missing" {
+		t.Error("Incorrect error")
+	}
+}

internal/config/config.go

@@ -21,18 +21,20 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"path/filepath"
 	"reflect"
 	"sort"
 	"strconv"
 
 	"code.google.com/p/go.crypto/bcrypt"
 	"github.com/syncthing/syncthing/internal/logger"
+	"github.com/syncthing/syncthing/internal/osutil"
 	"github.com/syncthing/syncthing/internal/protocol"
 )
 
 var l = logger.DefaultLogger
 
-const CurrentVersion = 5
+const CurrentVersion = 6
 
 type Configuration struct {
 	Version int                   `xml:"version,attr"`
@@ -64,6 +66,28 @@ type FolderConfiguration struct {
 	Deprecated_Nodes     []FolderDeviceConfiguration `xml:"node" json:"-"`
 }
 
+func (f *FolderConfiguration) CreateMarker() error {
+	if !f.HasMarker() {
+		marker := filepath.Join(f.Path, ".stfolder")
+		fd, err := os.Create(marker)
+		if err != nil {
+			return err
+		}
+		fd.Close()
+		osutil.HideFile(marker)
+	}
+
+	return nil
+}
+
+func (f *FolderConfiguration) HasMarker() bool {
+	_, err := os.Stat(filepath.Join(f.Path, ".stfolder"))
+	if err != nil {
+		return false
+	}
+	return true
+}
+
 func (r *FolderConfiguration) DeviceIDs() []protocol.DeviceID {
 	if r.deviceIDs == nil {
 		for _, n := range r.Devices {
@@ -272,6 +296,11 @@ func (cfg *Configuration) prepare(myID protocol.DeviceID) {
 		convertV4V5(cfg)
 	}
 
+	// Upgrade to v6 configuration if appropriate
+	if cfg.Version == 5 {
+		convertV5V6(cfg)
+	}
+
 	// Hash old cleartext passwords
 	if len(cfg.GUI.Password) > 0 && cfg.GUI.Password[0] != '$' {
 		hash, err := bcrypt.GenerateFromPassword([]byte(cfg.GUI.Password), 0)
@@ -344,6 +373,20 @@ func ChangeRequiresRestart(from, to Configuration) bool {
 	return false
 }
 
+func convertV5V6(cfg *Configuration) {
+	// Added ".stfolder" file at folder roots to identify mount issues
+	// Doesn't affect the config itself, but uses config migrations to identify
+	// the migration point.
+	for _, folder := range Wrap("", *cfg).Folders() {
+		err := folder.CreateMarker()
+		if err != nil {
+			panic(err)
+		}
+	}
+
+	cfg.Version = 6
+}
+
 func convertV4V5(cfg *Configuration) {
 	// Renamed a bunch of fields in the structs.
 	if cfg.Deprecated_Nodes == nil {

internal/config/config_test.go

@@ -16,6 +16,7 @@
 package config
 
 import (
+	"fmt"
 	"os"
 	"reflect"
 	"testing"
@@ -60,17 +61,26 @@ func TestDefaultValues(t *testing.T) {
 }
 
 func TestDeviceConfig(t *testing.T) {
-	for i, ver := range []string{"v1", "v2", "v3", "v4", "v5"} {
+	for i := 1; i <= CurrentVersion; i++ {
-		wr, err := Load("testdata/"+ver+".xml", device1)
+		os.Remove("testdata/.stfolder")
+		wr, err := Load(fmt.Sprintf("testdata/v%d.xml", i), device1)
 		if err != nil {
 			t.Fatal(err)
 		}
+
+		_, err = os.Stat("testdata/.stfolder")
+		if i < 6 && err != nil {
+			t.Fatal(err)
+		} else if i >= 6 && err == nil {
+			t.Fatal("Unexpected file")
+		}
+
 		cfg := wr.cfg
 
 		expectedFolders := []FolderConfiguration{
 			{
 				ID:              "test",
-				Path:            "~/Sync",
+				Path:            "testdata/",
 				Devices:         []FolderDeviceConfiguration{{DeviceID: device1}, {DeviceID: device4}},
 				ReadOnly:        true,
 				RescanIntervalS: 600,
@@ -92,8 +102,8 @@ func TestDeviceConfig(t *testing.T) {
 		}
 		expectedDeviceIDs := []protocol.DeviceID{device1, device4}
 
-		if cfg.Version != 5 {
+		if cfg.Version != CurrentVersion {
-			t.Errorf("%d: Incorrect version %d != 5", i, cfg.Version)
+			t.Errorf("%d: Incorrect version %d != %d", i, cfg.Version, CurrentVersion)
 		}
 		if !reflect.DeepEqual(cfg.Folders, expectedFolders) {
 			t.Errorf("%d: Incorrect Folders\n  A: %#v\n  E: %#v", i, cfg.Folders, expectedFolders)
@@ -296,7 +306,7 @@ func TestPrepare(t *testing.T) {
 }
 
 func TestRequiresRestart(t *testing.T) {
-	wr, err := Load("testdata/v5.xml", device1)
+	wr, err := Load("testdata/v6.xml", device1)
 	if err != nil {
 		t.Fatal(err)
 	}

internal/config/testdata/v1.xml

@@ -1,5 +1,5 @@
 <configuration version="1">
-    <repository id="test" directory="~/Sync">
+    <repository id="test" directory="testdata/">
         <node id="AIR6LPZ7K4PTTUXQSMUUCPQ5YWOEDFIIQJUG7772YQXXR5YD6AWQ" name="node one">
             <address>a</address>
         </node>

internal/config/testdata/v2.xml

@@ -1,5 +1,5 @@
 <configuration version="2">
-    <repository id="test" directory="~/Sync" ro="true">
+    <repository id="test" directory="testdata/" ro="true">
         <node id="P56IOI7MZJNU2IQGDREYDM2MGTMGL3BXNPQ6W5BTBBZ4TJXZWICQ"/>
         <node id="AIR6LPZ7K4PTTUXQSMUUCPQ5YWOEDFIIQJUG7772YQXXR5YD6AWQ"/>
         <node id="C4YBIESWDUAIGU62GOSRXCRAAJDWVE3TKCPMURZE2LH5QHAF576A"/>

internal/config/testdata/v3.xml

@@ -1,5 +1,5 @@
 <configuration version="3">
-    <repository id="test" directory="~/Sync" ro="true" ignorePerms="false">
+    <repository id="test" directory="testdata/" ro="true" ignorePerms="false">
         <node id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR" compression="false"></node>
         <node id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2" compression="false"></node>
     </repository>

internal/config/testdata/v4.xml

@@ -1,5 +1,5 @@
 <configuration version="4">
-    <repository id="test" directory="~/Sync" ro="true" ignorePerms="false" rescanIntervalS="600">
+    <repository id="test" directory="testdata/" ro="true" ignorePerms="false" rescanIntervalS="600">
         <node id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR"></node>
         <node id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2"></node>
     </repository>

internal/config/testdata/v5.xml

@@ -1,5 +1,5 @@
 <configuration version="5">
-    <folder id="test" path="~/Sync" ro="true" ignorePerms="false" rescanIntervalS="600">
+    <folder id="test" path="testdata/" ro="true" ignorePerms="false" rescanIntervalS="600">
         <device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR"></device>
         <device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2"></device>
     </folder>

internal/config/testdata/v6.xml

@@ -0,0 +1,12 @@
+<configuration version="6">
+    <folder id="test" path="testdata/" ro="true" ignorePerms="false" rescanIntervalS="600">
+        <device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR"></device>
+        <device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2"></device>
+    </folder>
+    <device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR" name="node one" compression="true">
+        <address>a</address>
+    </device>
+    <device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2" name="node two" compression="true">
+        <address>b</address>
+    </device>
+</configuration>

internal/config/testdata/versioningconfig.xml

@@ -1,5 +1,5 @@
 <configuration version="2">
-    <repository id="test" directory="~/Sync" ro="true">
+    <repository id="test" directory="testdata/" ro="true">
         <versioning type="simple">
             <param key="foo" val="bar"/>
             <param key="baz" val="quux"/>

internal/config/wrapper.go

@@ -167,6 +167,12 @@ func (w *ConfigWrapper) Folders() map[string]FolderConfiguration {
 	if w.folderMap == nil {
 		w.folderMap = make(map[string]FolderConfiguration, len(w.cfg.Folders))
 		for _, fld := range w.cfg.Folders {
+			path, err := osutil.ExpandTilde(fld.Path)
+			if err != nil {
+				l.Warnln("home:", err)
+				continue
+			}
+			fld.Path = path
 			w.folderMap[fld.ID] = fld
 		}
 	}

internal/scanner/walk.go

@@ -113,7 +113,7 @@ func (w *Walker) walkAndHashFiles(fchan chan protocol.FileInfo) filepath.WalkFun
 			return nil
 		}
 
-		if sn := filepath.Base(rn); sn == ".stignore" || sn == ".stversions" || w.Ignores.Match(rn) {
+		if sn := filepath.Base(rn); sn == ".stignore" || sn == ".stversions" || sn == ".stfolder" || w.Ignores.Match(rn) {
 			// An ignored file
 			if debug {
 				l.Debugln("ignored:", rn)