Add symlink support (fixes #873)

internal/model/model.go

@@ -39,6 +39,7 @@ import (
 	"github.com/syncthing/syncthing/internal/protocol"
 	"github.com/syncthing/syncthing/internal/scanner"
 	"github.com/syncthing/syncthing/internal/stats"
+	"github.com/syncthing/syncthing/internal/symlinks"
 	"github.com/syncthing/syncthing/internal/versioner"
 	"github.com/syndtr/goleveldb/leveldb"
 )
@@ -114,6 +115,8 @@ type Model struct {
 var (
 	ErrNoSuchFile = errors.New("no such file")
 	ErrInvalid    = errors.New("file is invalid")
+
+	SymlinkWarning = sync.Once{}
 )
 
 // NewModel creates and starts a new model. The model starts in read-only mode,
@@ -440,9 +443,9 @@ func (m *Model) Index(deviceID protocol.DeviceID, folder string, fs []protocol.F
 
 	for i := 0; i < len(fs); {
 		lamport.Default.Tick(fs[i].Version)
-		if ignores != nil && ignores.Match(fs[i].Name) {
+		if (ignores != nil && ignores.Match(fs[i].Name)) || symlinkInvalid(fs[i].IsSymlink()) {
 			if debug {
-				l.Debugln("dropping update for ignored", fs[i])
+				l.Debugln("dropping update for ignored/unsupported symlink", fs[i])
 			}
 			fs[i] = fs[len(fs)-1]
 			fs = fs[:len(fs)-1]
@@ -484,9 +487,9 @@ func (m *Model) IndexUpdate(deviceID protocol.DeviceID, folder string, fs []prot
 
 	for i := 0; i < len(fs); {
 		lamport.Default.Tick(fs[i].Version)
-		if ignores != nil && ignores.Match(fs[i].Name) {
+		if (ignores != nil && ignores.Match(fs[i].Name)) || symlinkInvalid(fs[i].IsSymlink()) {
 			if debug {
-				l.Debugln("dropping update for ignored", fs[i])
+				l.Debugln("dropping update for ignored/unsupported symlink", fs[i])
 			}
 			fs[i] = fs[len(fs)-1]
 			fs = fs[:len(fs)-1]
@@ -675,14 +678,26 @@ func (m *Model) Request(deviceID protocol.DeviceID, folder, name string, offset
 	m.fmut.RLock()
 	fn := filepath.Join(m.folderCfgs[folder].Path, name)
 	m.fmut.RUnlock()
-	fd, err := os.Open(fn) // XXX: Inefficient, should cache fd?
+
+	var reader io.ReaderAt
+	var err error
+	if lf.IsSymlink() {
+		target, _, err := symlinks.Read(fn)
 		if err != nil {
 			return nil, err
 		}
-	defer fd.Close()
+		reader = strings.NewReader(target)
+	} else {
+		reader, err = os.Open(fn) // XXX: Inefficient, should cache fd?
+		if err != nil {
+			return nil, err
+		}
+
+		defer reader.(*os.File).Close()
+	}
 
 	buf := make([]byte, size)
-	_, err = fd.ReadAt(buf, offset)
+	_, err = reader.ReadAt(buf, offset)
 	if err != nil {
 		return nil, err
 	}
@@ -892,9 +907,9 @@ func sendIndexTo(initial bool, minLocalVer uint64, conn protocol.Connection, fol
 			maxLocalVer = f.LocalVersion
 		}
 
-		if ignores != nil && ignores.Match(f.Name) {
+		if (ignores != nil && ignores.Match(f.Name)) || symlinkInvalid(f.IsSymlink()) {
 			if debug {
-				l.Debugln("not sending update for ignored", f)
+				l.Debugln("not sending update for ignored/unsupported symlink", f)
 			}
 			return true
 		}
@@ -1095,8 +1110,8 @@ func (m *Model) ScanFolderSub(folder, sub string) error {
 				batch = batch[:0]
 			}
 
-			if ignores != nil && ignores.Match(f.Name) {
-				// File has been ignored. Set invalid bit.
+			if (ignores != nil && ignores.Match(f.Name)) || symlinkInvalid(f.IsSymlink()) {
+				// File has been ignored or an unsupported symlink. Set invalid bit.
 				l.Debugln("setting invalid bit on ignored", f)
 				nf := protocol.FileInfo{
 					Name:     f.Name,
@@ -1112,7 +1127,7 @@ func (m *Model) ScanFolderSub(folder, sub string) error {
 					"size":     f.Size(),
 				})
 				batch = append(batch, nf)
-			} else if _, err := os.Stat(filepath.Join(dir, f.Name)); err != nil && os.IsNotExist(err) {
+			} else if _, err := os.Lstat(filepath.Join(dir, f.Name)); err != nil && os.IsNotExist(err) {
 				// File has been deleted
 				nf := protocol.FileInfo{
 					Name:     f.Name,
@@ -1326,3 +1341,13 @@ func (m *Model) leveldbPanicWorkaround() {
 		}
 	}
 }
+
+func symlinkInvalid(isLink bool) bool {
+	if !symlinks.Supported && isLink {
+		SymlinkWarning.Do(func() {
+			l.Warnln("Symlinks are unsupported as they require Administrator priviledges. This might cause your folder to appear out of sync.")
+		})
+		return true
+	}
+	return false
+}

internal/model/puller.go

@@ -20,6 +20,7 @@ import (
 	"crypto/sha256"
 	"errors"
 	"fmt"
+	"io/ioutil"
 	"os"
 	"path/filepath"
 	"sync"
@@ -32,6 +33,7 @@ import (
 	"github.com/syncthing/syncthing/internal/osutil"
 	"github.com/syncthing/syncthing/internal/protocol"
 	"github.com/syncthing/syncthing/internal/scanner"
+	"github.com/syncthing/syncthing/internal/symlinks"
 	"github.com/syncthing/syncthing/internal/versioner"
 )
 
@@ -314,14 +316,15 @@ func (p *Puller) pullerIteration(ncopiers, npullers, nfinishers int, checksum bo
 
 		switch {
 		case file.IsDeleted():
-			// A deleted file or directory
+			// A deleted file, directory or symlink
 			deletions = append(deletions, file)
-		case file.IsDirectory():
+		case file.IsDirectory() && !file.IsSymlink():
 			// A new or changed directory
 			p.handleDir(file)
 		default:
-			// A new or changed file. This is the only case where we do stuff
-			// in the background; the other three are done synchronously.
+			// A new or changed file or symlink. This is the only case where we
+			// do stuff in the background; the other three are done
+			// synchronously.
 			p.handleFile(file, copyChan, finisherChan)
 		}
 
@@ -459,24 +462,21 @@ func (p *Puller) deleteFile(file protocol.FileInfo) {
 func (p *Puller) handleFile(file protocol.FileInfo, copyChan chan<- copyBlocksState, finisherChan chan<- *sharedPullerState) {
 	curFile := p.model.CurrentFolderFile(p.folder, file.Name)
 
-	if len(curFile.Blocks) == len(file.Blocks) {
-		for i := range file.Blocks {
-			if !bytes.Equal(curFile.Blocks[i].Hash, file.Blocks[i].Hash) {
-				goto FilesAreDifferent
-			}
-		}
+	if len(curFile.Blocks) == len(file.Blocks) && scanner.BlocksEqual(curFile.Blocks, file.Blocks) {
 		// We are supposed to copy the entire file, and then fetch nothing. We
 		// are only updating metadata, so we don't actually *need* to make the
 		// copy.
 		if debug {
 			l.Debugln(p, "taking shortcut on", file.Name)
 		}
+		if file.IsSymlink() {
+			p.shortcutSymlink(curFile, file)
+		} else {
 			p.shortcutFile(file)
+		}
 		return
 	}
 
-FilesAreDifferent:
-
 	scanner.PopulateOffsets(file.Blocks)
 
 	// Figure out the absolute filenames we need once and for all
@@ -571,6 +571,17 @@ func (p *Puller) shortcutFile(file protocol.FileInfo) {
 	p.model.updateLocal(p.folder, file)
 }
 
+// shortcutSymlink changes the symlinks type if necessery.
+func (p *Puller) shortcutSymlink(curFile, file protocol.FileInfo) {
+	err := symlinks.ChangeType(filepath.Join(p.dir, file.Name), file.Flags)
+	if err != nil {
+		l.Infof("Puller (folder %q, file %q): symlink shortcut: %v", p.folder, file.Name, err)
+		return
+	}
+
+	p.model.updateLocal(p.folder, file)
+}
+
 // copierRoutine reads copierStates until the in channel closes and performs
 // the relevant copies when possible, or passes it to the puller routine.
 func (p *Puller) copierRoutine(in <-chan copyBlocksState, pullChan chan<- pullBlockState, out chan<- *sharedPullerState, checksum bool) {
@@ -791,6 +802,25 @@ func (p *Puller) finisherRoutine(in <-chan *sharedPullerState) {
 				continue
 			}
 
+			// If it's a symlink, the target of the symlink is inside the file.
+			if state.file.IsSymlink() {
+				content, err := ioutil.ReadFile(state.realName)
+				if err != nil {
+					l.Warnln("puller: final: reading symlink:", err)
+					continue
+				}
+
+				// Remove the file, and replace it with a symlink.
+				err = osutil.InWritableDir(func(path string) error {
+					os.Remove(path)
+					return symlinks.Create(path, string(content), state.file.Flags)
+				}, state.realName)
+				if err != nil {
+					l.Warnln("puller: final: creating symlink:", err)
+					continue
+				}
+			}
+
 			// Record the updated file in the index
 			p.model.updateLocal(p.folder, state.file)
 		}

internal/scanner/blockqueue.go

@@ -68,7 +68,7 @@ func HashFile(path string, blockSize int) ([]protocol.BlockInfo, error) {
 
 func hashFiles(dir string, blockSize int, outbox, inbox chan protocol.FileInfo) {
 	for f := range inbox {
-		if f.IsDirectory() || f.IsDeleted() {
+		if f.IsDirectory() || f.IsDeleted() || f.IsSymlink() {
 			outbox <- f
 			continue
 		}

internal/scanner/blocks.go

@@ -129,3 +129,18 @@ func Verify(r io.Reader, blocksize int, blocks []protocol.BlockInfo) error {
 
 	return nil
 }
+
+// BlockEqual returns whether two slices of blocks are exactly the same hash
+// and index pair wise.
+func BlocksEqual(src, tgt []protocol.BlockInfo) bool {
+	if len(tgt) != len(src) {
+		return false
+	}
+
+	for i, sblk := range src {
+		if !bytes.Equal(sblk.Hash, tgt[i].Hash) {
+			return false
+		}
+	}
+	return true
+}

internal/scanner/walk.go

@@ -27,6 +27,7 @@ import (
 	"github.com/syncthing/syncthing/internal/ignore"
 	"github.com/syncthing/syncthing/internal/lamport"
 	"github.com/syncthing/syncthing/internal/protocol"
+	"github.com/syncthing/syncthing/internal/symlinks"
 )
 
 type Walker struct {
@@ -131,6 +132,70 @@ func (w *Walker) walkAndHashFiles(fchan chan protocol.FileInfo) filepath.WalkFun
 			return nil
 		}
 
+		// We must perform this check, as symlinks on Windows are always
+		// .IsRegular or .IsDir unlike on Unix.
+		// Index wise symlinks are always files, regardless of what the target
+		// is, because symlinks carry their target path as their content.
+		isSymlink, _ := symlinks.IsSymlink(p)
+		if isSymlink {
+			var rval error
+			// If the target is a directory, do NOT descend down there.
+			// This will cause files to get tracked, and removing the symlink
+			// will as a result remove files in their real location.
+			// But do not SkipDir if the target is not a directory, as it will
+			// stop scanning the current directory.
+			if info.IsDir() {
+				rval = filepath.SkipDir
+			}
+
+			// We always rehash symlinks as they have no modtime or
+			// permissions.
+			// We check if they point to the old target by checking that
+			// their existing blocks match with the blocks in the index.
+			// If we don't have a filer or don't support symlinks, skip.
+			if w.CurrentFiler == nil || !symlinks.Supported {
+				return rval
+			}
+
+			target, flags, err := symlinks.Read(p)
+			flags = flags & protocol.SymlinkTypeMask
+			if err != nil {
+				if debug {
+					l.Debugln("readlink error:", p, err)
+				}
+				return rval
+			}
+
+			blocks, err := Blocks(strings.NewReader(target), w.BlockSize, 0)
+			if err != nil {
+				if debug {
+					l.Debugln("hash link error:", p, err)
+				}
+				return rval
+			}
+
+			cf := w.CurrentFiler.CurrentFile(rn)
+			if !cf.IsDeleted() && cf.IsSymlink() && SymlinkTypeEqual(flags, cf.Flags) && BlocksEqual(cf.Blocks, blocks) {
+				return rval
+			}
+
+			f := protocol.FileInfo{
+				Name:     rn,
+				Version:  lamport.Default.Tick(0),
+				Flags:    protocol.FlagSymlink | flags | protocol.FlagNoPermBits | 0666,
+				Modified: 0,
+				Blocks:   blocks,
+			}
+
+			if debug {
+				l.Debugln("symlink to hash:", p, f)
+			}
+
+			fchan <- f
+
+			return rval
+		}
+
 		if info.Mode().IsDir() {
 			if w.CurrentFiler != nil {
 				cf := w.CurrentFiler.CurrentFile(rn)
@@ -215,3 +280,19 @@ func PermsEqual(a, b uint32) bool {
 		return a&0777 == b&0777
 	}
 }
+
+// If the target is missing, Unix never knows what type of symlink it is
+// and Windows always knows even if there is no target.
+// Which means that without this special check a Unix node would be fighting
+// with a Windows node about whether or not the target is known.
+// Basically, if you don't know and someone else knows, just accept it.
+// The fact that you don't know means you are on Unix, and on Unix you don't
+// really care what the target type is. The moment you do know, and if something
+// doesn't match, that will propogate throught the cluster.
+func SymlinkTypeEqual(disk, index uint32) bool {
+	if disk&protocol.FlagSymlinkMissingTarget != 0 && index&protocol.FlagSymlinkMissingTarget == 0 {
+		return true
+	}
+	return disk&protocol.SymlinkTypeMask == index&protocol.SymlinkTypeMask
+
+}