octoleo/syncthing
1
0
Fork 0
mirror of https://github.com/octoleo/syncthing.git synced 2024-09-19 05:09:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #953 from syncthing/symlink

Browse Source 
Symlink support
This commit is contained in:
Jakob Borg 2014-11-20 16:34:12 +01:00
commit 2cd9e7fb55
17 changed files with 814 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
internal/files/blockmap.go
View File

@ -29,6 +29,7 @@ import (
"sync"
"github.com/syncthing/syncthing/internal/config"
"github.com/syncthing/syncthing/internal/osutil"
"github.com/syncthing/syncthing/internal/protocol"
"github.com/syndtr/goleveldb/leveldb"
@ -171,7 +172,7 @@ func (f *BlockFinder) Iterate(hash []byte, iterFn func(string, string, uint32) b
for iter.Next() && iter.Error() == nil {
folder, file := fromBlockKey(iter.Key())
index := binary.BigEndian.Uint32(iter.Value())
if iterFn(folder, nativeFilename(file), index) {
if iterFn(folder, osutil.NativeFilename(file), index) {
return true
}
}

17
internal/files/set.go
View File

@ -25,6 +25,7 @@ import (
"sync"
"github.com/syncthing/syncthing/internal/lamport"
"github.com/syncthing/syncthing/internal/osutil"
"github.com/syncthing/syncthing/internal/protocol"
"github.com/syndtr/goleveldb/leveldb"
)
@ -174,19 +175,19 @@ func (s *Set) WithGlobalTruncated(fn fileIterator) {
}
func (s *Set) Get(device protocol.DeviceID, file string) protocol.FileInfo {
f := ldbGet(s.db, []byte(s.folder), device[:], []byte(normalizedFilename(file)))
f.Name = nativeFilename(f.Name)
f := ldbGet(s.db, []byte(s.folder), device[:], []byte(osutil.NormalizedFilename(file)))
f.Name = osutil.NativeFilename(f.Name)
return f
}
func (s *Set) GetGlobal(file string) protocol.FileInfo {
f := ldbGetGlobal(s.db, []byte(s.folder), []byte(normalizedFilename(file)))
f.Name = nativeFilename(f.Name)
f := ldbGetGlobal(s.db, []byte(s.folder), []byte(osutil.NormalizedFilename(file)))
f.Name = osutil.NativeFilename(f.Name)
return f
}
func (s *Set) Availability(file string) []protocol.DeviceID {
return ldbAvailability(s.db, []byte(s.folder), []byte(normalizedFilename(file)))
return ldbAvailability(s.db, []byte(s.folder), []byte(osutil.NormalizedFilename(file)))
}
func (s *Set) LocalVersion(device protocol.DeviceID) uint64 {
@ -213,7 +214,7 @@ func DropFolder(db *leveldb.DB, folder string) {
func normalizeFilenames(fs []protocol.FileInfo) {
for i := range fs {
fs[i].Name = normalizedFilename(fs[i].Name)
fs[i].Name = osutil.NormalizedFilename(fs[i].Name)
}
}
@ -221,10 +222,10 @@ func nativeFileIterator(fn fileIterator) fileIterator {
return func(fi protocol.FileIntf) bool {
switch f := fi.(type) {
case protocol.FileInfo:
f.Name = nativeFilename(f.Name)
f.Name = osutil.NativeFilename(f.Name)
return fn(f)
case protocol.FileInfoTruncated:
f.Name = nativeFilename(f.Name)
f.Name = osutil.NativeFilename(f.Name)
return fn(f)
default:
panic("unknown interface type")

57
internal/model/model.go
View File

@ -39,6 +39,7 @@ import (
"github.com/syncthing/syncthing/internal/protocol"
"github.com/syncthing/syncthing/internal/scanner"
"github.com/syncthing/syncthing/internal/stats"
"github.com/syncthing/syncthing/internal/symlinks"
"github.com/syncthing/syncthing/internal/versioner"
"github.com/syndtr/goleveldb/leveldb"
)
@ -114,6 +115,8 @@ type Model struct {
var (
ErrNoSuchFile = errors.New("no such file")
ErrInvalid = errors.New("file is invalid")
SymlinkWarning = sync.Once{}
)
// NewModel creates and starts a new model. The model starts in read-only mode,
@ -440,9 +443,9 @@ func (m *Model) Index(deviceID protocol.DeviceID, folder string, fs []protocol.F
for i := 0; i < len(fs); {
lamport.Default.Tick(fs[i].Version)
if ignores != nil && ignores.Match(fs[i].Name) {
if (ignores != nil && ignores.Match(fs[i].Name)) || symlinkInvalid(fs[i].IsSymlink()) {
if debug {
l.Debugln("dropping update for ignored", fs[i])
l.Debugln("dropping update for ignored/unsupported symlink", fs[i])
}
fs[i] = fs[len(fs)-1]
fs = fs[:len(fs)-1]
@ -484,9 +487,9 @@ func (m *Model) IndexUpdate(deviceID protocol.DeviceID, folder string, fs []prot
for i := 0; i < len(fs); {
lamport.Default.Tick(fs[i].Version)
if ignores != nil && ignores.Match(fs[i].Name) {
if (ignores != nil && ignores.Match(fs[i].Name)) || symlinkInvalid(fs[i].IsSymlink()) {
if debug {
l.Debugln("dropping update for ignored", fs[i])
l.Debugln("dropping update for ignored/unsupported symlink", fs[i])
}
fs[i] = fs[len(fs)-1]
fs = fs[:len(fs)-1]
@ -655,7 +658,7 @@ func (m *Model) Request(deviceID protocol.DeviceID, folder, name string, offset
}
lf := r.Get(protocol.LocalDeviceID, name)
if protocol.IsInvalid(lf.Flags) || protocol.IsDeleted(lf.Flags) {
if lf.IsInvalid() || lf.IsDeleted() {
if debug {
l.Debugf("%v REQ(in): %s: %q / %q o=%d s=%d; invalid: %v", m, deviceID, folder, name, offset, size, lf)
}
@ -675,14 +678,26 @@ func (m *Model) Request(deviceID protocol.DeviceID, folder, name string, offset
m.fmut.RLock()
fn := filepath.Join(m.folderCfgs[folder].Path, name)
m.fmut.RUnlock()
fd, err := os.Open(fn) // XXX: Inefficient, should cache fd?
if err != nil {
return nil, err
var reader io.ReaderAt
var err error
if lf.IsSymlink() {
target, _, err := symlinks.Read(fn)
if err != nil {
return nil, err
}
reader = strings.NewReader(target)
} else {
reader, err = os.Open(fn) // XXX: Inefficient, should cache fd?
if err != nil {
return nil, err
}
defer reader.(*os.File).Close()
}
defer fd.Close()
buf := make([]byte, size)
_, err = fd.ReadAt(buf, offset)
_, err = reader.ReadAt(buf, offset)
if err != nil {
return nil, err
}
@ -892,9 +907,9 @@ func sendIndexTo(initial bool, minLocalVer uint64, conn protocol.Connection, fol
maxLocalVer = f.LocalVersion
}
if ignores != nil && ignores.Match(f.Name) {
if (ignores != nil && ignores.Match(f.Name)) || symlinkInvalid(f.IsSymlink()) {
if debug {
l.Debugln("not sending update for ignored", f)
l.Debugln("not sending update for ignored/unsupported symlink", f)
}
return true
}
@ -1085,7 +1100,7 @@ func (m *Model) ScanFolderSub(folder, sub string) error {
}
seenPrefix = true
if !protocol.IsDeleted(f.Flags) {
if !f.IsDeleted() {
if f.IsInvalid() {
return true
}
@ -1095,8 +1110,8 @@ func (m *Model) ScanFolderSub(folder, sub string) error {
batch = batch[:0]
}
if ignores != nil && ignores.Match(f.Name) {
// File has been ignored. Set invalid bit.
if (ignores != nil && ignores.Match(f.Name)) || symlinkInvalid(f.IsSymlink()) {
// File has been ignored or an unsupported symlink. Set invalid bit.
l.Debugln("setting invalid bit on ignored", f)
nf := protocol.FileInfo{
Name: f.Name,
@ -1112,7 +1127,7 @@ func (m *Model) ScanFolderSub(folder, sub string) error {
"size": f.Size(),
})
batch = append(batch, nf)
} else if _, err := os.Stat(filepath.Join(dir, f.Name)); err != nil && os.IsNotExist(err) {
} else if _, err := os.Lstat(filepath.Join(dir, f.Name)); err != nil && os.IsNotExist(err) {
// File has been deleted
nf := protocol.FileInfo{
Name: f.Name,
@ -1326,3 +1341,13 @@ func (m *Model) leveldbPanicWorkaround() {
}
}
}
func symlinkInvalid(isLink bool) bool {
if !symlinks.Supported && isLink {
SymlinkWarning.Do(func() {
l.Warnln("Symlinks are unsupported as they require Administrator priviledges. This might cause your folder to appear out of sync.")
})
return true
}
return false
}

58
internal/model/puller.go
View File

@ -20,6 +20,7 @@ import (
"crypto/sha256"
"errors"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"sync"
@ -32,6 +33,7 @@ import (
"github.com/syncthing/syncthing/internal/osutil"
"github.com/syncthing/syncthing/internal/protocol"
"github.com/syncthing/syncthing/internal/scanner"
"github.com/syncthing/syncthing/internal/symlinks"
"github.com/syncthing/syncthing/internal/versioner"
)
@ -313,15 +315,16 @@ func (p *Puller) pullerIteration(ncopiers, npullers, nfinishers int, checksum bo
}
switch {
case protocol.IsDeleted(file.Flags):
// A deleted file or directory
case file.IsDeleted():
// A deleted file, directory or symlink
deletions = append(deletions, file)
case protocol.IsDirectory(file.Flags):
case file.IsDirectory() && !file.IsSymlink():
// A new or changed directory
p.handleDir(file)
default:
// A new or changed file. This is the only case where we do stuff
// in the background; the other three are done synchronously.
// A new or changed file or symlink. This is the only case where we
// do stuff in the background; the other three are done
// synchronously.
p.handleFile(file, copyChan, finisherChan)
}
@ -459,24 +462,21 @@ func (p *Puller) deleteFile(file protocol.FileInfo) {
func (p *Puller) handleFile(file protocol.FileInfo, copyChan chan<- copyBlocksState, finisherChan chan<- *sharedPullerState) {
curFile := p.model.CurrentFolderFile(p.folder, file.Name)
if len(curFile.Blocks) == len(file.Blocks) {
for i := range file.Blocks {
if !bytes.Equal(curFile.Blocks[i].Hash, file.Blocks[i].Hash) {
goto FilesAreDifferent
}
}
if len(curFile.Blocks) == len(file.Blocks) && scanner.BlocksEqual(curFile.Blocks, file.Blocks) {
// We are supposed to copy the entire file, and then fetch nothing. We
// are only updating metadata, so we don't actually *need* to make the
// copy.
if debug {
l.Debugln(p, "taking shortcut on", file.Name)
}
p.shortcutFile(file)
if file.IsSymlink() {
p.shortcutSymlink(curFile, file)
} else {
p.shortcutFile(file)
}
return
}
FilesAreDifferent:
scanner.PopulateOffsets(file.Blocks)
// Figure out the absolute filenames we need once and for all
@ -571,6 +571,17 @@ func (p *Puller) shortcutFile(file protocol.FileInfo) {
p.model.updateLocal(p.folder, file)
}
// shortcutSymlink changes the symlinks type if necessery.
func (p *Puller) shortcutSymlink(curFile, file protocol.FileInfo) {
err := symlinks.ChangeType(filepath.Join(p.dir, file.Name), file.Flags)
if err != nil {
l.Infof("Puller (folder %q, file %q): symlink shortcut: %v", p.folder, file.Name, err)
return
}
p.model.updateLocal(p.folder, file)
}
// copierRoutine reads copierStates until the in channel closes and performs
// the relevant copies when possible, or passes it to the puller routine.
func (p *Puller) copierRoutine(in <-chan copyBlocksState, pullChan chan<- pullBlockState, out chan<- *sharedPullerState, checksum bool) {
@ -791,6 +802,25 @@ func (p *Puller) finisherRoutine(in <-chan *sharedPullerState) {
continue
}
// If it's a symlink, the target of the symlink is inside the file.
if state.file.IsSymlink() {
content, err := ioutil.ReadFile(state.realName)
if err != nil {
l.Warnln("puller: final: reading symlink:", err)
continue
}
// Remove the file, and replace it with a symlink.
err = osutil.InWritableDir(func(path string) error {
os.Remove(path)
return symlinks.Create(path, string(content), state.file.Flags)
}, state.realName)
if err != nil {
l.Warnln("puller: final: creating symlink:", err)
continue
}
}
// Record the updated file in the index
p.model.updateLocal(p.folder, state.file)
}

6
internal/files/filenames_darwin.go → internal/osutil/filenames_darwin.go
View File

@ -13,14 +13,14 @@
// You should have received a copy of the GNU General Public License along
// with this program. If not, see <http://www.gnu.org/licenses/>.
package files
package osutil
import "code.google.com/p/go.text/unicode/norm"
func normalizedFilename(s string) string {
func NormalizedFilename(s string) string {
return norm.NFC.String(s)
}
func nativeFilename(s string) string {
func NativeFilename(s string) string {
return norm.NFD.String(s)
}

6
internal/files/filenames_unix.go → internal/osutil/filenames_unix.go
View File

@ -15,14 +15,14 @@
// +build !windows,!darwin
package files
package osutil
import "code.google.com/p/go.text/unicode/norm"
func normalizedFilename(s string) string {
func NormalizedFilename(s string) string {
return norm.NFC.String(s)
}
func nativeFilename(s string) string {
func NativeFilename(s string) string {
return s
}

6
internal/files/filenames_windows.go → internal/osutil/filenames_windows.go
View File

@ -13,7 +13,7 @@
// You should have received a copy of the GNU General Public License along
// with this program. If not, see <http://www.gnu.org/licenses/>.
package files
package osutil
import (
"path/filepath"
@ -21,10 +21,10 @@ import (
"code.google.com/p/go.text/unicode/norm"
)
func normalizedFilename(s string) string {
func NormalizedFilename(s string) string {
return norm.NFC.String(filepath.ToSlash(s))
}
func nativeFilename(s string) string {
func NativeFilename(s string) string {
return filepath.FromSlash(s)
}

37
internal/protocol/message.go
View File

@ -37,7 +37,7 @@ func (f FileInfo) String() string {
}
func (f FileInfo) Size() (bytes int64) {
if IsDeleted(f.Flags) || IsDirectory(f.Flags) {
if f.IsDeleted() || f.IsDirectory() {
return 128
}
for _, b := range f.Blocks {
@ -47,15 +47,23 @@ func (f FileInfo) Size() (bytes int64) {
}
func (f FileInfo) IsDeleted() bool {
return IsDeleted(f.Flags)
return f.Flags&FlagDeleted != 0
}
func (f FileInfo) IsInvalid() bool {
return IsInvalid(f.Flags)
return f.Flags&FlagInvalid != 0
}
func (f FileInfo) IsDirectory() bool {
return IsDirectory(f.Flags)
return f.Flags&FlagDirectory != 0
}
func (f FileInfo) IsSymlink() bool {
return f.Flags&FlagSymlink != 0
}
func (f FileInfo) HasPermissionBits() bool {
return f.Flags&FlagNoPermBits == 0
}
// Used for unmarshalling a FileInfo structure but skipping the actual block list
@ -75,7 +83,7 @@ func (f FileInfoTruncated) String() string {
// Returns a statistical guess on the size, not the exact figure
func (f FileInfoTruncated) Size() int64 {
if IsDeleted(f.Flags) || IsDirectory(f.Flags) {
if f.IsDeleted() || f.IsDirectory() {
return 128
}
if f.NumBlocks < 2 {
@ -86,17 +94,32 @@ func (f FileInfoTruncated) Size() int64 {
}
func (f FileInfoTruncated) IsDeleted() bool {
return IsDeleted(f.Flags)
return f.Flags&FlagDeleted != 0
}
func (f FileInfoTruncated) IsInvalid() bool {
return IsInvalid(f.Flags)
return f.Flags&FlagInvalid != 0
}
func (f FileInfoTruncated) IsDirectory() bool {
return f.Flags&FlagDirectory != 0
}
func (f FileInfoTruncated) IsSymlink() bool {
return f.Flags&FlagSymlink != 0
}
func (f FileInfoTruncated) HasPermissionBits() bool {
return f.Flags&FlagNoPermBits == 0
}
type FileIntf interface {
Size() int64
IsDeleted() bool
IsInvalid() bool
IsDirectory() bool
IsSymlink() bool
HasPermissionBits() bool
}
type BlockInfo struct {

28
internal/protocol/protocol.go
View File

@ -49,10 +49,14 @@ const (
)
const (
FlagDeleted uint32 = 1 << 12
FlagInvalid = 1 << 13
FlagDirectory = 1 << 14
FlagNoPermBits = 1 << 15
FlagDeleted uint32 = 1 << 12
FlagInvalid = 1 << 13
FlagDirectory = 1 << 14
FlagNoPermBits = 1 << 15
FlagSymlink = 1 << 16
FlagSymlinkMissingTarget = 1 << 17
SymlinkTypeMask = FlagDirectory | FlagSymlinkMissingTarget
)
const (
@ -637,19 +641,3 @@ func (c *rawConnection) Statistics() Statistics {
OutBytesTotal: c.cw.Tot(),
}
}
func IsDeleted(bits uint32) bool {
return bits&FlagDeleted != 0
}
func IsInvalid(bits uint32) bool {
return bits&FlagInvalid != 0
}
func IsDirectory(bits uint32) bool {
return bits&FlagDirectory != 0
}
func HasPermissionBits(bits uint32) bool {
return bits&FlagNoPermBits == 0
}

2
internal/scanner/blockqueue.go
View File

@ -68,7 +68,7 @@ func HashFile(path string, blockSize int) ([]protocol.BlockInfo, error) {
func hashFiles(dir string, blockSize int, outbox, inbox chan protocol.FileInfo) {
for f := range inbox {
if protocol.IsDirectory(f.Flags) || protocol.IsDeleted(f.Flags) {
if f.IsDirectory() || f.IsDeleted() || f.IsSymlink() {
outbox <- f
continue
}

15
internal/scanner/blocks.go
View File

@ -129,3 +129,18 @@ func Verify(r io.Reader, blocksize int, blocks []protocol.BlockInfo) error {
return nil
}
// BlockEqual returns whether two slices of blocks are exactly the same hash
// and index pair wise.
func BlocksEqual(src, tgt []protocol.BlockInfo) bool {
if len(tgt) != len(src) {
return false
}
for i, sblk := range src {
if !bytes.Equal(sblk.Hash, tgt[i].Hash) {
return false
}
}
return true
}

89
internal/scanner/walk.go
View File

@ -27,6 +27,7 @@ import (
"github.com/syncthing/syncthing/internal/ignore"
"github.com/syncthing/syncthing/internal/lamport"
"github.com/syncthing/syncthing/internal/protocol"
"github.com/syncthing/syncthing/internal/symlinks"
)
type Walker struct {
@ -131,11 +132,75 @@ func (w *Walker) walkAndHashFiles(fchan chan protocol.FileInfo) filepath.WalkFun
return nil
}
// We must perform this check, as symlinks on Windows are always
// .IsRegular or .IsDir unlike on Unix.
// Index wise symlinks are always files, regardless of what the target
// is, because symlinks carry their target path as their content.
isSymlink, _ := symlinks.IsSymlink(p)
if isSymlink {
var rval error
// If the target is a directory, do NOT descend down there.
// This will cause files to get tracked, and removing the symlink
// will as a result remove files in their real location.
// But do not SkipDir if the target is not a directory, as it will
// stop scanning the current directory.
if info.IsDir() {
rval = filepath.SkipDir
}
// We always rehash symlinks as they have no modtime or
// permissions.
// We check if they point to the old target by checking that
// their existing blocks match with the blocks in the index.
// If we don't have a filer or don't support symlinks, skip.
if w.CurrentFiler == nil || !symlinks.Supported {
return rval
}
target, flags, err := symlinks.Read(p)
flags = flags & protocol.SymlinkTypeMask
if err != nil {
if debug {
l.Debugln("readlink error:", p, err)
}
return rval
}
blocks, err := Blocks(strings.NewReader(target), w.BlockSize, 0)
if err != nil {
if debug {
l.Debugln("hash link error:", p, err)
}
return rval
}
cf := w.CurrentFiler.CurrentFile(rn)
if !cf.IsDeleted() && cf.IsSymlink() && SymlinkTypeEqual(flags, cf.Flags) && BlocksEqual(cf.Blocks, blocks) {
return rval
}
f := protocol.FileInfo{
Name: rn,
Version: lamport.Default.Tick(0),
Flags: protocol.FlagSymlink | flags | protocol.FlagNoPermBits | 0666,
Modified: 0,
Blocks: blocks,
}
if debug {
l.Debugln("symlink to hash:", p, f)
}
fchan <- f
return rval
}
if info.Mode().IsDir() {
if w.CurrentFiler != nil {
cf := w.CurrentFiler.CurrentFile(rn)
permUnchanged := w.IgnorePerms || !protocol.HasPermissionBits(cf.Flags) || PermsEqual(cf.Flags, uint32(info.Mode()))
if !protocol.IsDeleted(cf.Flags) && protocol.IsDirectory(cf.Flags) && permUnchanged {
permUnchanged := w.IgnorePerms || !cf.HasPermissionBits() || PermsEqual(cf.Flags, uint32(info.Mode()))
if !cf.IsDeleted() && cf.IsDirectory() && permUnchanged {
return nil
}
}
@ -162,8 +227,8 @@ func (w *Walker) walkAndHashFiles(fchan chan protocol.FileInfo) filepath.WalkFun
if info.Mode().IsRegular() {
if w.CurrentFiler != nil {
cf := w.CurrentFiler.CurrentFile(rn)
permUnchanged := w.IgnorePerms || !protocol.HasPermissionBits(cf.Flags) || PermsEqual(cf.Flags, uint32(info.Mode()))
if !protocol.IsDeleted(cf.Flags) && cf.Modified == info.ModTime().Unix() && permUnchanged {
permUnchanged := w.IgnorePerms || !cf.HasPermissionBits() || PermsEqual(cf.Flags, uint32(info.Mode()))
if !cf.IsDeleted() && cf.Modified == info.ModTime().Unix() && permUnchanged {
return nil
}
@ -215,3 +280,19 @@ func PermsEqual(a, b uint32) bool {
return a&0777 == b&0777
}
}
// If the target is missing, Unix never knows what type of symlink it is
// and Windows always knows even if there is no target.
// Which means that without this special check a Unix node would be fighting
// with a Windows node about whether or not the target is known.
// Basically, if you don't know and someone else knows, just accept it.
// The fact that you don't know means you are on Unix, and on Unix you don't
// really care what the target type is. The moment you do know, and if something
// doesn't match, that will propogate throught the cluster.
func SymlinkTypeEqual(disk, index uint32) bool {
if disk&protocol.FlagSymlinkMissingTarget != 0 && index&protocol.FlagSymlinkMissingTarget == 0 {
return true
}
return disk&protocol.SymlinkTypeMask == index&protocol.SymlinkTypeMask
}

58
internal/symlinks/symlink_unix.go Normal file
View File

@ -0,0 +1,58 @@
// Copyright (C) 2014 The Syncthing Authors.
//
// This program is free software: you can redistribute it and/or modify it
// under the terms of the GNU General Public License as published by the Free
// Software Foundation, either version 3 of the License, or (at your option)
// any later version.
//
// This program is distributed in the hope that it will be useful, but WITHOUT
// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
// more details.
//
// You should have received a copy of the GNU General Public License along
// with this program. If not, see <http://www.gnu.org/licenses/>.
// +build !windows
package symlinks
import (
"os"
"github.com/syncthing/syncthing/internal/osutil"
"github.com/syncthing/syncthing/internal/protocol"
)
var (
Supported = true
)
func Read(path string) (string, uint32, error) {
var mode uint32
stat, err := os.Stat(path)
if err != nil {
mode = protocol.FlagSymlinkMissingTarget
} else if stat.IsDir() {
mode = protocol.FlagDirectory
}
path, err = os.Readlink(path)
return osutil.NormalizedFilename(path), mode, err
}
func IsSymlink(path string) (bool, error) {
lstat, err := os.Lstat(path)
if err != nil {
return false, err
}
return lstat.Mode()&os.ModeSymlink != 0, nil
}
func Create(source, target string, flags uint32) error {
return os.Symlink(osutil.NativeFilename(target), source)
}
func ChangeType(path string, flags uint32) error {
return nil
}

203
internal/symlinks/symlink_windows.go Normal file
View File

@ -0,0 +1,203 @@
// Copyright (C) 2014 The Syncthing Authors.
//
// This program is free software: you can redistribute it and/or modify it
// under the terms of the GNU General Public License as published by the Free
// Software Foundation, either version 3 of the License, or (at your option)
// any later version.
//
// This program is distributed in the hope that it will be useful, but WITHOUT
// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
// more details.
//
// You should have received a copy of the GNU General Public License along
// with this program. If not, see <http://www.gnu.org/licenses/>.
// +build windows
package symlinks
import (
"os"
"path/filepath"
"github.com/syncthing/syncthing/internal/osutil"
"github.com/syncthing/syncthing/internal/protocol"
"syscall"
"unicode/utf16"
"unsafe"
)
const (
FSCTL_GET_REPARSE_POINT = 0x900a8
FILE_FLAG_OPEN_REPARSE_POINT = 0x00200000
FILE_ATTRIBUTE_REPARSE_POINT = 0x400
IO_REPARSE_TAG_SYMLINK = 0xA000000C
SYMBOLIC_LINK_FLAG_DIRECTORY = 0x1
)
var (
modkernel32 = syscall.NewLazyDLL("kernel32.dll")
procDeviceIoControl = modkernel32.NewProc("DeviceIoControl")
procCreateSymbolicLink = modkernel32.NewProc("CreateSymbolicLinkW")
Supported = false
)
func init() {
// Needs administrator priviledges.
// Let's check that everything works.
// This could be done more officially:
// http://stackoverflow.com/questions/2094663/determine-if-windows-process-has-privilege-to-create-symbolic-link
// But I don't want to define 10 more structs just to look this up.
base := os.TempDir()
path := filepath.Join(base, "symlinktest")
defer os.Remove(path)
err := Create(path, base, protocol.FlagDirectory)
if err != nil {
return
}
isLink, err := IsSymlink(path)
if err != nil || !isLink {
return
}
target, flags, err := Read(path)
if err != nil || osutil.NativeFilename(target) != base || flags&protocol.FlagDirectory == 0 {
return
}
Supported = true
}
type reparseData struct {
reparseTag uint32
reparseDataLength uint16
reserved uint16
substitueNameOffset uint16
substitueNameLength uint16
printNameOffset uint16
printNameLength uint16
flags uint32
// substituteName - 264 widechars max = 528 bytes
// printName - 260 widechars max = 520 bytes
// = 1048 bytes total
buffer [1048]uint16
}
func (r *reparseData) PrintName() string {
// No clue why the offset and length is doubled...
offset := r.printNameOffset / 2
length := r.printNameLength / 2
return string(utf16.Decode(r.buffer[offset : offset+length]))
}
func (r *reparseData) SubstituteName() string {
// No clue why the offset and length is doubled...
offset := r.substitueNameOffset / 2
length := r.substitueNameLength / 2
return string(utf16.Decode(r.buffer[offset : offset+length]))
}
func Read(path string) (string, uint32, error) {
ptr, err := syscall.UTF16PtrFromString(path)
if err != nil {
return "", protocol.FlagSymlinkMissingTarget, err
}
handle, err := syscall.CreateFile(ptr, 0, syscall.FILE_SHARE_READ|syscall.FILE_SHARE_WRITE|syscall.FILE_SHARE_DELETE, nil, syscall.OPEN_EXISTING, syscall.FILE_FLAG_BACKUP_SEMANTICS|FILE_FLAG_OPEN_REPARSE_POINT, 0)
if err != nil || handle == syscall.InvalidHandle {
return "", protocol.FlagSymlinkMissingTarget, err
}
defer syscall.Close(handle)
var ret uint16
var data reparseData
r1, _, err := syscall.Syscall9(procDeviceIoControl.Addr(), 8, uintptr(handle), FSCTL_GET_REPARSE_POINT, 0, 0, uintptr(unsafe.Pointer(&data)), unsafe.Sizeof(data), uintptr(unsafe.Pointer(&ret)), 0, 0)
if r1 == 0 {
return "", protocol.FlagSymlinkMissingTarget, err
}
var flags uint32 = 0
attr, err := syscall.GetFileAttributes(ptr)
if err != nil {
flags = protocol.FlagSymlinkMissingTarget
} else if attr&syscall.FILE_ATTRIBUTE_DIRECTORY != 0 {
flags = protocol.FlagDirectory
}
return osutil.NormalizedFilename(data.PrintName()), flags, nil
}
func IsSymlink(path string) (bool, error) {
ptr, err := syscall.UTF16PtrFromString(path)
if err != nil {
return false, err
}
attr, err := syscall.GetFileAttributes(ptr)
if err != nil {
return false, err
}
return attr&FILE_ATTRIBUTE_REPARSE_POINT != 0, nil
}
func Create(source, target string, flags uint32) error {
srcp, err := syscall.UTF16PtrFromString(source)
if err != nil {
return err
}
trgp, err := syscall.UTF16PtrFromString(osutil.NativeFilename(target))
if err != nil {
return err
}
// Sadly for Windows we need to specify the type of the symlink,
// whether it's a directory symlink or a file symlink.
// If the flags doesn't reveal the target type, try to evaluate it
// ourselves, and worst case default to the symlink pointing to a file.
mode := 0
if flags&protocol.FlagSymlinkMissingTarget != 0 {
path := target
if !filepath.IsAbs(target) {
path = filepath.Join(filepath.Dir(source), target)
}
stat, err := os.Stat(path)
if err == nil && stat.IsDir() {
mode = SYMBOLIC_LINK_FLAG_DIRECTORY
}
} else if flags&protocol.FlagDirectory != 0 {
mode = SYMBOLIC_LINK_FLAG_DIRECTORY
}
r0, _, err := syscall.Syscall(procCreateSymbolicLink.Addr(), 3, uintptr(unsafe.Pointer(srcp)), uintptr(unsafe.Pointer(trgp)), uintptr(mode))
if r0 == 1 {
return nil
}
return err
}
func ChangeType(path string, flags uint32) error {
target, cflags, err := Read(path)
if err != nil {
return err
}
// If it's the same type, nothing to do.
if cflags&protocol.SymlinkTypeMask == flags&protocol.SymlinkTypeMask {
return nil
}
// If the actual type is unknown, but the new type is file, nothing to do
if cflags&protocol.FlagSymlinkMissingTarget != 0 && flags&protocol.FlagDirectory == 0 {
return nil
}
return osutil.InWritableDir(func(path string) error {
// It should be a symlink as well hence no need to change permissions on
// the file.
os.Remove(path)
return Create(path, target, flags)
}, path)
}

13
protocol/PROTOCOL.md
View File

@ -439,7 +439,7 @@ The Flags field is made up of the following single bit flags:
0 1 2 3
0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| Reserved |P|I|D| Unix Perm. & Mode |
| Reserved |U|S|P|I|D| Unix Perm. & Mode |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- The lower 12 bits hold the common Unix permission and mode bits. An
@ -461,7 +461,16 @@ The Flags field is made up of the following single bit flags:
disregarded on files with this bit set. The permissions bits MUST be
set to the octal value 0666.
- Bit 0 through 16 are reserved for future use and SHALL be set to
- Bit 16 ("S") is set when the file is a symbolic link. The block list
SHALL be of one or more blocks since the target of the symlink is
stored within the blocks of the file.
- Bit 15 ("U") is set when the symbolic links target does not exist.
On systems where symbolic links have types, this bit being means
that the default file symlink SHALL be used. If this bit is unset
bit 19 will decide the type of symlink to be created.
- Bit 0 through 14 are reserved for future use and SHALL be set to
zero.
The hash algorithm is implied by the Hash length. Currently, the hash

19
test/common_test.go
View File

@ -30,6 +30,8 @@ import (
"os/exec"
"path/filepath"
"time"
"github.com/syncthing/syncthing/internal/symlinks"
)
func init() {
@ -355,7 +357,22 @@ func startWalker(dir string, res chan<- fileInfo, abort <-chan struct{}) {
}
var f fileInfo
if info.IsDir() {
if ok, err := symlinks.IsSymlink(path); err == nil && ok {
f = fileInfo{
name: rn,
mode: os.ModeSymlink,
}
tgt, _, err := symlinks.Read(path)
if err != nil {
return err
}
h := md5.New()
h.Write([]byte(tgt))
hash := h.Sum(nil)
copy(f.hash[:], hash)
} else if info.IsDir() {
f = fileInfo{
name: rn,
mode: info.Mode(),

280
test/symlink_test.go Normal file
View File

@ -0,0 +1,280 @@
// Copyright (C) 2014 The Syncthing Authors.
//
// This program is free software: you can redistribute it and/or modify it
// under the terms of the GNU General Public License as published by the Free
// Software Foundation, either version 3 of the License, or (at your option)
// any later version.
//
// This program is distributed in the hope that it will be useful, but WITHOUT
// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
// more details.
//
// You should have received a copy of the GNU General Public License along
// with this program. If not, see <http://www.gnu.org/licenses/>.
// +build integration
package integration_test
import (
"log"
"os"
"strings"
"testing"
"time"
"github.com/syncthing/syncthing/internal/symlinks"
)
func TestSymlinks(t *testing.T) {
log.Println("Cleaning...")
err := removeAll("s1", "s2", "h1/index", "h2/index")
if err != nil {
t.Fatal(err)
}
log.Println("Generating files...")
err = generateFiles("s1", 100, 20, "../bin/syncthing")
if err != nil {
t.Fatal(err)
}
// A file that we will replace with a symlink later
fd, err := os.Create("s1/fileToReplace")
if err != nil {
t.Fatal(err)
}
fd.Close()
// A directory that we will replace with a symlink later
err = os.Mkdir("s1/dirToReplace", 0755)
if err != nil {
t.Fatal(err)
}
// A file and a symlink to that file
fd, err = os.Create("s1/file")
if err != nil {
t.Fatal(err)
}
fd.Close()
err = symlinks.Create("s1/fileLink", "file", 0)
if err != nil {
log.Fatal(err)
}
// A directory and a symlink to that directory
err = os.Mkdir("s1/dir", 0755)
if err != nil {
t.Fatal(err)
}
err = symlinks.Create("s1/dirLink", "dir", 0)
if err != nil {
log.Fatal(err)
}
// A link to something in the repo that does not exist
err = symlinks.Create("s1/noneLink", "does/not/exist", 0)
if err != nil {
log.Fatal(err)
}
// A link we will replace with a file later
err = symlinks.Create("s1/repFileLink", "does/not/exist", 0)
if err != nil {
log.Fatal(err)
}
// A link we will replace with a directory later
err = symlinks.Create("s1/repDirLink", "does/not/exist", 0)
if err != nil {
log.Fatal(err)
}
// Verify that the files and symlinks sync to the other side
log.Println("Syncing...")
sender := syncthingProcess{ // id1
log: "1.out",
argv: []string{"-home", "h1"},
port: 8081,
apiKey: apiKey,
}
err = sender.start()
if err != nil {
t.Fatal(err)
}
receiver := syncthingProcess{ // id2
log: "2.out",
argv: []string{"-home", "h2"},
port: 8082,
apiKey: apiKey,
}
err = receiver.start()
if err != nil {
sender.stop()
t.Fatal(err)
}
for {
comp, err := sender.peerCompletion()
if err != nil {
if strings.Contains(err.Error(), "use of closed network connection") {
time.Sleep(time.Second)
continue
}
sender.stop()
receiver.stop()
t.Fatal(err)
}
curComp := comp[id2]
if curComp == 100 {
sender.stop()
receiver.stop()
break
}
time.Sleep(time.Second)
}
sender.stop()
receiver.stop()
log.Println("Comparing directories...")
err = compareDirectories("s1", "s2")
if err != nil {
t.Fatal(err)
}
log.Println("Making some changes...")
// Remove one symlink
err = os.Remove("s1/fileLink")
if err != nil {
log.Fatal(err)
}
// Change the target of another
err = os.Remove("s1/dirLink")
if err != nil {
log.Fatal(err)
}
err = symlinks.Create("s1/dirLink", "file", 0)
if err != nil {
log.Fatal(err)
}
// Replace one with a file
err = os.Remove("s1/repFileLink")
if err != nil {
log.Fatal(err)
}
fd, err = os.Create("s1/repFileLink")
if err != nil {
log.Fatal(err)
}
fd.Close()
/* Currently fails, to be fixed with #80
// Replace one with a directory
err = os.Remove("s1/repDirLink")
if err != nil {
log.Fatal(err)
}
err = os.Mkdir("s1/repDirLink", 0755)
if err != nil {
log.Fatal(err)
}
*/
// Replace a file with a symlink
err = os.Remove("s1/fileToReplace")
if err != nil {
log.Fatal(err)
}
err = symlinks.Create("s1/fileToReplace", "somewhere/non/existent", 0)
if err != nil {
log.Fatal(err)
}
/* Currently fails, to be fixed with #80
// Replace a directory with a symlink
err = os.RemoveAll("s1/dirToReplace")
if err != nil {
log.Fatal(err)
}
err = symlinks.Create("s1/dirToReplace", "somewhere/non/existent", 0)
if err != nil {
log.Fatal(err)
}
*/
// Sync these changes and recheck
log.Println("Syncing...")
err = sender.start()
if err != nil {
t.Fatal(err)
}
err = receiver.start()
if err != nil {
sender.stop()
t.Fatal(err)
}
for {
comp, err := sender.peerCompletion()
if err != nil {
if strings.Contains(err.Error(), "use of closed network connection") {
time.Sleep(time.Second)
continue
}
sender.stop()
receiver.stop()
t.Fatal(err)
}
curComp := comp[id2]
if curComp == 100 {
sender.stop()
receiver.stop()
break
}
time.Sleep(time.Second)
}
sender.stop()
receiver.stop()
log.Println("Comparing directories...")
err = compareDirectories("s1", "s2")
if err != nil {
t.Fatal(err)
}
}