lib/fs: The interface and basicfs

GitHub-Pull-Request: https://github.com/syncthing/syncthing/pull/3748

lib/fs/basicfs.go

@@ -0,0 +1,96 @@
+// Copyright (C) 2016 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package fs
+
+import (
+	"os"
+	"time"
+)
+
+// The BasicFilesystem implements all aspects by delegating to package os.
+type BasicFilesystem struct {
+}
+
+func NewBasicFilesystem() *BasicFilesystem {
+	return new(BasicFilesystem)
+}
+
+func (f *BasicFilesystem) Chmod(name string, mode FileMode) error {
+	return os.Chmod(name, os.FileMode(mode))
+}
+
+func (f *BasicFilesystem) Chtimes(name string, atime time.Time, mtime time.Time) error {
+	return os.Chtimes(name, atime, mtime)
+}
+
+func (f *BasicFilesystem) Mkdir(name string, perm FileMode) error {
+	return os.Mkdir(name, os.FileMode(perm))
+}
+
+func (f *BasicFilesystem) Lstat(name string) (FileInfo, error) {
+	fi, err := os.Lstat(name)
+	if err != nil {
+		return nil, err
+	}
+	return fsFileInfo{fi}, err
+}
+
+func (f *BasicFilesystem) Remove(name string) error {
+	return os.Remove(name)
+}
+
+func (f *BasicFilesystem) Rename(oldpath, newpath string) error {
+	return os.Rename(oldpath, newpath)
+}
+
+func (f *BasicFilesystem) Stat(name string) (FileInfo, error) {
+	fi, err := os.Stat(name)
+	if err != nil {
+		return nil, err
+	}
+	return fsFileInfo{fi}, err
+}
+
+func (f *BasicFilesystem) DirNames(name string) ([]string, error) {
+	fd, err := os.OpenFile(name, os.O_RDONLY, 0777)
+	if err != nil {
+		return nil, err
+	}
+	defer fd.Close()
+
+	names, err := fd.Readdirnames(-1)
+	if err != nil {
+		return nil, err
+	}
+
+	return names, nil
+}
+
+func (f *BasicFilesystem) Open(name string) (File, error) {
+	return os.Open(name)
+}
+
+func (f *BasicFilesystem) Create(name string) (File, error) {
+	return os.Create(name)
+}
+
+// fsFileInfo implements the fs.FileInfo interface on top of an os.FileInfo.
+type fsFileInfo struct {
+	os.FileInfo
+}
+
+func (e fsFileInfo) Mode() FileMode {
+	return FileMode(e.FileInfo.Mode())
+}
+
+func (e fsFileInfo) IsRegular() bool {
+	return e.FileInfo.Mode().IsRegular()
+}
+
+func (e fsFileInfo) IsSymlink() bool {
+	return e.FileInfo.Mode()&os.ModeSymlink == os.ModeSymlink
+}

lib/fs/basicfs_symlink_unix.go

@@ -0,0 +1,43 @@
+// Copyright (C) 2016 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+// +build !windows
+
+package fs
+
+import "os"
+
+var symlinksSupported = true
+
+func DisableSymlinks() {
+	symlinksSupported = false
+}
+
+func (BasicFilesystem) SymlinksSupported() bool {
+	return symlinksSupported
+}
+
+func (BasicFilesystem) CreateSymlink(name, target string, _ LinkTargetType) error {
+	return os.Symlink(target, name)
+}
+
+func (BasicFilesystem) ChangeSymlinkType(_ string, _ LinkTargetType) error {
+	return nil
+}
+
+func (BasicFilesystem) ReadSymlink(path string) (string, LinkTargetType, error) {
+	tt := LinkTargetUnknown
+	if stat, err := os.Stat(path); err == nil {
+		if stat.IsDir() {
+			tt = LinkTargetDirectory
+		} else {
+			tt = LinkTargetFile
+		}
+	}
+
+	path, err := os.Readlink(path)
+	return path, tt, err
+}

lib/fs/basicfs_symlink_windows.go

@@ -0,0 +1,195 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+// +build windows
+
+package fs
+
+import (
+	"os"
+	"path/filepath"
+
+	"github.com/syncthing/syncthing/lib/osutil"
+
+	"syscall"
+	"unicode/utf16"
+	"unsafe"
+)
+
+const (
+	win32FsctlGetReparsePoint      = 0x900a8
+	win32FileFlagOpenReparsePoint  = 0x00200000
+	win32SymbolicLinkFlagDirectory = 0x1
+)
+
+var (
+	modkernel32            = syscall.NewLazyDLL("kernel32.dll")
+	procDeviceIoControl    = modkernel32.NewProc("DeviceIoControl")
+	procCreateSymbolicLink = modkernel32.NewProc("CreateSymbolicLinkW")
+	symlinksSupported      = false
+)
+
+func init() {
+	defer func() {
+		if err := recover(); err != nil {
+			// Ensure that the supported flag is disabled when we hit an
+			// error, even though it should already be. Also, silently swallow
+			// the error since it's fine for a system not to support symlinks.
+			symlinksSupported = false
+		}
+	}()
+
+	// Needs administrator privileges.
+	// Let's check that everything works.
+	// This could be done more officially:
+	// http://stackoverflow.com/questions/2094663/determine-if-windows-process-has-privilege-to-create-symbolic-link
+	// But I don't want to define 10 more structs just to look this up.
+	base := os.TempDir()
+	path := filepath.Join(base, "symlinktest")
+	defer os.Remove(path)
+
+	err := DefaultFilesystem.CreateSymlink(path, base, LinkTargetDirectory)
+	if err != nil {
+		return
+	}
+
+	stat, err := osutil.Lstat(path)
+	if err != nil || stat.Mode()&os.ModeSymlink == 0 {
+		return
+	}
+
+	target, tt, err := DefaultFilesystem.ReadSymlink(path)
+	if err != nil || osutil.NativeFilename(target) != base || tt != LinkTargetDirectory {
+		return
+	}
+	symlinksSupported = true
+}
+
+func DisableSymlinks() {
+	symlinksSupported = false
+}
+
+func (BasicFilesystem) SymlinksSupported() bool {
+	return symlinksSupported
+}
+
+func (BasicFilesystem) ReadSymlink(path string) (string, LinkTargetType, error) {
+	ptr, err := syscall.UTF16PtrFromString(path)
+	if err != nil {
+		return "", LinkTargetUnknown, err
+	}
+	handle, err := syscall.CreateFile(ptr, 0, syscall.FILE_SHARE_READ|syscall.FILE_SHARE_WRITE|syscall.FILE_SHARE_DELETE, nil, syscall.OPEN_EXISTING, syscall.FILE_FLAG_BACKUP_SEMANTICS|win32FileFlagOpenReparsePoint, 0)
+	if err != nil || handle == syscall.InvalidHandle {
+		return "", LinkTargetUnknown, err
+	}
+	defer syscall.Close(handle)
+	var ret uint16
+	var data reparseData
+
+	r1, _, err := syscall.Syscall9(procDeviceIoControl.Addr(), 8, uintptr(handle), win32FsctlGetReparsePoint, 0, 0, uintptr(unsafe.Pointer(&data)), unsafe.Sizeof(data), uintptr(unsafe.Pointer(&ret)), 0, 0)
+	if r1 == 0 {
+		return "", LinkTargetUnknown, err
+	}
+
+	tt := LinkTargetUnknown
+	if attr, err := syscall.GetFileAttributes(ptr); err == nil {
+		if attr&syscall.FILE_ATTRIBUTE_DIRECTORY != 0 {
+			tt = LinkTargetDirectory
+		} else {
+			tt = LinkTargetFile
+		}
+	}
+
+	return osutil.NormalizedFilename(data.printName()), tt, nil
+}
+
+func (BasicFilesystem) CreateSymlink(path, target string, tt LinkTargetType) error {
+	srcp, err := syscall.UTF16PtrFromString(path)
+	if err != nil {
+		return err
+	}
+
+	trgp, err := syscall.UTF16PtrFromString(osutil.NativeFilename(target))
+	if err != nil {
+		return err
+	}
+
+	// Sadly for Windows we need to specify the type of the symlink,
+	// whether it's a directory symlink or a file symlink.
+	// If the flags doesn't reveal the target type, try to evaluate it
+	// ourselves, and worst case default to the symlink pointing to a file.
+	mode := 0
+	if tt == LinkTargetUnknown {
+		path := target
+		if !filepath.IsAbs(target) {
+			path = filepath.Join(filepath.Dir(path), target)
+		}
+
+		stat, err := os.Stat(path)
+		if err == nil && stat.IsDir() {
+			mode = win32SymbolicLinkFlagDirectory
+		}
+	} else if tt == LinkTargetDirectory {
+		mode = win32SymbolicLinkFlagDirectory
+	}
+
+	r0, _, err := syscall.Syscall(procCreateSymbolicLink.Addr(), 3, uintptr(unsafe.Pointer(srcp)), uintptr(unsafe.Pointer(trgp)), uintptr(mode))
+	if r0 == 1 {
+		return nil
+	}
+	return err
+}
+
+func (fs BasicFilesystem) ChangeSymlinkType(path string, tt LinkTargetType) error {
+	target, existingTargetType, err := fs.ReadSymlink(path)
+	if err != nil {
+		return err
+	}
+	// If it's the same type, nothing to do.
+	if tt == existingTargetType {
+		return nil
+	}
+
+	// If the actual type is unknown, but the new type is file, nothing to do
+	if existingTargetType == LinkTargetUnknown && tt != LinkTargetDirectory {
+		return nil
+	}
+	return osutil.InWritableDir(func(path string) error {
+		// It should be a symlink as well hence no need to change permissions on
+		// the file.
+		os.Remove(path)
+		return fs.CreateSymlink(path, target, tt)
+	}, path)
+}
+
+type reparseData struct {
+	reparseTag          uint32
+	reparseDataLength   uint16
+	reserved            uint16
+	substitueNameOffset uint16
+	substitueNameLength uint16
+	printNameOffset     uint16
+	printNameLength     uint16
+	flags               uint32
+	// substituteName - 264 widechars max = 528 bytes
+	// printName      - 260 widechars max = 520 bytes
+	//                                    = 1048 bytes total
+	buffer [1048 / 2]uint16
+}
+
+func (r *reparseData) printName() string {
+	// offset and length are in bytes but we're indexing a []uint16
+	offset := r.printNameOffset / 2
+	length := r.printNameLength / 2
+	return string(utf16.Decode(r.buffer[offset : offset+length]))
+}
+
+func (r *reparseData) substituteName() string {
+	// offset and length are in bytes but we're indexing a []uint16
+	offset := r.substitueNameOffset / 2
+	length := r.substitueNameLength / 2
+	return string(utf16.Decode(r.buffer[offset : offset+length]))
+}

lib/fs/basicfs_walk.go

@@ -0,0 +1,81 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This part copied directly from golang.org/src/path/filepath/path.go (Go
+// 1.6) and lightly modified to be methods on BasicFilesystem.
+
+// In our Walk() all paths given to a WalkFunc() are relative to the
+// filesystem root.
+
+package fs
+
+import "path/filepath"
+
+// WalkFunc is the type of the function called for each file or directory
+// visited by Walk. The path argument contains the argument to Walk as a
+// prefix; that is, if Walk is called with "dir", which is a directory
+// containing the file "a", the walk function will be called with argument
+// "dir/a". The info argument is the FileInfo for the named path.
+//
+// If there was a problem walking to the file or directory named by path, the
+// incoming error will describe the problem and the function can decide how
+// to handle that error (and Walk will not descend into that directory). If
+// an error is returned, processing stops. The sole exception is when the function
+// returns the special value SkipDir. If the function returns SkipDir when invoked
+// on a directory, Walk skips the directory's contents entirely.
+// If the function returns SkipDir when invoked on a non-directory file,
+// Walk skips the remaining files in the containing directory.
+type WalkFunc func(path string, info FileInfo, err error) error
+
+// walk recursively descends path, calling walkFn.
+func (f *BasicFilesystem) walk(path string, info FileInfo, walkFn WalkFunc) error {
+	err := walkFn(path, info, nil)
+	if err != nil {
+		if info.IsDir() && err == SkipDir {
+			return nil
+		}
+		return err
+	}
+
+	if !info.IsDir() {
+		return nil
+	}
+
+	names, err := f.DirNames(path)
+	if err != nil {
+		return walkFn(path, info, err)
+	}
+
+	for _, name := range names {
+		filename := filepath.Join(path, name)
+		fileInfo, err := f.Lstat(filename)
+		if err != nil {
+			if err := walkFn(filename, fileInfo, err); err != nil && err != SkipDir {
+				return err
+			}
+		} else {
+			err = f.walk(filename, fileInfo, walkFn)
+			if err != nil {
+				if !fileInfo.IsDir() || err != SkipDir {
+					return err
+				}
+			}
+		}
+	}
+	return nil
+}
+
+// Walk walks the file tree rooted at root, calling walkFn for each file or
+// directory in the tree, including root. All errors that arise visiting files
+// and directories are filtered by walkFn. The files are walked in lexical
+// order, which makes the output deterministic but means that for very
+// large directories Walk can be inefficient.
+// Walk does not follow symbolic links.
+func (f *BasicFilesystem) Walk(root string, walkFn WalkFunc) error {
+	info, err := f.Lstat(root)
+	if err != nil {
+		return walkFn(root, nil, err)
+	}
+	return f.walk(root, info, walkFn)
+}

lib/fs/filesystem.go

@@ -0,0 +1,77 @@
+// Copyright (C) 2016 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package fs
+
+import (
+	"errors"
+	"io"
+	"time"
+)
+
+type LinkTargetType int
+
+const (
+	LinkTargetFile LinkTargetType = iota
+	LinkTargetDirectory
+	LinkTargetUnknown
+)
+
+// The Filesystem interface abstracts access to the file system.
+type Filesystem interface {
+	ChangeSymlinkType(name string, tt LinkTargetType) error
+	Chmod(name string, mode FileMode) error
+	Chtimes(name string, atime time.Time, mtime time.Time) error
+	Create(name string) (File, error)
+	CreateSymlink(name, target string, tt LinkTargetType) error
+	DirNames(name string) ([]string, error)
+	Lstat(name string) (FileInfo, error)
+	Mkdir(name string, perm FileMode) error
+	Open(name string) (File, error)
+	ReadSymlink(name string) (string, LinkTargetType, error)
+	Remove(name string) error
+	Rename(oldname, newname string) error
+	Stat(name string) (FileInfo, error)
+	SymlinksSupported() bool
+	Walk(root string, walkFn WalkFunc) error
+}
+
+// The File interface abstracts access to a regular file, being a somewhat
+// smaller interface than os.File
+type File interface {
+	io.Reader
+	io.WriterAt
+	io.Closer
+	Truncate(size int64) error
+}
+
+// The FileInfo interface is almost the same as os.FileInfo, but with the
+// Sys method removed (as we don't want to expose whatever is underlying)
+// and with a couple of convenience methods added.
+type FileInfo interface {
+	// Standard things present in os.FileInfo
+	Name() string
+	Mode() FileMode
+	Size() int64
+	ModTime() time.Time
+	IsDir() bool
+	// Extensions
+	IsRegular() bool
+	IsSymlink() bool
+}
+
+// FileMode is similar to os.FileMode
+type FileMode uint32
+
+// DefaultFilesystem is the fallback to use when nothing explicitly has
+// been passed.
+var DefaultFilesystem Filesystem = new(BasicFilesystem)
+
+// SkipDir is used as a return value from WalkFuncs to indicate that
+// the directory named in the call is to be skipped. It is not returned
+// as an error by any function.
+var errSkipDir = errors.New("skip this directory")
+var SkipDir = errSkipDir // silences the lint warning...