From 0ca1f26ff8c01bf7d93ffdb4d375db32863ec215 Mon Sep 17 00:00:00 2001 From: Audrius Butkevicius Date: Sun, 28 Apr 2019 23:30:16 +0100 Subject: [PATCH] lib/versioner: Restore for all versioners, cross-device support (#5514) * lib/versioner: Restore for all versioners, cross-device support Fixes #4631 Fixes #4586 Fixes #1634 Fixes #5338 Fixes #5419 --- lib/fs/util.go | 52 ++++++++ lib/fs/util_test.go | 46 +++++++ lib/model/folder_sendrecv.go | 6 +- lib/model/model.go | 111 ++-------------- lib/model/model_test.go | 9 +- lib/osutil/osutil.go | 71 +++++++---- lib/osutil/osutil_test.go | 79 +++++++++++- lib/versioner/external.go | 9 ++ lib/versioner/simple.go | 68 ++++------ lib/versioner/staggered.go | 65 ++-------- lib/versioner/trashcan.go | 96 ++++++-------- lib/versioner/trashcan_test.go | 84 ++++++++++++ lib/versioner/util.go | 225 ++++++++++++++++++++++++++++++++- lib/versioner/versioner.go | 4 + 14 files changed, 636 insertions(+), 289 deletions(-) create mode 100644 lib/fs/util_test.go diff --git a/lib/fs/util.go b/lib/fs/util.go index e1d9836ad..e3f231830 100644 --- a/lib/fs/util.go +++ b/lib/fs/util.go @@ -103,3 +103,55 @@ func IsParent(path, parent string) bool { } return strings.HasPrefix(path, parent) } + +func CommonPrefix(first, second string) string { + if filepath.IsAbs(first) != filepath.IsAbs(second) { + // Whatever + return "" + } + + firstParts := strings.Split(filepath.Clean(first), string(PathSeparator)) + secondParts := strings.Split(filepath.Clean(second), string(PathSeparator)) + + isAbs := filepath.IsAbs(first) && filepath.IsAbs(second) + + count := len(firstParts) + if len(secondParts) < len(firstParts) { + count = len(secondParts) + } + + common := make([]string, 0, count) + for i := 0; i < count; i++ { + if firstParts[i] != secondParts[i] { + break + } + common = append(common, firstParts[i]) + } + + if isAbs { + if runtime.GOOS == "windows" && isVolumeNameOnly(common) { + // Because strings.Split strips out path separators, if we're at the volume name, we end up without a separator + // Wedge an empty element to be joined with. + common = append(common, "") + } else if len(common) == 1 { + // If isAbs on non Windows, first element in both first and second is "", hence joining that returns nothing. + return string(PathSeparator) + } + } + + // This should only be true on Windows when drive letters are different or when paths are relative. + // In case of UNC paths we should end up with more than a single element hence joining is fine + if len(common) == 0 { + return "" + } + + // This has to be strings.Join, because filepath.Join([]string{"", "", "?", "C:", "Audrius"}...) returns garbage + result := strings.Join(common, string(PathSeparator)) + return filepath.Clean(result) +} + +func isVolumeNameOnly(parts []string) bool { + isNormalVolumeName := len(parts) == 1 && strings.HasSuffix(parts[0], ":") + isUNCVolumeName := len(parts) == 4 && strings.HasSuffix(parts[3], ":") + return isNormalVolumeName || isUNCVolumeName +} diff --git a/lib/fs/util_test.go b/lib/fs/util_test.go new file mode 100644 index 000000000..65a972334 --- /dev/null +++ b/lib/fs/util_test.go @@ -0,0 +1,46 @@ +// Copyright (C) 2019 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at https://mozilla.org/MPL/2.0/. + +package fs + +import ( + "runtime" + "testing" +) + +func TestCommonPrefix(t *testing.T) { + test := func(first, second, expect string) { + t.Helper() + res := CommonPrefix(first, second) + if res != expect { + t.Errorf("Expected %s got %s", expect, res) + } + } + + if runtime.GOOS == "windows" { + test(`c:\Audrius\Downloads`, `c:\Audrius\Docs`, `c:\Audrius`) + test(`c:\Audrius\Downloads`, `C:\Audrius\Docs`, ``) // Case differences :( + test(`C:\Audrius-a\Downloads`, `C:\Audrius-b\Docs`, `C:\`) + test(`\\?\C:\Audrius-a\Downloads`, `\\?\C:\Audrius-b\Docs`, `\\?\C:\`) + test(`\\?\C:\Audrius\Downloads`, `\\?\C:\Audrius\Docs`, `\\?\C:\Audrius`) + test(`Audrius-a\Downloads`, `Audrius-b\Docs`, ``) + test(`Audrius\Downloads`, `Audrius\Docs`, `Audrius`) + test(`c:\Audrius\Downloads`, `Audrius\Docs`, ``) + test(`c:\`, `c:\`, `c:\`) + test(`\\?\c:\`, `\\?\c:\`, `\\?\c:\`) + } else { + test(`/Audrius/Downloads`, `/Audrius/Docs`, `/Audrius`) + test(`/Audrius\Downloads`, `/Audrius\Docs`, `/`) + test(`/Audrius-a/Downloads`, `/Audrius-b/Docs`, `/`) + test(`Audrius\Downloads`, `Audrius\Docs`, ``) // Windows separators + test(`Audrius/Downloads`, `Audrius/Docs`, `Audrius`) + test(`Audrius-a\Downloads`, `Audrius-b\Docs`, ``) + test(`/Audrius/Downloads`, `Audrius/Docs`, ``) + test(`/`, `/`, `/`) + } + test(`Audrius`, `Audrius`, `Audrius`) + test(`.`, `.`, `.`) +} diff --git a/lib/model/folder_sendrecv.go b/lib/model/folder_sendrecv.go index 12499be38..5b0921073 100644 --- a/lib/model/folder_sendrecv.go +++ b/lib/model/folder_sendrecv.go @@ -941,13 +941,13 @@ func (f *sendReceiveFolder) renameFile(cur, source, target protocol.FileInfo, db if f.versioner != nil { err = f.CheckAvailableSpace(source.Size) if err == nil { - err = osutil.Copy(f.fs, source.Name, tempName) + err = osutil.Copy(f.fs, f.fs, source.Name, tempName) if err == nil { err = osutil.InWritableDir(f.versioner.Archive, f.fs, source.Name) } } } else { - err = osutil.TryRename(f.fs, source.Name, tempName) + err = osutil.RenameOrCopy(f.fs, f.fs, source.Name, tempName) } if err != nil { return err @@ -1510,7 +1510,7 @@ func (f *sendReceiveFolder) performFinish(file, curFile protocol.FileInfo, hasCu // Replace the original content with the new one. If it didn't work, // leave the temp file in place for reuse. - if err := osutil.TryRename(f.fs, tempName, file.Name); err != nil { + if err := osutil.RenameOrCopy(f.fs, f.fs, tempName, file.Name); err != nil { return err } diff --git a/lib/model/model.go b/lib/model/model.go index ad9eabb64..bb94fd104 100644 --- a/lib/model/model.go +++ b/lib/model/model.go @@ -2310,58 +2310,12 @@ func (m *model) GetFolderVersions(folder string) (map[string][]versioner.FileVer return nil, errFolderMissing } - files := make(map[string][]versioner.FileVersion) - - filesystem := fcfg.Filesystem() - err := filesystem.Walk(".stversions", func(path string, f fs.FileInfo, err error) error { - // Skip root (which is ok to be a symlink) - if path == ".stversions" { - return nil - } - - // Skip walking if we cannot walk... - if err != nil { - return err - } - - // Ignore symlinks - if f.IsSymlink() { - return fs.SkipDir - } - - // No records for directories - if f.IsDir() { - return nil - } - - // Strip .stversions prefix. - path = strings.TrimPrefix(path, ".stversions"+string(fs.PathSeparator)) - - name, tag := versioner.UntagFilename(path) - // Something invalid - if name == "" || tag == "" { - return nil - } - - name = osutil.NormalizedFilename(name) - - versionTime, err := time.ParseInLocation(versioner.TimeFormat, tag, locationLocal) - if err != nil { - return nil - } - - files[name] = append(files[name], versioner.FileVersion{ - VersionTime: versionTime.Truncate(time.Second), - ModTime: f.ModTime().Truncate(time.Second), - Size: f.Size(), - }) - return nil - }) - if err != nil { - return nil, err + ver := fcfg.Versioner() + if ver == nil { + return nil, errors.New("no versioner configured") } - return files, nil + return ver.GetVersions() } func (m *model) RestoreFolderVersions(folder string, versions map[string]time.Time) (map[string]string, error) { @@ -2370,69 +2324,22 @@ func (m *model) RestoreFolderVersions(folder string, versions map[string]time.Ti return nil, errFolderMissing } - filesystem := fcfg.Filesystem() ver := fcfg.Versioner() - restore := make(map[string]string) - errors := make(map[string]string) + restoreErrors := make(map[string]string) - // Validation for file, version := range versions { - file = osutil.NativeFilename(file) - tag := version.In(locationLocal).Truncate(time.Second).Format(versioner.TimeFormat) - versionedTaggedFilename := filepath.Join(".stversions", versioner.TagFilename(file, tag)) - // Check that the thing we've been asked to restore is actually a file - // and that it exists. - if info, err := filesystem.Lstat(versionedTaggedFilename); err != nil { - errors[file] = err.Error() - continue - } else if !info.IsRegular() { - errors[file] = "not a file" - continue - } - - // Check that the target location of where we are supposed to restore - // either does not exist, or is actually a file. - if info, err := filesystem.Lstat(file); err == nil && !info.IsRegular() { - errors[file] = "cannot replace a non-file" - continue - } else if err != nil && !fs.IsNotExist(err) { - errors[file] = err.Error() - continue - } - - restore[file] = versionedTaggedFilename - } - - // Execution - var err error - for target, source := range restore { - err = nil - if _, serr := filesystem.Lstat(target); serr == nil { - if ver != nil { - err = osutil.InWritableDir(ver.Archive, filesystem, target) - } else { - err = osutil.InWritableDir(filesystem.Remove, filesystem, target) - } - } - - filesystem.MkdirAll(filepath.Dir(target), 0755) - if err == nil { - err = osutil.Copy(filesystem, source, target) - } - - if err != nil { - errors[target] = err.Error() - continue + if err := ver.Restore(file, version); err != nil { + restoreErrors[file] = err.Error() } } // Trigger scan if !fcfg.FSWatcherEnabled { - m.ScanFolder(folder) + go func() { _ = m.ScanFolder(folder) }() } - return errors, nil + return restoreErrors, nil } func (m *model) Availability(folder string, file protocol.FileInfo, block protocol.BlockInfo) []Availability { diff --git a/lib/model/model_test.go b/lib/model/model_test.go index 4d20eed99..902d8f652 100644 --- a/lib/model/model_test.go +++ b/lib/model/model_test.go @@ -3151,8 +3151,8 @@ func TestVersionRestore(t *testing.T) { ".stversions/dir/file~20171210-040406.txt", ".stversions/very/very/deep/one~20171210-040406.txt", // lives deep down, no directory exists. ".stversions/dir/existing~20171210-040406.txt", // exists, should expect to be archived. - ".stversions/dir/file.txt~20171210-040405", // incorrect tag format, ignored. - ".stversions/dir/cat", // incorrect tag format, ignored. + ".stversions/dir/file.txt~20171210-040405", // old tag format, supported + ".stversions/dir/cat", // untagged which was used by trashcan, supported // "file.txt" will be restored "existing", @@ -3182,9 +3182,10 @@ func TestVersionRestore(t *testing.T) { "file.txt": 1, "existing": 1, "something": 1, - "dir/file.txt": 3, + "dir/file.txt": 4, "dir/existing.txt": 1, "very/very/deep/one.txt": 1, + "dir/cat": 1, } for name, vers := range versions { @@ -3229,7 +3230,7 @@ func TestVersionRestore(t *testing.T) { ferr, err := m.RestoreFolderVersions("default", restore) must(t, err) - if err, ok := ferr["something"]; len(ferr) > 1 || !ok || err != "cannot replace a non-file" { + if err, ok := ferr["something"]; len(ferr) > 1 || !ok || err != "cannot restore on top of a directory" { t.Fatalf("incorrect error or count: %d %s", len(ferr), ferr) } diff --git a/lib/osutil/osutil.go b/lib/osutil/osutil.go index 256ca1dab..a05f9ff60 100644 --- a/lib/osutil/osutil.go +++ b/lib/osutil/osutil.go @@ -22,37 +22,62 @@ import ( // often enough that there is any contention on this lock. var renameLock = sync.NewMutex() -// TryRename renames a file, leaving source file intact in case of failure. +// RenameOrCopy renames a file, leaving source file intact in case of failure. // Tries hard to succeed on various systems by temporarily tweaking directory // permissions and removing the destination file when necessary. -func TryRename(filesystem fs.Filesystem, from, to string) error { +func RenameOrCopy(src, dst fs.Filesystem, from, to string) error { renameLock.Lock() defer renameLock.Unlock() - return withPreparedTarget(filesystem, from, to, func() error { - return filesystem.Rename(from, to) - }) -} + return withPreparedTarget(dst, from, to, func() error { + // Optimisation 1 + if src.Type() == dst.Type() && src.URI() == dst.URI() { + return src.Rename(from, to) + } -// Rename moves a temporary file to its final place. -// Will make sure to delete the from file if the operation fails, so use only -// for situations like committing a temp file to its final location. -// Tries hard to succeed on various systems by temporarily tweaking directory -// permissions and removing the destination file when necessary. -func Rename(filesystem fs.Filesystem, from, to string) error { - // Don't leave a dangling temp file in case of rename error - if !(runtime.GOOS == "windows" && strings.EqualFold(from, to)) { - defer filesystem.Remove(from) - } - return TryRename(filesystem, from, to) + // "Optimisation" 2 + // Try to find a common prefix between the two filesystems, use that as the base for the new one + // and try a rename. + if src.Type() == dst.Type() { + commonPrefix := fs.CommonPrefix(src.URI(), dst.URI()) + if len(commonPrefix) > 0 { + commonFs := fs.NewFilesystem(src.Type(), commonPrefix) + err := commonFs.Rename( + filepath.Join(strings.TrimPrefix(src.URI(), commonPrefix), from), + filepath.Join(strings.TrimPrefix(dst.URI(), commonPrefix), to), + ) + if err == nil { + return nil + } + } + } + + // Everything is sad, do a copy and delete. + if _, err := dst.Stat(to); !fs.IsNotExist(err) { + err := dst.Remove(to) + if err != nil { + return err + } + } + + err := copyFileContents(src, dst, from, to) + if err != nil { + _ = dst.Remove(to) + return err + } + + return withPreparedTarget(src, from, from, func() error { + return src.Remove(from) + }) + }) } // Copy copies the file content from source to destination. // Tries hard to succeed on various systems by temporarily tweaking directory // permissions and removing the destination file when necessary. -func Copy(filesystem fs.Filesystem, from, to string) (err error) { - return withPreparedTarget(filesystem, from, to, func() error { - return copyFileContents(filesystem, from, to) +func Copy(src, dst fs.Filesystem, from, to string) (err error) { + return withPreparedTarget(dst, from, to, func() error { + return copyFileContents(src, dst, from, to) }) } @@ -115,13 +140,13 @@ func withPreparedTarget(filesystem fs.Filesystem, from, to string, f func() erro // by dst. The file will be created if it does not already exist. If the // destination file exists, all its contents will be replaced by the contents // of the source file. -func copyFileContents(filesystem fs.Filesystem, src, dst string) (err error) { - in, err := filesystem.Open(src) +func copyFileContents(srcFs, dstFs fs.Filesystem, src, dst string) (err error) { + in, err := srcFs.Open(src) if err != nil { return } defer in.Close() - out, err := filesystem.Create(dst) + out, err := dstFs.Create(dst) if err != nil { return } diff --git a/lib/osutil/osutil_test.go b/lib/osutil/osutil_test.go index c6ee2d487..f367a56ee 100644 --- a/lib/osutil/osutil_test.go +++ b/lib/osutil/osutil_test.go @@ -7,6 +7,7 @@ package osutil_test import ( + "io/ioutil" "os" "path/filepath" "runtime" @@ -192,7 +193,7 @@ func TestInWritableDirWindowsRename(t *testing.T) { } rename := func(path string) error { - return osutil.Rename(fs, path, path+"new") + return osutil.RenameOrCopy(fs, fs, path, path+"new") } for _, path := range []string{"testdata/windows/ro/readonly", "testdata/windows/ro", "testdata/windows"} { @@ -268,3 +269,79 @@ func TestIsDeleted(t *testing.T) { testFs.Chmod("inacc", 0777) os.RemoveAll("testdata") } + +func TestRenameOrCopy(t *testing.T) { + mustTempDir := func() string { + t.Helper() + tmpDir, err := ioutil.TempDir("", "") + if err != nil { + t.Fatal(err) + } + return tmpDir + } + sameFs := fs.NewFilesystem(fs.FilesystemTypeBasic, mustTempDir()) + tests := []struct { + src fs.Filesystem + dst fs.Filesystem + file string + }{ + { + src: sameFs, + dst: sameFs, + file: "file", + }, + { + src: fs.NewFilesystem(fs.FilesystemTypeBasic, mustTempDir()), + dst: fs.NewFilesystem(fs.FilesystemTypeBasic, mustTempDir()), + file: "file", + }, + { + src: fs.NewFilesystem(fs.FilesystemTypeFake, `fake://fake/?files=1&seed=42`), + dst: fs.NewFilesystem(fs.FilesystemTypeBasic, mustTempDir()), + file: osutil.NativeFilename(`05/7a/4d52f284145b9fe8`), + }, + } + + for _, test := range tests { + content := test.src.URI() + if _, err := test.src.Lstat(test.file); err != nil { + if !fs.IsNotExist(err) { + t.Fatal(err) + } + if fd, err := test.src.Create(test.file); err != nil { + t.Fatal(err) + } else { + if _, err := fd.Write([]byte(test.src.URI())); err != nil { + t.Fatal(err) + } + _ = fd.Close() + } + } else { + fd, err := test.src.Open(test.file) + if err != nil { + t.Fatal(err) + } + buf, err := ioutil.ReadAll(fd) + if err != nil { + t.Fatal(err) + } + _ = fd.Close() + content = string(buf) + } + + err := osutil.RenameOrCopy(test.src, test.dst, test.file, "new") + if err != nil { + t.Fatal(err) + } + + if fd, err := test.dst.Open("new"); err != nil { + t.Fatal(err) + } else { + if buf, err := ioutil.ReadAll(fd); err != nil { + t.Fatal(err) + } else if string(buf) != content { + t.Fatalf("expected %s got %s", content, string(buf)) + } + } + } +} diff --git a/lib/versioner/external.go b/lib/versioner/external.go index 5ab4edc3b..d2276fe05 100644 --- a/lib/versioner/external.go +++ b/lib/versioner/external.go @@ -12,6 +12,7 @@ import ( "os/exec" "runtime" "strings" + "time" "github.com/syncthing/syncthing/lib/fs" @@ -103,3 +104,11 @@ func (v External) Archive(filePath string) error { } return errors.New("Versioner: file was not removed by external script") } + +func (v External) GetVersions() (map[string][]FileVersion, error) { + return nil, ErrRestorationNotSupported +} + +func (v External) Restore(filePath string, versionTime time.Time) error { + return ErrRestorationNotSupported +} diff --git a/lib/versioner/simple.go b/lib/versioner/simple.go index 79dad81c5..43f55ce01 100644 --- a/lib/versioner/simple.go +++ b/lib/versioner/simple.go @@ -9,9 +9,9 @@ package versioner import ( "path/filepath" "strconv" + "time" "github.com/syncthing/syncthing/lib/fs" - "github.com/syncthing/syncthing/lib/osutil" "github.com/syncthing/syncthing/lib/util" ) @@ -21,19 +21,21 @@ func init() { } type Simple struct { - keep int - fs fs.Filesystem + keep int + folderFs fs.Filesystem + versionsFs fs.Filesystem } -func NewSimple(folderID string, fs fs.Filesystem, params map[string]string) Versioner { +func NewSimple(folderID string, folderFs fs.Filesystem, params map[string]string) Versioner { keep, err := strconv.Atoi(params["keep"]) if err != nil { keep = 5 // A reasonable default } s := Simple{ - keep: keep, - fs: fs, + keep: keep, + folderFs: folderFs, + versionsFs: fsFromParams(folderFs, params), } l.Debugf("instantiated %#v", s) @@ -43,51 +45,17 @@ func NewSimple(folderID string, fs fs.Filesystem, params map[string]string) Vers // Archive moves the named file away to a version archive. If this function // returns nil, the named file does not exist any more (has been archived). func (v Simple) Archive(filePath string) error { - info, err := v.fs.Lstat(filePath) - if fs.IsNotExist(err) { - l.Debugln("not archiving nonexistent file", filePath) - return nil - } else if err != nil { + err := archiveFile(v.folderFs, v.versionsFs, filePath, TagFilename) + if err != nil { return err } - if info.IsSymlink() { - panic("bug: attempting to version a symlink") - } - - versionsDir := ".stversions" - _, err = v.fs.Stat(versionsDir) - if err != nil { - if fs.IsNotExist(err) { - l.Debugln("creating versions dir .stversions") - v.fs.Mkdir(versionsDir, 0755) - v.fs.Hide(versionsDir) - } else { - return err - } - } - - l.Debugln("archiving", filePath) file := filepath.Base(filePath) - inFolderPath := filepath.Dir(filePath) - - dir := filepath.Join(versionsDir, inFolderPath) - err = v.fs.MkdirAll(dir, 0755) - if err != nil && !fs.IsExist(err) { - return err - } - - ver := TagFilename(file, info.ModTime().Format(TimeFormat)) - dst := filepath.Join(dir, ver) - l.Debugln("moving to", dst) - err = osutil.Rename(v.fs, filePath, dst) - if err != nil { - return err - } + dir := filepath.Dir(filePath) // Glob according to the new file~timestamp.ext pattern. pattern := filepath.Join(dir, TagFilename(file, TimeGlob)) - newVersions, err := v.fs.Glob(pattern) + newVersions, err := v.versionsFs.Glob(pattern) if err != nil { l.Warnln("globbing:", err, "for", pattern) return nil @@ -95,7 +63,7 @@ func (v Simple) Archive(filePath string) error { // Also according to the old file.ext~timestamp pattern. pattern = filepath.Join(dir, file+"~"+TimeGlob) - oldVersions, err := v.fs.Glob(pattern) + oldVersions, err := v.versionsFs.Glob(pattern) if err != nil { l.Warnln("globbing:", err, "for", pattern) return nil @@ -108,7 +76,7 @@ func (v Simple) Archive(filePath string) error { if len(versions) > v.keep { for _, toRemove := range versions[:len(versions)-v.keep] { l.Debugln("cleaning out", toRemove) - err = v.fs.Remove(toRemove) + err = v.versionsFs.Remove(toRemove) if err != nil { l.Warnln("removing old version:", err) } @@ -117,3 +85,11 @@ func (v Simple) Archive(filePath string) error { return nil } + +func (v Simple) GetVersions() (map[string][]FileVersion, error) { + return retrieveVersions(v.versionsFs) +} + +func (v Simple) Restore(filepath string, versionTime time.Time) error { + return restoreFile(v.versionsFs, v.folderFs, filepath, versionTime, TagFilename) +} diff --git a/lib/versioner/staggered.go b/lib/versioner/staggered.go index beea0aa67..624959723 100644 --- a/lib/versioner/staggered.go +++ b/lib/versioner/staggered.go @@ -7,7 +7,6 @@ package versioner import ( - "os" "path/filepath" "strconv" "time" @@ -48,16 +47,9 @@ func NewStaggered(folderID string, folderFs fs.Filesystem, params map[string]str cleanInterval = 3600 // Default: clean once per hour } - // Use custom path if set, otherwise .stversions in folderPath - var versionsFs fs.Filesystem - if params["versionsPath"] == "" { - versionsFs = fs.NewFilesystem(folderFs.Type(), filepath.Join(folderFs.URI(), ".stversions")) - } else if filepath.IsAbs(params["versionsPath"]) { - versionsFs = fs.NewFilesystem(folderFs.Type(), params["versionsPath"]) - } else { - versionsFs = fs.NewFilesystem(folderFs.Type(), filepath.Join(folderFs.URI(), params["versionsPath"])) - } - l.Debugln("%s folder using %s (%s) staggered versioner dir", folderID, versionsFs.URI(), versionsFs.Type()) + // Backwards compatibility + params["fsPath"] = params["versionsPath"] + versionsFs := fsFromParams(folderFs, params) s := &Staggered{ cleanInterval: cleanInterval, @@ -225,53 +217,12 @@ func (v *Staggered) Archive(filePath string) error { v.mutex.Lock() defer v.mutex.Unlock() - info, err := v.folderFs.Lstat(filePath) - if fs.IsNotExist(err) { - l.Debugln("not archiving nonexistent file", filePath) - return nil - } else if err != nil { + if err := archiveFile(v.folderFs, v.versionsFs, filePath, TagFilename); err != nil { return err } - if info.IsSymlink() { - panic("bug: attempting to version a symlink") - } - - if _, err := v.versionsFs.Stat("."); err != nil { - if fs.IsNotExist(err) { - l.Debugln("creating versions dir", v.versionsFs) - v.versionsFs.MkdirAll(".", 0755) - v.versionsFs.Hide(".") - } else { - return err - } - } - - l.Debugln("archiving", filePath) file := filepath.Base(filePath) inFolderPath := filepath.Dir(filePath) - if err != nil { - return err - } - - err = v.versionsFs.MkdirAll(inFolderPath, 0755) - if err != nil && !fs.IsExist(err) { - return err - } - - ver := TagFilename(file, time.Now().Format(TimeFormat)) - dst := filepath.Join(inFolderPath, ver) - l.Debugln("moving to", dst) - - /// TODO: Fix this when we have an alternative filesystem implementation - if v.versionsFs.Type() != fs.FilesystemTypeBasic { - panic("bug: staggered versioner used with unsupported filesystem") - } - - err = os.Rename(filepath.Join(v.folderFs.URI(), filePath), filepath.Join(v.versionsFs.URI(), dst)) - if err != nil { - return err - } // Glob according to the new file~timestamp.ext pattern. pattern := filepath.Join(inFolderPath, TagFilename(file, TimeGlob)) @@ -295,3 +246,11 @@ func (v *Staggered) Archive(filePath string) error { return nil } + +func (v *Staggered) GetVersions() (map[string][]FileVersion, error) { + return retrieveVersions(v.versionsFs) +} + +func (v *Staggered) Restore(filepath string, versionTime time.Time) error { + return restoreFile(v.versionsFs, v.folderFs, filepath, versionTime, TagFilename) +} diff --git a/lib/versioner/trashcan.go b/lib/versioner/trashcan.go index 70e65ac83..311636ac2 100644 --- a/lib/versioner/trashcan.go +++ b/lib/versioner/trashcan.go @@ -8,12 +8,10 @@ package versioner import ( "fmt" - "path/filepath" "strconv" "time" "github.com/syncthing/syncthing/lib/fs" - "github.com/syncthing/syncthing/lib/osutil" ) func init() { @@ -22,17 +20,19 @@ func init() { } type Trashcan struct { - fs fs.Filesystem + folderFs fs.Filesystem + versionsFs fs.Filesystem cleanoutDays int stop chan struct{} } -func NewTrashcan(folderID string, fs fs.Filesystem, params map[string]string) Versioner { +func NewTrashcan(folderID string, folderFs fs.Filesystem, params map[string]string) Versioner { cleanoutDays, _ := strconv.Atoi(params["cleanoutDays"]) // On error we default to 0, "do not clean out the trash can" s := &Trashcan{ - fs: fs, + folderFs: folderFs, + versionsFs: fsFromParams(folderFs, params), cleanoutDays: cleanoutDays, stop: make(chan struct{}), } @@ -44,49 +44,9 @@ func NewTrashcan(folderID string, fs fs.Filesystem, params map[string]string) Ve // Archive moves the named file away to a version archive. If this function // returns nil, the named file does not exist any more (has been archived). func (t *Trashcan) Archive(filePath string) error { - info, err := t.fs.Lstat(filePath) - if fs.IsNotExist(err) { - l.Debugln("not archiving nonexistent file", filePath) - return nil - } else if err != nil { - return err - } - if info.IsSymlink() { - panic("bug: attempting to version a symlink") - } - - versionsDir := ".stversions" - if _, err := t.fs.Stat(versionsDir); err != nil { - if !fs.IsNotExist(err) { - return err - } - - l.Debugln("creating versions dir", versionsDir) - if err := t.fs.MkdirAll(versionsDir, 0777); err != nil { - return err - } - t.fs.Hide(versionsDir) - } - - l.Debugln("archiving", filePath) - - archivedPath := filepath.Join(versionsDir, filePath) - if err := t.fs.MkdirAll(filepath.Dir(archivedPath), 0777); err != nil && !fs.IsExist(err) { - return err - } - - l.Debugln("moving to", archivedPath) - - if err := osutil.Rename(t.fs, filePath, archivedPath); err != nil { - return err - } - - // Set the mtime to the time the file was deleted. This is used by the - // cleanout routine. If this fails things won't work optimally but there's - // not much we can do about it so we ignore the error. - t.fs.Chtimes(archivedPath, time.Now(), time.Now()) - - return nil + return archiveFile(t.folderFs, t.versionsFs, filePath, func(name, tag string) string { + return name + }) } func (t *Trashcan) Serve() { @@ -124,8 +84,7 @@ func (t *Trashcan) String() string { } func (t *Trashcan) cleanoutArchive() error { - versionsDir := ".stversions" - if _, err := t.fs.Lstat(versionsDir); fs.IsNotExist(err) { + if _, err := t.versionsFs.Lstat("."); fs.IsNotExist(err) { return nil } @@ -144,20 +103,45 @@ func (t *Trashcan) cleanoutArchive() error { if info.ModTime().Before(cutoff) { // The file is too old; remove it. - t.fs.Remove(path) + err = t.versionsFs.Remove(path) } else { // Keep this file, and remember it so we don't unnecessarily try // to remove this directory. dirTracker.addFile(path) } - return nil - } - - if err := t.fs.Walk(versionsDir, walkFn); err != nil { return err } - dirTracker.deleteEmptyDirs(t.fs) + if err := t.versionsFs.Walk(".", walkFn); err != nil { + return err + } + + dirTracker.deleteEmptyDirs(t.versionsFs) return nil } + +func (t *Trashcan) GetVersions() (map[string][]FileVersion, error) { + return retrieveVersions(t.versionsFs) +} + +func (t *Trashcan) Restore(filepath string, versionTime time.Time) error { + // If we have an untagged file A and want to restore it on top of existing file A, we can't first archive the + // existing A as we'd overwrite the old A version, therefore when we archive existing file, we archive it with a + // tag but when the restoration is finished, we rename it (untag it). This is only important if when restoring A, + // there already exists a file at the same location + + taggedName := "" + tagger := func(name, tag string) string { + // We can't use TagFilename here, as restoreFii would discover that as a valid version and restore that instead. + taggedName = fs.TempName(name) + return taggedName + } + + err := restoreFile(t.versionsFs, t.folderFs, filepath, versionTime, tagger) + if taggedName == "" { + return err + } + + return t.versionsFs.Rename(taggedName, filepath) +} diff --git a/lib/versioner/trashcan_test.go b/lib/versioner/trashcan_test.go index 023e5806b..1fc26e28d 100644 --- a/lib/versioner/trashcan_test.go +++ b/lib/versioner/trashcan_test.go @@ -75,3 +75,87 @@ func TestTrashcanCleanout(t *testing.T) { t.Error("empty directory should have been removed") } } + +func TestTrashcanArchiveRestoreSwitcharoo(t *testing.T) { + // This tests that trashcan versioner restoration correctly archives existing file, because trashcan versioner + // files are untagged, archiving existing file to replace with a restored version technically should collide in + // in names. + tmpDir1, err := ioutil.TempDir("", "") + if err != nil { + t.Fatal(err) + } + + tmpDir2, err := ioutil.TempDir("", "") + if err != nil { + t.Fatal(err) + } + + folderFs := fs.NewFilesystem(fs.FilesystemTypeBasic, tmpDir1) + versionsFs := fs.NewFilesystem(fs.FilesystemTypeBasic, tmpDir2) + + writeFile(t, folderFs, "file", "A") + + versioner := NewTrashcan("", folderFs, map[string]string{ + "fsType": "basic", + "fsPath": tmpDir2, + }) + + if err := versioner.Archive("file"); err != nil { + t.Fatal(err) + } + + if _, err := folderFs.Stat("file"); !fs.IsNotExist(err) { + t.Fatal(err) + } + + versionInfo, err := versionsFs.Stat("file") + if err != nil { + t.Fatal(err) + } + + if content := readFile(t, versionsFs, "file"); content != "A" { + t.Errorf("expected A got %s", content) + } + + writeFile(t, folderFs, "file", "B") + + if err := versioner.Restore("file", versionInfo.ModTime().Truncate(time.Second)); err != nil { + t.Fatal(err) + } + + if content := readFile(t, folderFs, "file"); content != "A" { + t.Errorf("expected A got %s", content) + } + + if content := readFile(t, versionsFs, "file"); content != "B" { + t.Errorf("expected B got %s", content) + } +} + +func readFile(t *testing.T, filesystem fs.Filesystem, name string) string { + fd, err := filesystem.Open(name) + if err != nil { + t.Fatal(err) + } + defer fd.Close() + buf, err := ioutil.ReadAll(fd) + if err != nil { + t.Fatal(err) + } + return string(buf) +} + +func writeFile(t *testing.T, filesystem fs.Filesystem, name, content string) { + fd, err := filesystem.OpenFile(name, fs.OptReadWrite|fs.OptCreate, 0777) + if err != nil { + t.Fatal(err) + } + defer fd.Close() + if err := fd.Truncate(int64(len(content))); err != nil { + t.Fatal(err) + } + + if n, err := fd.Write([]byte(content)); err != nil || n != len(content) { + t.Fatal(n, len(content), err) + } +} diff --git a/lib/versioner/util.go b/lib/versioner/util.go index 79d9074b2..addfe7d86 100644 --- a/lib/versioner/util.go +++ b/lib/versioner/util.go @@ -7,11 +7,30 @@ package versioner import ( + "fmt" "path/filepath" "regexp" "strings" + "time" + + "github.com/pkg/errors" + "github.com/syncthing/syncthing/lib/fs" + "github.com/syncthing/syncthing/lib/osutil" ) +var locationLocal *time.Location +var errDirectory = fmt.Errorf("cannot restore on top of a directory") +var errNotFound = fmt.Errorf("version not found") +var errFileAlreadyExists = fmt.Errorf("file already exists") + +func init() { + var err error + locationLocal, err = time.LoadLocation("Local") + if err != nil { + panic(err.Error()) + } +} + // Inserts ~tag just before the extension of the filename. func TagFilename(name, tag string) string { dir, file := filepath.Dir(name), filepath.Base(name) @@ -38,11 +57,215 @@ func UntagFilename(path string) (string, string) { versionTag := ExtractTag(path) // Files tagged with old style tags cannot be untagged. - if versionTag == "" || strings.HasSuffix(ext, versionTag) { + if versionTag == "" { return "", "" } + // Old style tag + if strings.HasSuffix(ext, versionTag) { + return strings.TrimSuffix(path, "~"+versionTag), versionTag + } + withoutExt := path[:len(path)-len(ext)-len(versionTag)-1] name := withoutExt + ext return name, versionTag } + +func retrieveVersions(fileSystem fs.Filesystem) (map[string][]FileVersion, error) { + files := make(map[string][]FileVersion) + + err := fileSystem.Walk(".", func(path string, f fs.FileInfo, err error) error { + // Skip root (which is ok to be a symlink) + if path == "." { + return nil + } + + // Skip walking if we cannot walk... + if err != nil { + return err + } + + // Ignore symlinks + if f.IsSymlink() { + return fs.SkipDir + } + + // No records for directories + if f.IsDir() { + return nil + } + + path = osutil.NormalizedFilename(path) + + name, tag := UntagFilename(path) + // Something invalid, assume it's an untagged file + if name == "" || tag == "" { + versionTime := f.ModTime().Truncate(time.Second) + files[path] = append(files[path], FileVersion{ + VersionTime: versionTime, + ModTime: versionTime, + Size: f.Size(), + }) + return nil + } + + versionTime, err := time.ParseInLocation(TimeFormat, tag, locationLocal) + if err != nil { + // Can't parse it, welp, continue + return nil + } + + if err == nil { + files[name] = append(files[name], FileVersion{ + VersionTime: versionTime.Truncate(time.Second), + ModTime: f.ModTime().Truncate(time.Second), + Size: f.Size(), + }) + } + + return nil + }) + + if err != nil { + return nil, err + } + + return files, nil +} + +type fileTagger func(string, string) string + +func archiveFile(srcFs, dstFs fs.Filesystem, filePath string, tagger fileTagger) error { + filePath = osutil.NativeFilename(filePath) + info, err := srcFs.Lstat(filePath) + if fs.IsNotExist(err) { + l.Debugln("not archiving nonexistent file", filePath) + return nil + } else if err != nil { + return err + } + if info.IsSymlink() { + panic("bug: attempting to version a symlink") + } + + _, err = dstFs.Stat(".") + if err != nil { + if fs.IsNotExist(err) { + l.Debugln("creating versions dir") + err := dstFs.Mkdir(".", 0755) + if err != nil { + return err + } + _ = dstFs.Hide(".") + } else { + return err + } + } + + l.Debugln("archiving", filePath) + + file := filepath.Base(filePath) + inFolderPath := filepath.Dir(filePath) + + err = dstFs.MkdirAll(inFolderPath, 0755) + if err != nil && !fs.IsExist(err) { + return err + } + + ver := tagger(file, info.ModTime().Format(TimeFormat)) + dst := filepath.Join(inFolderPath, ver) + l.Debugln("moving to", dst) + err = osutil.RenameOrCopy(srcFs, dstFs, filePath, dst) + + // Set the mtime to the time the file was deleted. This can be used by the + // cleanout routine. If this fails things won't work optimally but there's + // not much we can do about it so we ignore the error. + _ = dstFs.Chtimes(dst, time.Now(), time.Now()) + + return err +} + +func restoreFile(src, dst fs.Filesystem, filePath string, versionTime time.Time, tagger fileTagger) error { + // If the something already exists where we are restoring to, archive existing file for versioning + // remove if it's a symlink, or fail if it's a directory + if info, err := dst.Lstat(filePath); err == nil { + switch { + case info.IsDir(): + return errDirectory + case info.IsSymlink(): + // Remove existing symlinks (as we don't want to archive them) + if err := dst.Remove(filePath); err != nil { + return errors.Wrap(err, "removing existing symlink") + } + case info.IsRegular(): + if err := archiveFile(dst, src, filePath, tagger); err != nil { + return errors.Wrap(err, "archiving existing file") + } + default: + panic("bug: unknown item type") + } + } else if !fs.IsNotExist(err) { + return err + } + + filePath = osutil.NativeFilename(filePath) + tag := versionTime.In(locationLocal).Truncate(time.Second).Format(TimeFormat) + + taggedFilename := TagFilename(filePath, tag) + oldTaggedFilename := filePath + tag + untaggedFileName := filePath + + // Check that the thing we've been asked to restore is actually a file + // and that it exists. + sourceFile := "" + for _, candidate := range []string{taggedFilename, oldTaggedFilename, untaggedFileName} { + if info, err := src.Lstat(candidate); fs.IsNotExist(err) || !info.IsRegular() { + continue + } else if err != nil { + // All other errors are fatal + return err + } else if candidate == untaggedFileName && !info.ModTime().Truncate(time.Second).Equal(versionTime) { + // No error, and untagged file, but mtime does not match, skip + continue + } + + sourceFile = candidate + break + } + + if sourceFile == "" { + return errNotFound + } + + // Check that the target location of where we are supposed to restore does not exist. + // This should have been taken care of by the first few lines of this function. + if _, err := dst.Lstat(filePath); err == nil { + return errFileAlreadyExists + } else if !fs.IsNotExist(err) { + return err + } + + _ = dst.MkdirAll(filepath.Dir(filePath), 0755) + return osutil.RenameOrCopy(src, dst, sourceFile, filePath) +} + +func fsFromParams(folderFs fs.Filesystem, params map[string]string) (versionsFs fs.Filesystem) { + if params["fsType"] == "" && params["fsPath"] == "" { + versionsFs = fs.NewFilesystem(folderFs.Type(), filepath.Join(folderFs.URI(), ".stversions")) + + } else if params["fsType"] == "" { + uri := params["fsPath"] + // We only know how to deal with relative folders for basic filesystems, as that's the only one we know + // how to check if it's absolute or relative. + if folderFs.Type() == fs.FilesystemTypeBasic && !filepath.IsAbs(params["fsPath"]) { + uri = filepath.Join(folderFs.URI(), params["fsPath"]) + } + versionsFs = fs.NewFilesystem(folderFs.Type(), uri) + } else { + var fsType fs.FilesystemType + _ = fsType.UnmarshalText([]byte(params["fsType"])) + versionsFs = fs.NewFilesystem(fsType, params["fsPath"]) + } + l.Debugln("%s (%s) folder using %s (%s) versioner dir", folderFs.URI(), folderFs.Type(), versionsFs.URI(), versionsFs.Type()) + return +} diff --git a/lib/versioner/versioner.go b/lib/versioner/versioner.go index 0139b68d2..8f9ee89b7 100644 --- a/lib/versioner/versioner.go +++ b/lib/versioner/versioner.go @@ -9,6 +9,7 @@ package versioner import ( + "fmt" "time" "github.com/syncthing/syncthing/lib/fs" @@ -16,6 +17,8 @@ import ( type Versioner interface { Archive(filePath string) error + GetVersions() (map[string][]FileVersion, error) + Restore(filePath string, versionTime time.Time) error } type FileVersion struct { @@ -25,6 +28,7 @@ type FileVersion struct { } var Factories = map[string]func(folderID string, filesystem fs.Filesystem, params map[string]string) Versioner{} +var ErrRestorationNotSupported = fmt.Errorf("version restoration not supported with the current versioner") const ( TimeFormat = "20060102-150405"