Merge pull request #1935 from calmh/trashcan

Add trash can file versioning (fixes #1931)

cmd/syncthing/main.go

@@ -589,6 +589,7 @@ func syncthingMain() {
 
 	m := model.NewModel(cfg, myID, myName, "syncthing", Version, ldb)
 	cfg.Subscribe(m)
+	mainSvc.Add(m)
 
 	if t := os.Getenv("STDEADLOCKTIMEOUT"); len(t) > 0 {
 		it, err := strconv.Atoi(t)

gui/index.html

@@ -263,6 +263,7 @@
                     <tr ng-if="folder.versioning.type">
                       <th><span class="glyphicon glyphicon-tags"></span>&nbsp;<span translate>File Versioning</span></th>
                       <td class="text-right" ng-switch="folder.versioning.type">
+                        <span ng-switch-when="trashcan" translate>Trash Can File Versioning</span>
                         <span ng-switch-when="staggered" translate>Staggered File Versioning</span>
                         <span ng-switch-when="simple" translate>Simple File Versioning</span>
                         <span ng-switch-when="external" translate>External File Versioning</span>
@@ -681,27 +682,27 @@
                   </select>
                 </div>
                 <div class="form-group">
-                  <label translate>File Versioning</label>
+                  <label translate>File Versioning</label>&emsp;<a href="http://docs.syncthing.net/users/versioning.html" target="_blank"><span class="glyphicon glyphicon-book"></span>&nbsp;<span translate>Help</span></a>
-                  <div class="radio">
+                  <select class="form-control" ng-model="currentFolder.fileVersioningSelector">
-                    <label>
+                    <option value="none" translate>No File Versioning</option>
-                      <input type="radio" ng-model="currentFolder.fileVersioningSelector" value="none"> <span translate>No File Versioning</span>
+                    <option value="trashcan" translate>Trash Can File Versioning</option>
-                    </label>
+                    <option value="simple" translate>Simple File Versioning</option>
+                    <option value="staggered" translate>Staggered File Versioning</option>
+                    <option value="external" translate>External File Versioning</option>
+                  </select>
                 </div>
-                  <div class="radio">
+                <div class="form-group" ng-if="currentFolder.fileVersioningSelector=='trashcan'" ng-class="{'has-error': folderEditor.trashcanClean.$invalid && folderEditor.trashcanClean.$dirty}">
-                    <label>
+                  <p translate class="help-block">Files are moved to .stversions folder when replaced or deleted by Syncthing.</p>
-                      <input type="radio" ng-model="currentFolder.fileVersioningSelector" value="simple"> <span translate>Simple File Versioning</span>
+                  <label translate for="trashcanClean">Clean out after</label>
-                    </label>
+                  <div class="input-group">
-                  </div>
+                    <input name="trashcanClean" id="trashcanClean" class="form-control text-right" type="number" ng-model="currentFolder.trashcanClean" required min="0"></input>
-                  <div class="radio">
+                    <div class="input-group-addon">days</div>
-                    <label>
-                      <input type="radio" ng-model="currentFolder.fileVersioningSelector" value="staggered"> <span translate>Staggered File Versioning</span>
-                    </label>
-                  </div>
-                  <div class="radio">
-                    <label>
-                      <input type="radio" ng-model="currentFolder.fileVersioningSelector" value="external"> <span translate>External File Versioning</span>
-                    </label>
                   </div>
+                  <p class="help-block">
+                    <span translate ng-if="folderEditor.trashcanClean.$valid || folderEditor.trashcanClean.$pristine">The number of days to keep files in the trash can. Zero means forever.</span>
+                    <span translate ng-if="folderEditor.trashcanClean.$error.required && folderEditor.trashcanClean.$dirty">The number of days must be a number and cannot be blank.</span>
+                    <span translate ng-if="folderEditor.trashcanClean.$error.min && folderEditor.trashcanClean.$dirty">A negative number of days doesn't make sense.</span>
+                  </p>
                 </div>
                 <div class="form-group" ng-if="currentFolder.fileVersioningSelector=='simple'" ng-class="{'has-error': folderEditor.simpleKeep.$invalid && folderEditor.simpleKeep.$dirty}">
                   <p translate class="help-block">Files are moved to date stamped versions in a .stversions folder when replaced or deleted by Syncthing.</p>

gui/scripts/syncthing/core/controllers/syncthingController.js

@@ -990,7 +990,11 @@ angular.module('syncthing.core')
             $scope.currentFolder.devices.forEach(function (n) {
                 $scope.currentFolder.selectedDevices[n.deviceID] = true;
             });
-            if ($scope.currentFolder.versioning && $scope.currentFolder.versioning.type === "simple") {
+            if ($scope.currentFolder.versioning && $scope.currentFolder.versioning.type === "trashcan") {
+                $scope.currentFolder.trashcanFileVersioning = true;
+                $scope.currentFolder.fileVersioningSelector = "trashcan";
+                $scope.currentFolder.trashcanClean = +$scope.currentFolder.versioning.params.cleanoutDays;
+            } else if ($scope.currentFolder.versioning && $scope.currentFolder.versioning.type === "simple") {
                 $scope.currentFolder.simpleFileVersioning = true;
                 $scope.currentFolder.fileVersioningSelector = "simple";
                 $scope.currentFolder.simpleKeep = +$scope.currentFolder.versioning.params.keep;
@@ -1007,6 +1011,7 @@ angular.module('syncthing.core')
             } else {
                 $scope.currentFolder.fileVersioningSelector = "none";
             }
+            $scope.currentFolder.trashcanClean = $scope.currentFolder.trashcanClean || 0; // weeds out nulls and undefineds
             $scope.currentFolder.simpleKeep = $scope.currentFolder.simpleKeep || 5;
             $scope.currentFolder.staggeredCleanInterval = $scope.currentFolder.staggeredCleanInterval || 3600;
             $scope.currentFolder.staggeredVersionsPath = $scope.currentFolder.staggeredVersionsPath || "";
@@ -1030,6 +1035,7 @@ angular.module('syncthing.core')
             };
             $scope.currentFolder.rescanIntervalS = 60;
             $scope.currentFolder.fileVersioningSelector = "none";
+            $scope.currentFolder.trashcanClean = 0;
             $scope.currentFolder.simpleKeep = 5;
             $scope.currentFolder.staggeredMaxAge = 365;
             $scope.currentFolder.staggeredCleanInterval = 3600;
@@ -1051,6 +1057,7 @@ angular.module('syncthing.core')
 
             $scope.currentFolder.rescanIntervalS = 60;
             $scope.currentFolder.fileVersioningSelector = "none";
+            $scope.currentFolder.trashcanClean = 0;
             $scope.currentFolder.simpleKeep = 5;
             $scope.currentFolder.staggeredMaxAge = 365;
             $scope.currentFolder.staggeredCleanInterval = 3600;
@@ -1087,7 +1094,16 @@ angular.module('syncthing.core')
             }
             delete folderCfg.selectedDevices;
 
-            if (folderCfg.fileVersioningSelector === "simple") {
+            if (folderCfg.fileVersioningSelector === "trashcan") {
+                folderCfg.versioning = {
+                    'Type': 'trashcan',
+                    'Params': {
+                        'cleanoutDays': '' + folderCfg.trashcanClean
+                    }
+                };
+                delete folderCfg.trashcanFileVersioning;
+                delete folderCfg.trashcanClean;
+            } else if (folderCfg.fileVersioningSelector === "simple") {
                 folderCfg.versioning = {
                     'Type': 'simple',
                     'Params': {

internal/model/model.go

@@ -35,6 +35,7 @@ import (
 	"github.com/syncthing/syncthing/internal/sync"
 	"github.com/syncthing/syncthing/internal/versioner"
 	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/thejerf/suture"
 )
 
 // How many files to send in each Index/IndexUpdate message.
@@ -61,6 +62,8 @@ type service interface {
 }
 
 type Model struct {
+	*suture.Supervisor
+
 	cfg             *config.Wrapper
 	db              *leveldb.DB
 	finder          *db.BlockFinder
@@ -103,6 +106,13 @@ var (
 // for file data without altering the local folder in any way.
 func NewModel(cfg *config.Wrapper, id protocol.DeviceID, deviceName, clientName, clientVersion string, ldb *leveldb.DB) *Model {
 	m := &Model{
+		Supervisor: suture.New("model", suture.Spec{
+			Log: func(line string) {
+				if debug {
+					l.Debugln(line)
+				}
+			},
+		}),
 		cfg:                cfg,
 		db:                 ldb,
 		finder:             db.NewBlockFinder(ldb, cfg),
@@ -168,7 +178,14 @@ func (m *Model) StartFolderRW(folder string) {
 		if !ok {
 			l.Fatalf("Requested versioning type %q that does not exist", cfg.Versioning.Type)
 		}
-		p.versioner = factory(folder, cfg.Path(), cfg.Versioning.Params)
+		versioner := factory(folder, cfg.Path(), cfg.Versioning.Params)
+		if service, ok := versioner.(suture.Service); ok {
+			// The versioner implements the suture.Service interface, so
+			// expects to be run in the background in addition to being called
+			// when files are going to be archived.
+			m.Add(service)
+		}
+		p.versioner = versioner
 	}
 
 	go p.Serve()

internal/versioner/.gitignore

@@ -0,0 +1 @@
+testdata

internal/versioner/trashcan.go

@@ -0,0 +1,187 @@
+// Copyright (C) 2015 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package versioner
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strconv"
+	"time"
+
+	"github.com/syncthing/syncthing/internal/osutil"
+)
+
+func init() {
+	// Register the constructor for this type of versioner
+	Factories["trashcan"] = NewTrashcan
+}
+
+type Trashcan struct {
+	folderPath   string
+	cleanoutDays int
+	stop         chan struct{}
+}
+
+func NewTrashcan(folderID, folderPath string, params map[string]string) Versioner {
+	cleanoutDays, _ := strconv.Atoi(params["cleanoutDays"])
+	// On error we default to 0, "do not clean out the trash can"
+
+	s := &Trashcan{
+		folderPath:   folderPath,
+		cleanoutDays: cleanoutDays,
+		stop:         make(chan struct{}),
+	}
+
+	if debug {
+		l.Debugf("instantiated %#v", s)
+	}
+	return s
+}
+
+// Archive moves the named file away to a version archive. If this function
+// returns nil, the named file does not exist any more (has been archived).
+func (t *Trashcan) Archive(filePath string) error {
+	_, err := osutil.Lstat(filePath)
+	if os.IsNotExist(err) {
+		if debug {
+			l.Debugln("not archiving nonexistent file", filePath)
+		}
+		return nil
+	} else if err != nil {
+		return err
+	}
+
+	versionsDir := filepath.Join(t.folderPath, ".stversions")
+	if _, err := os.Stat(versionsDir); err != nil {
+		if !os.IsNotExist(err) {
+			return err
+		}
+
+		if debug {
+			l.Debugln("creating versions dir", versionsDir)
+		}
+		if err := osutil.MkdirAll(versionsDir, 0777); err != nil {
+			return err
+		}
+		osutil.HideFile(versionsDir)
+	}
+
+	if debug {
+		l.Debugln("archiving", filePath)
+	}
+
+	relativePath, err := filepath.Rel(t.folderPath, filePath)
+	if err != nil {
+		return err
+	}
+
+	archivedPath := filepath.Join(versionsDir, relativePath)
+	if err := osutil.MkdirAll(filepath.Dir(archivedPath), 0777); err != nil && !os.IsExist(err) {
+		return err
+	}
+
+	if debug {
+		l.Debugln("moving to", archivedPath)
+	}
+
+	if err := osutil.Rename(filePath, archivedPath); err != nil {
+		return err
+	}
+
+	// Set the mtime to the time the file was deleted. This is used by the
+	// cleanout routine. If this fails things won't work optimally but there's
+	// not much we can do about it so we ignore the error.
+	os.Chtimes(archivedPath, time.Now(), time.Now())
+
+	return nil
+}
+
+func (t *Trashcan) Serve() {
+	if debug {
+		l.Debugln(t, "starting")
+		defer l.Debugln(t, "stopping")
+	}
+
+	// Do the first cleanup one minute after startup.
+	timer := time.NewTimer(time.Minute)
+	defer timer.Stop()
+
+	for {
+		select {
+		case <-t.stop:
+			return
+
+		case <-timer.C:
+			if t.cleanoutDays > 0 {
+				if err := t.cleanoutArchive(); err != nil {
+					l.Infoln("Cleaning trashcan:", err)
+				}
+			}
+
+			// Cleanups once a day should be enough.
+			timer.Reset(24 * time.Hour)
+		}
+	}
+}
+
+func (t *Trashcan) Stop() {
+	close(t.stop)
+}
+
+func (t *Trashcan) String() string {
+	return fmt.Sprintf("trashcan@%p", t)
+}
+
+func (t *Trashcan) cleanoutArchive() error {
+	versionsDir := filepath.Join(t.folderPath, ".stversions")
+	if _, err := osutil.Lstat(versionsDir); os.IsNotExist(err) {
+		return nil
+	}
+
+	cutoff := time.Now().Add(time.Duration(-24*t.cleanoutDays) * time.Hour)
+	currentDir := ""
+	filesInDir := 0
+	walkFn := func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if info.IsDir() {
+			// We have entered a new directory. Lets check if the previous
+			// directory was empty and try to remove it. We ignore failure for
+			// the time being.
+			if currentDir != "" && filesInDir == 0 {
+				osutil.Remove(currentDir)
+			}
+			currentDir = path
+			filesInDir = 0
+			return nil
+		}
+
+		if info.ModTime().Before(cutoff) {
+			// The file is too old; remove it.
+			osutil.Remove(path)
+		} else {
+			// Keep this file, and remember it so we don't unnecessarily try
+			// to remove this directory.
+			filesInDir++
+		}
+		return nil
+	}
+
+	if err := filepath.Walk(versionsDir, walkFn); err != nil {
+		return err
+	}
+
+	// The last directory seen by the walkFn may not have been removed as it
+	// should be.
+	if currentDir != "" && filesInDir == 0 {
+		osutil.Remove(currentDir)
+	}
+	return nil
+}

internal/versioner/trashcan_test.go

@@ -0,0 +1,69 @@
+// Copyright (C) 2015 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package versioner
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+)
+
+func TestTrashcanCleanout(t *testing.T) {
+	// Verify that files older than the cutoff are removed, that files newer
+	// than the cutoff are *not* removed, and that empty directories are
+	// removed (best effort).
+
+	var testcases = []struct {
+		file         string
+		shouldRemove bool
+	}{
+		{"testdata/.stversions/file1", false},
+		{"testdata/.stversions/file2", true},
+		{"testdata/.stversions/keep1/file1", false},
+		{"testdata/.stversions/keep1/file2", false},
+		{"testdata/.stversions/keep2/file1", false},
+		{"testdata/.stversions/keep2/file2", true},
+		{"testdata/.stversions/remove/file1", true},
+		{"testdata/.stversions/remove/file2", true},
+	}
+
+	os.RemoveAll("testdata")
+	defer os.RemoveAll("testdata")
+
+	oldTime := time.Now().Add(-8 * 24 * time.Hour)
+	for _, tc := range testcases {
+		os.MkdirAll(filepath.Dir(tc.file), 0777)
+		if err := ioutil.WriteFile(tc.file, []byte("data"), 0644); err != nil {
+			t.Fatal(err)
+		}
+		if tc.shouldRemove {
+			if err := os.Chtimes(tc.file, oldTime, oldTime); err != nil {
+				t.Fatal(err)
+			}
+		}
+	}
+
+	versioner := NewTrashcan("default", "testdata", map[string]string{"cleanoutDays": "7"}).(*Trashcan)
+	if err := versioner.cleanoutArchive(); err != nil {
+		t.Fatal(err)
+	}
+
+	for _, tc := range testcases {
+		_, err := os.Lstat(tc.file)
+		if tc.shouldRemove && !os.IsNotExist(err) {
+			t.Error(tc.file, "should have been removed")
+		} else if !tc.shouldRemove && err != nil {
+			t.Error(tc.file, "should not have been removed")
+		}
+	}
+
+	if _, err := os.Lstat("testdata/.stversions/remove"); !os.IsNotExist(err) {
+		t.Error("empty directory should have been removed")
+	}
+}