syncthing/internal/versioner/staggered.go

366 lines
8.5 KiB
Go
Raw Normal View History

2014-11-16 20:13:20 +00:00
// Copyright (C) 2014 The Syncthing Authors.
2014-09-29 19:43:32 +00:00
//
// This program is free software: you can redistribute it and/or modify it
// under the terms of the GNU General Public License as published by the Free
// Software Foundation, either version 3 of the License, or (at your option)
// any later version.
//
// This program is distributed in the hope that it will be useful, but WITHOUT
// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
// more details.
//
// You should have received a copy of the GNU General Public License along
// with this program. If not, see <http://www.gnu.org/licenses/>.
2014-08-21 22:41:17 +00:00
package versioner
import (
"os"
"path/filepath"
"sort"
"strconv"
"strings"
"sync"
"time"
2014-08-22 16:16:05 +00:00
2014-09-22 19:42:11 +00:00
"github.com/syncthing/syncthing/internal/osutil"
2014-08-21 22:41:17 +00:00
)
func init() {
// Register the constructor for this type of versioner with the name "staggered"
Factories["staggered"] = NewStaggered
}
type Interval struct {
step int64
end int64
}
// The type holds our configuration
type Staggered struct {
versionsPath string
cleanInterval int64
2014-09-28 11:05:25 +00:00
folderPath string
2014-08-21 22:41:17 +00:00
interval [4]Interval
mutex *sync.Mutex
}
// Check if file or dir
func isFile(path string) bool {
2014-08-23 08:37:58 +00:00
fileInfo, err := os.Stat(path)
2014-08-21 22:41:17 +00:00
if err != nil {
l.Infoln("versioner isFile:", err)
return false
}
return fileInfo.Mode().IsRegular()
}
const TimeLayout = "20060102-150405"
func versionExt(path string) string {
pathSplit := strings.Split(path, "~")
if len(pathSplit) > 1 {
return pathSplit[len(pathSplit)-1]
} else {
return ""
}
}
// Rename versions with old version format
func (v Staggered) renameOld() {
err := filepath.Walk(v.versionsPath, func(path string, f os.FileInfo, err error) error {
2014-09-11 18:14:27 +00:00
if err != nil {
return err
}
if f.Mode().IsRegular() {
versionUnix, err := strconv.ParseInt(strings.Replace(filepath.Ext(path), ".v", "", 1), 10, 0)
if err == nil {
l.Infoln("Renaming file", path, "from old to new version format")
versiondate := time.Unix(versionUnix, 0)
name := path[:len(path)-len(filepath.Ext(path))]
err = osutil.Rename(path, name+"~"+versiondate.Format(TimeLayout))
if err != nil {
l.Infoln("Error renaming to new format", err)
}
}
}
return nil
})
if err != nil {
l.Infoln("Versioner: error scanning versions dir", err)
return
}
}
2014-08-21 22:41:17 +00:00
// The constructor function takes a map of parameters and creates the type.
func NewStaggered(folderID, folderPath string, params map[string]string) Versioner {
2014-08-21 22:41:17 +00:00
maxAge, err := strconv.ParseInt(params["maxAge"], 10, 0)
if err != nil {
maxAge = 31536000 // Default: ~1 year
}
cleanInterval, err := strconv.ParseInt(params["cleanInterval"], 10, 0)
if err != nil {
cleanInterval = 3600 // Default: clean once per hour
}
// Use custom path if set, otherwise .stversions in folderPath
2014-08-21 22:41:17 +00:00
var versionsDir string
if params["versionsPath"] == "" {
if debug {
l.Debugln("using default dir .stversions")
}
versionsDir = filepath.Join(folderPath, ".stversions")
2014-08-21 22:41:17 +00:00
} else {
if debug {
l.Debugln("using dir", params["versionsPath"])
}
versionsDir = params["versionsPath"]
}
var mutex sync.Mutex
s := Staggered{
versionsPath: versionsDir,
cleanInterval: cleanInterval,
2014-09-28 11:05:25 +00:00
folderPath: folderPath,
2014-08-21 22:41:17 +00:00
interval: [4]Interval{
2014-11-16 20:13:20 +00:00
{30, 3600}, // first hour -> 30 sec between versions
{3600, 86400}, // next day -> 1 h between versions
{86400, 592000}, // next 30 days -> 1 day between versions
{604800, maxAge}, // next year -> 1 week between versions
2014-08-21 22:41:17 +00:00
},
mutex: &mutex,
}
if debug {
l.Debugf("instantiated %#v", s)
}
// Rename version with old version format
s.renameOld()
2014-08-21 22:41:17 +00:00
go func() {
s.clean()
for _ = range time.Tick(time.Duration(cleanInterval) * time.Second) {
s.clean()
}
}()
return s
}
func (v Staggered) clean() {
if debug {
l.Debugln("Versioner clean: Waiting for lock on", v.versionsPath)
}
v.mutex.Lock()
defer v.mutex.Unlock()
if debug {
l.Debugln("Versioner clean: Cleaning", v.versionsPath)
}
_, err := os.Stat(v.versionsPath)
if err != nil {
if os.IsNotExist(err) {
if debug {
l.Debugln("creating versions dir", v.versionsPath)
}
os.MkdirAll(v.versionsPath, 0755)
osutil.HideFile(v.versionsPath)
} else {
2014-08-22 16:16:05 +00:00
l.Warnln("Versioner: can't create versions dir", err)
2014-08-21 22:41:17 +00:00
}
}
2014-08-23 08:43:48 +00:00
versionsPerFile := make(map[string][]string)
filesPerDir := make(map[string]int)
2014-08-21 22:41:17 +00:00
err = filepath.Walk(v.versionsPath, func(path string, f os.FileInfo, err error) error {
if err != nil {
return err
}
2014-08-21 22:41:17 +00:00
switch mode := f.Mode(); {
case mode.IsDir():
2014-08-23 08:43:48 +00:00
filesPerDir[path] = 0
if path != v.versionsPath {
dir := filepath.Dir(path)
filesPerDir[dir]++
}
2014-08-21 22:41:17 +00:00
case mode.IsRegular():
extension := versionExt(path)
2014-08-23 08:43:48 +00:00
dir := filepath.Dir(path)
name := path[:len(path)-len(extension)-1]
2014-08-23 08:43:48 +00:00
filesPerDir[dir]++
versionsPerFile[name] = append(versionsPerFile[name], path)
2014-08-21 22:41:17 +00:00
}
return nil
})
if err != nil {
2014-08-22 16:16:05 +00:00
l.Warnln("Versioner: error scanning versions dir", err)
2014-08-23 08:43:48 +00:00
return
2014-08-21 22:41:17 +00:00
}
2014-08-23 08:43:48 +00:00
for _, versionList := range versionsPerFile {
// List from filepath.Walk is sorted
v.expire(versionList)
2014-08-21 22:41:17 +00:00
}
2014-08-23 08:43:48 +00:00
for path, numFiles := range filesPerDir {
if numFiles > 0 {
continue
}
2014-08-23 08:43:48 +00:00
if path == v.versionsPath {
2014-08-21 22:41:17 +00:00
if debug {
2014-08-23 08:43:48 +00:00
l.Debugln("Cleaner: versions dir is empty, don't delete", path)
2014-08-21 22:41:17 +00:00
}
continue
}
2014-08-21 22:41:17 +00:00
if debug {
2014-08-23 08:43:48 +00:00
l.Debugln("Cleaner: deleting empty directory", path)
2014-08-21 22:41:17 +00:00
}
2014-08-23 08:43:48 +00:00
err = os.Remove(path)
2014-08-21 22:41:17 +00:00
if err != nil {
2014-08-23 08:43:48 +00:00
l.Warnln("Versioner: can't remove directory", path, err)
2014-08-21 22:41:17 +00:00
}
}
if debug {
l.Debugln("Cleaner: Finished cleaning", v.versionsPath)
}
}
2014-08-23 08:43:48 +00:00
func (v Staggered) expire(versions []string) {
2014-08-21 22:41:17 +00:00
if debug {
l.Debugln("Versioner: Expiring versions", versions)
}
var prevAge int64
firstFile := true
for _, file := range versions {
if isFile(file) {
versionTime, err := time.Parse(TimeLayout, versionExt(file))
2014-08-21 22:41:17 +00:00
if err != nil {
2014-08-28 19:35:48 +00:00
l.Infof("Versioner: file name %q is invalid: %v", file, err)
2014-08-21 22:41:17 +00:00
continue
}
age := int64(time.Since(versionTime).Seconds())
2014-08-21 22:41:17 +00:00
2014-08-23 08:43:48 +00:00
// If the file is older than the max age of the last interval, remove it
2014-08-22 16:16:05 +00:00
if lastIntv := v.interval[len(v.interval)-1]; lastIntv.end > 0 && age > lastIntv.end {
2014-08-21 22:41:17 +00:00
if debug {
l.Debugln("Versioner: File over maximum age -> delete ", file)
}
err = os.Remove(file)
if err != nil {
2014-08-23 08:43:48 +00:00
l.Warnf("Versioner: can't remove %q: %v", file, err)
2014-08-21 22:41:17 +00:00
}
continue
}
2014-08-23 08:43:48 +00:00
// If it's the first (oldest) file in the list we can skip the interval checks
2014-08-21 22:41:17 +00:00
if firstFile {
prevAge = age
firstFile = false
continue
}
2014-08-23 08:43:48 +00:00
// Find the interval the file fits in
var usedInterval Interval
for _, usedInterval = range v.interval {
if age < usedInterval.end {
break
}
}
2014-08-21 22:41:17 +00:00
if prevAge-age < usedInterval.step {
if debug {
l.Debugln("too many files in step -> delete", file)
}
err = os.Remove(file)
if err != nil {
2014-08-23 08:43:48 +00:00
l.Warnf("Versioner: can't remove %q: %v", file, err)
2014-08-21 22:41:17 +00:00
}
continue
}
2014-08-23 08:43:48 +00:00
prevAge = age
2014-08-21 22:41:17 +00:00
} else {
2014-08-28 19:35:48 +00:00
l.Infof("non-file %q is named like a file version", file)
2014-08-21 22:41:17 +00:00
}
}
}
// Move away the named file to a version archive. If this function returns
// nil, the named file does not exist any more (has been archived).
func (v Staggered) Archive(filePath string) error {
if debug {
l.Debugln("Waiting for lock on ", v.versionsPath)
}
v.mutex.Lock()
defer v.mutex.Unlock()
fileInfo, err := os.Stat(filePath)
2014-08-21 22:41:17 +00:00
if err != nil {
if os.IsNotExist(err) {
if debug {
l.Debugln("not archiving nonexistent file", filePath)
}
return nil
} else {
return err
}
}
_, err = os.Stat(v.versionsPath)
if err != nil {
if os.IsNotExist(err) {
if debug {
l.Debugln("creating versions dir", v.versionsPath)
}
os.MkdirAll(v.versionsPath, 0755)
osutil.HideFile(v.versionsPath)
} else {
return err
}
}
if debug {
l.Debugln("archiving", filePath)
}
file := filepath.Base(filePath)
inFolderPath, err := filepath.Rel(v.folderPath, filepath.Dir(filePath))
2014-08-21 22:41:17 +00:00
if err != nil {
return err
}
dir := filepath.Join(v.versionsPath, inFolderPath)
2014-08-21 22:41:17 +00:00
err = os.MkdirAll(dir, 0755)
if err != nil && !os.IsExist(err) {
return err
}
2014-08-23 08:43:48 +00:00
ver := file + "~" + fileInfo.ModTime().Format(TimeLayout)
2014-08-21 22:41:17 +00:00
dst := filepath.Join(dir, ver)
if debug {
l.Debugln("moving to", dst)
}
err = osutil.Rename(filePath, dst)
if err != nil {
return err
}
versions, err := filepath.Glob(filepath.Join(dir, file+"~[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]-[0-9][0-9][0-9][0-9][0-9][0-9]"))
2014-08-21 22:41:17 +00:00
if err != nil {
l.Warnln("Versioner: error finding versions for", file, err)
return nil
}
sort.Strings(versions)
2014-08-23 08:43:48 +00:00
v.expire(versions)
2014-08-21 22:41:17 +00:00
return nil
}