From d51fd436b541567acba42842b0b381400839b496 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Wed, 24 Jun 2015 18:17:01 +0200 Subject: [PATCH] Add locking functions --- cmd/restic/cmd_cat.go | 14 ++- doc/Design.md | 5 + lock.go | 236 ++++++++++++++++++++++++++++++++++++++++++ lock_test.go | 170 ++++++++++++++++++++++++++++++ 4 files changed, 424 insertions(+), 1 deletion(-) create mode 100644 lock.go create mode 100644 lock_test.go diff --git a/cmd/restic/cmd_cat.go b/cmd/restic/cmd_cat.go index 65345cfe7..4356c1d51 100644 --- a/cmd/restic/cmd_cat.go +++ b/cmd/restic/cmd_cat.go @@ -126,7 +126,19 @@ func (cmd CmdCat) Execute(args []string) error { fmt.Println(string(buf)) return nil case "lock": - return errors.New("not yet implemented") + lock, err := restic.LoadLock(s, id) + if err != nil { + return err + } + + buf, err := json.MarshalIndent(&lock, "", " ") + if err != nil { + return err + } + + fmt.Println(string(buf)) + + return nil } // load index, handle all the other types diff --git a/doc/Design.md b/doc/Design.md index 483ce3621..8dcab200d 100644 --- a/doc/Design.md +++ b/doc/Design.md @@ -375,6 +375,11 @@ As can be seen from the output of the program `sha256sum`, the hash matches the plaintext hash from the map included in the tree above, so the correct data has been returned. +Locks +----- + + + Backups and Deduplication ========================= diff --git a/lock.go b/lock.go new file mode 100644 index 000000000..07869a28b --- /dev/null +++ b/lock.go @@ -0,0 +1,236 @@ +package restic + +import ( + "errors" + "os" + "os/signal" + "os/user" + "strconv" + "sync" + "syscall" + "time" + + "github.com/restic/restic/backend" + "github.com/restic/restic/debug" + "github.com/restic/restic/repository" +) + +// Lock represents a process locking the repository for an operation. +// +// There are two types of locks: exclusive and non-exclusive. There may be many +// different non-exclusive locks, but at most one exclusive lock, which can +// only be acquired while no non-exclusive lock is held. +type Lock struct { + Time time.Time `json:"time"` + Exclusive bool `json:"exclusive"` + Hostname string `json:"hostname"` + Username string `json:"username"` + PID int `json:"pid"` + UID uint32 `json:"uid,omitempty"` + GID uint32 `json:"gid,omitempty"` + + repo *repository.Repository + lockID backend.ID +} + +var ( + ErrAlreadyLocked = errors.New("already locked") + ErrStaleLockFound = errors.New("stale lock found") +) + +// NewLock returns a new, non-exclusive lock for the repository. If an +// exclusive lock is already held by another process, ErrAlreadyLocked is +// returned. +func NewLock(repo *repository.Repository) (*Lock, error) { + return newLock(repo, false) +} + +// NewExclusiveLock returns a new, exclusive lock for the repository. If +// another lock (normal and exclusive) is already held by another process, +// ErrAlreadyLocked is returned. +func NewExclusiveLock(repo *repository.Repository) (*Lock, error) { + return newLock(repo, true) +} + +func newLock(repo *repository.Repository, excl bool) (*Lock, error) { + lock := &Lock{ + Time: time.Now(), + PID: os.Getpid(), + Exclusive: excl, + repo: repo, + } + + hn, err := os.Hostname() + if err == nil { + lock.Hostname = hn + } + + if err = lock.fillUserInfo(); err != nil { + return nil, err + } + + if err = lock.checkForOtherLocks(); err != nil { + return nil, err + } + + err = lock.createLock() + if err != nil { + return nil, err + } + + return lock, nil +} + +func (l *Lock) fillUserInfo() error { + usr, err := user.Current() + if err != nil { + return nil + } + l.Username = usr.Username + + uid, err := strconv.ParseInt(usr.Uid, 10, 32) + if err != nil { + return err + } + l.UID = uint32(uid) + + gid, err := strconv.ParseInt(usr.Gid, 10, 32) + if err != nil { + return err + } + l.GID = uint32(gid) + + return nil +} + +// checkForOtherLocks looks for other locks that currently exist in the repository. +// +// If an exclusive lock is to be created, checkForOtherLocks returns an error +// if there are any other locks, regardless if exclusive or not. If a +// non-exclusive lock is to be created, an error is only returned when an +// exclusive lock is found. +func (l *Lock) checkForOtherLocks() error { + return eachLock(l.repo, func(id backend.ID, lock *Lock, err error) error { + // ignore locks that cannot be loaded + if err != nil { + return nil + } + + if l.Exclusive { + return ErrAlreadyLocked + } + + if !l.Exclusive && lock.Exclusive { + return ErrAlreadyLocked + } + + return nil + }) +} + +func eachLock(repo *repository.Repository, f func(backend.ID, *Lock, error) error) error { + done := make(chan struct{}) + defer close(done) + + for id := range repo.List(backend.Lock, done) { + lock, err := LoadLock(repo, id) + err = f(id, lock, err) + if err != nil { + return err + } + } + + return nil +} + +// createLock acquires the lock by creating a file in the repository. +func (l *Lock) createLock() error { + id, err := l.repo.SaveJSONUnpacked(backend.Lock, l) + if err != nil { + return err + } + + l.lockID = id + return nil +} + +// Unlock removes the lock from the repository. +func (l *Lock) Unlock() error { + if l == nil || l.lockID == nil { + return nil + } + + return l.repo.Backend().Remove(backend.Lock, l.lockID.String()) +} + +var staleTimeout = 30 * time.Minute + +// Stale returns true if the lock is stale. A lock is stale if the timestamp is +// older than 30 minutes or if it was created on the current machine and the +// process isn't alive any more. +func (l *Lock) Stale() bool { + debug.Log("Lock.Stale", "testing if lock %v for process %d is stale", l.lockID.Str(), l.PID) + if time.Now().Sub(l.Time) > staleTimeout { + debug.Log("Lock.Stale", "lock is stale, timestamp is too old: %v\n", l.Time) + return true + } + + proc, err := os.FindProcess(l.PID) + defer proc.Release() + if err != nil { + debug.Log("Lock.Stale", "error searching for process %d: %v\n", l.PID, err) + return true + } + + debug.Log("Lock.Stale", "sending SIGHUP to process %d\n", l.PID) + err = proc.Signal(syscall.SIGHUP) + if err != nil { + debug.Log("Lock.Stale", "signal error: %v, lock is probably stale\n", err) + return true + } + + debug.Log("Lock.Stale", "lock not stale\n") + return false +} + +// listen for incoming SIGHUP and ignore +var ignoreSIGHUP sync.Once + +func init() { + ignoreSIGHUP.Do(func() { + go func() { + c := make(chan os.Signal) + signal.Notify(c, syscall.SIGHUP) + for s := range c { + debug.Log("lock.ignoreSIGHUP", "Signal received: %v\n", s) + } + }() + }) +} + +// LoadLock loads and unserializes a lock from a repository. +func LoadLock(repo *repository.Repository, id backend.ID) (*Lock, error) { + lock := &Lock{} + if err := repo.LoadJSONUnpacked(backend.Lock, id, lock); err != nil { + return nil, err + } + lock.lockID = id + + return lock, nil +} + +// RemoveStaleLocks deletes all locks detected as stale from the repository. +func RemoveStaleLocks(repo *repository.Repository) error { + return eachLock(repo, func(id backend.ID, lock *Lock, err error) error { + // ignore locks that cannot be loaded + if err != nil { + return nil + } + + if lock.Stale() { + return repo.Backend().Remove(backend.Lock, id.String()) + } + + return nil + }) +} diff --git a/lock_test.go b/lock_test.go new file mode 100644 index 000000000..a3a89c9e1 --- /dev/null +++ b/lock_test.go @@ -0,0 +1,170 @@ +package restic_test + +import ( + "os" + "testing" + "time" + + "github.com/restic/restic" + "github.com/restic/restic/backend" + "github.com/restic/restic/repository" + . "github.com/restic/restic/test" +) + +func TestLock(t *testing.T) { + repo := SetupRepo() + defer TeardownRepo(repo) + + lock, err := restic.NewLock(repo) + OK(t, err) + + OK(t, lock.Unlock()) +} + +func TestDoubleUnlock(t *testing.T) { + repo := SetupRepo() + defer TeardownRepo(repo) + + lock, err := restic.NewLock(repo) + OK(t, err) + + OK(t, lock.Unlock()) + + err = lock.Unlock() + Assert(t, err != nil, + "double unlock didn't return an error, got %v", err) +} + +func TestMultipleLock(t *testing.T) { + repo := SetupRepo() + defer TeardownRepo(repo) + + lock1, err := restic.NewLock(repo) + OK(t, err) + + lock2, err := restic.NewLock(repo) + OK(t, err) + + OK(t, lock1.Unlock()) + OK(t, lock2.Unlock()) +} + +func TestLockExclusive(t *testing.T) { + repo := SetupRepo() + defer TeardownRepo(repo) + + elock, err := restic.NewExclusiveLock(repo) + OK(t, err) + OK(t, elock.Unlock()) +} + +func TestLockOnExclusiveLockedRepo(t *testing.T) { + repo := SetupRepo() + defer TeardownRepo(repo) + + elock, err := restic.NewExclusiveLock(repo) + OK(t, err) + + lock, err := restic.NewLock(repo) + Assert(t, err == restic.ErrAlreadyLocked, + "create normal lock with exclusively locked repo didn't return an error") + + OK(t, lock.Unlock()) + OK(t, elock.Unlock()) +} + +func TestExclusiveLockOnLockedRepo(t *testing.T) { + repo := SetupRepo() + defer TeardownRepo(repo) + + elock, err := restic.NewLock(repo) + OK(t, err) + + lock, err := restic.NewExclusiveLock(repo) + Assert(t, err == restic.ErrAlreadyLocked, + "create exclusive lock with locked repo didn't return an error") + + OK(t, lock.Unlock()) + OK(t, elock.Unlock()) +} + +func createFakeLock(repo *repository.Repository, t time.Time, pid int) (backend.ID, error) { + newLock := &restic.Lock{Time: t, PID: pid} + return repo.SaveJSONUnpacked(backend.Lock, &newLock) +} + +func removeLock(repo *repository.Repository, id backend.ID) error { + return repo.Backend().Remove(backend.Lock, id.String()) +} + +var staleLockTests = []struct { + timestamp time.Time + stale bool + pid int +}{ + { + timestamp: time.Now(), + stale: false, + pid: os.Getpid(), + }, + { + timestamp: time.Now().Add(-time.Hour), + stale: true, + pid: os.Getpid(), + }, + { + timestamp: time.Now().Add(3 * time.Minute), + stale: false, + pid: os.Getpid(), + }, + { + timestamp: time.Now(), + stale: true, + pid: os.Getpid() + 500, + }, +} + +func TestLockStale(t *testing.T) { + for i, test := range staleLockTests { + lock := restic.Lock{ + Time: test.timestamp, + PID: test.pid, + } + + Assert(t, lock.Stale() == test.stale, + "TestStaleLock: test %d failed: expected stale: %v, got %v", + i, test.stale, !test.stale) + } +} + +func lockExists(repo *repository.Repository, t testing.TB, id backend.ID) bool { + exists, err := repo.Backend().Test(backend.Lock, id.String()) + OK(t, err) + + return exists +} + +func TestLockWithStaleLock(t *testing.T) { + repo := SetupRepo() + defer TeardownRepo(repo) + + id1, err := createFakeLock(repo, time.Now().Add(-time.Hour), os.Getpid()) + OK(t, err) + + id2, err := createFakeLock(repo, time.Now().Add(-time.Minute), os.Getpid()) + OK(t, err) + + id3, err := createFakeLock(repo, time.Now().Add(-time.Minute), os.Getpid()+500) + OK(t, err) + + OK(t, restic.RemoveStaleLocks(repo)) + + Assert(t, lockExists(repo, t, id1) == false, + "stale lock still exists after RemoveStaleLocks was called") + Assert(t, lockExists(repo, t, id2) == true, + "non-stale lock was removed by RemoveStaleLocks") + Assert(t, lockExists(repo, t, id3) == false, + "stale lock still exists after RemoveStaleLocks was called") + + OK(t, removeLock(repo, id2)) +}