2015-06-27 13:05:20 +00:00
package main
import (
2017-06-04 09:16:55 +00:00
"context"
2015-07-12 20:10:01 +00:00
"sync"
"time"
2015-06-27 13:05:20 +00:00
2017-07-23 12:21:03 +00:00
"github.com/restic/restic/internal/debug"
2023-01-14 16:38:20 +00:00
"github.com/restic/restic/internal/errors"
2017-07-24 15:42:25 +00:00
"github.com/restic/restic/internal/restic"
2015-06-27 13:05:20 +00:00
)
2021-10-31 22:25:36 +00:00
type lockContext struct {
cancel context . CancelFunc
refreshWG sync . WaitGroup
}
2015-07-12 20:10:01 +00:00
var globalLocks struct {
2021-10-31 22:25:36 +00:00
locks map [ * restic . Lock ] * lockContext
2015-07-12 20:10:01 +00:00
sync . Mutex
2020-08-09 11:54:39 +00:00
sync . Once
2015-07-12 20:10:01 +00:00
}
2015-06-27 13:05:20 +00:00
2021-11-14 15:32:03 +00:00
func lockRepo ( ctx context . Context , repo restic . Repository ) ( * restic . Lock , context . Context , error ) {
2020-08-09 11:24:47 +00:00
return lockRepository ( ctx , repo , false )
2015-06-27 13:05:20 +00:00
}
2021-11-14 15:32:03 +00:00
func lockRepoExclusive ( ctx context . Context , repo restic . Repository ) ( * restic . Lock , context . Context , error ) {
2020-08-09 11:24:47 +00:00
return lockRepository ( ctx , repo , true )
2015-06-27 13:50:36 +00:00
}
2021-10-31 22:25:36 +00:00
// lockRepository wraps the ctx such that it is cancelled when the repository is unlocked
// cancelling the original context also stops the lock refresh
2021-11-14 15:32:03 +00:00
func lockRepository ( ctx context . Context , repo restic . Repository , exclusive bool ) ( * restic . Lock , context . Context , error ) {
2020-08-09 11:54:39 +00:00
// make sure that a repository is unlocked properly and after cancel() was
// called by the cleanup handler in global.go
globalLocks . Do ( func ( ) {
AddCleanupHandler ( unlockAll )
} )
2015-06-27 13:50:36 +00:00
lockFn := restic . NewLock
if exclusive {
lockFn = restic . NewExclusiveLock
}
2020-08-09 11:24:47 +00:00
lock , err := lockFn ( ctx , repo )
2023-01-14 16:45:25 +00:00
if restic . IsInvalidLock ( err ) {
return nil , ctx , errors . Fatalf ( "%v\n\nthe `unlock --remove-all` command can be used to remove invalid locks. Make sure that no other restic process is accessing the repository when running the command" , err )
}
2015-06-27 13:05:20 +00:00
if err != nil {
2023-01-14 16:38:20 +00:00
return nil , ctx , errors . Fatalf ( "unable to create lock in backend: %v" , err )
2015-06-27 13:05:20 +00:00
}
2016-12-28 09:53:31 +00:00
debug . Log ( "create lock %p (exclusive %v)" , lock , exclusive )
2015-06-27 13:05:20 +00:00
2021-10-31 22:25:36 +00:00
ctx , cancel := context . WithCancel ( ctx )
lockInfo := & lockContext {
cancel : cancel ,
2015-07-12 20:10:01 +00:00
}
2021-10-31 22:25:36 +00:00
lockInfo . refreshWG . Add ( 2 )
refreshChan := make ( chan struct { } )
2015-07-12 20:10:01 +00:00
2021-10-31 22:25:36 +00:00
globalLocks . Lock ( )
globalLocks . locks [ lock ] = lockInfo
go refreshLocks ( ctx , lock , lockInfo , refreshChan )
go monitorLockRefresh ( ctx , lock , lockInfo , refreshChan )
2015-07-12 20:10:01 +00:00
globalLocks . Unlock ( )
2015-06-27 13:05:20 +00:00
2021-10-31 22:19:27 +00:00
return lock , ctx , err
2015-06-27 13:05:20 +00:00
}
2015-07-12 20:10:01 +00:00
var refreshInterval = 5 * time . Minute
2021-10-31 22:25:36 +00:00
// consider a lock refresh failed a bit before the lock actually becomes stale
// the difference allows to compensate for a small time drift between clients.
var refreshabilityTimeout = restic . StaleLockTimeout - refreshInterval * 3 / 2
func refreshLocks ( ctx context . Context , lock * restic . Lock , lockInfo * lockContext , refreshed chan <- struct { } ) {
2016-09-27 20:35:08 +00:00
debug . Log ( "start" )
2021-10-31 22:25:36 +00:00
ticker := time . NewTicker ( refreshInterval )
lastRefresh := lock . Time
2015-07-12 20:10:01 +00:00
defer func ( ) {
2021-10-31 22:25:36 +00:00
ticker . Stop ( )
// ensure that the context was cancelled before removing the lock
lockInfo . cancel ( )
2015-07-12 20:10:01 +00:00
2021-10-31 22:25:36 +00:00
// remove the lock from the repo
debug . Log ( "unlocking repository with lock %v" , lock )
if err := lock . Unlock ( ) ; err != nil {
debug . Log ( "error while unlocking: %v" , err )
Warnf ( "error while unlocking: %v" , err )
}
lockInfo . refreshWG . Done ( )
} ( )
2015-07-12 20:10:01 +00:00
for {
select {
2021-10-31 22:25:36 +00:00
case <- ctx . Done ( ) :
2016-09-27 20:35:08 +00:00
debug . Log ( "terminate" )
2015-07-12 20:10:01 +00:00
return
case <- ticker . C :
2021-10-31 22:25:36 +00:00
if time . Since ( lastRefresh ) > refreshabilityTimeout {
// the lock is too old, wait until the expiry monitor cancels the context
continue
}
2016-09-27 20:35:08 +00:00
debug . Log ( "refreshing locks" )
2021-10-31 22:25:36 +00:00
err := lock . Refresh ( context . TODO ( ) )
if err != nil {
Warnf ( "unable to refresh lock: %v\n" , err )
} else {
lastRefresh = lock . Time
// inform monitor gorountine about successful refresh
select {
case <- ctx . Done ( ) :
case refreshed <- struct { } { } :
2015-07-12 20:10:01 +00:00
}
}
2021-10-31 22:25:36 +00:00
}
}
}
func monitorLockRefresh ( ctx context . Context , lock * restic . Lock , lockInfo * lockContext , refreshed <- chan struct { } ) {
2022-09-10 17:25:06 +00:00
// time.Now() might use a monotonic timer which is paused during standby
// convert to unix time to ensure we compare real time values
2022-10-15 20:02:11 +00:00
lastRefresh := time . Now ( ) . UnixNano ( )
2022-09-10 17:25:06 +00:00
pollDuration := 1 * time . Second
if refreshInterval < pollDuration {
// require for TestLockFailedRefresh
pollDuration = refreshInterval / 5
}
// timers are paused during standby, which is a problem as the refresh timeout
// _must_ expire if the host was too long in standby. Thus fall back to periodic checks
// https://github.com/golang/go/issues/35012
timer := time . NewTimer ( pollDuration )
2021-10-31 22:25:36 +00:00
defer func ( ) {
timer . Stop ( )
lockInfo . cancel ( )
lockInfo . refreshWG . Done ( )
} ( )
for {
select {
case <- ctx . Done ( ) :
debug . Log ( "terminate expiry monitoring" )
return
case <- refreshed :
2022-10-23 20:40:21 +00:00
lastRefresh = time . Now ( ) . UnixNano ( )
2021-10-31 22:25:36 +00:00
case <- timer . C :
2022-10-15 20:02:11 +00:00
if time . Now ( ) . UnixNano ( ) - lastRefresh < refreshabilityTimeout . Nanoseconds ( ) {
2022-09-10 17:25:06 +00:00
// restart timer
timer . Reset ( pollDuration )
continue
}
2021-10-31 22:25:36 +00:00
Warnf ( "Fatal: failed to refresh lock in time\n" )
return
2015-07-12 20:10:01 +00:00
}
}
}
2020-11-28 12:52:47 +00:00
func unlockRepo ( lock * restic . Lock ) {
2020-08-09 11:25:13 +00:00
if lock == nil {
2020-11-28 12:52:47 +00:00
return
2020-08-09 11:25:13 +00:00
}
2015-07-12 20:10:01 +00:00
globalLocks . Lock ( )
2021-10-31 22:25:36 +00:00
lockInfo , exists := globalLocks . locks [ lock ]
delete ( globalLocks . locks , lock )
globalLocks . Unlock ( )
2018-02-25 12:11:03 +00:00
2021-10-31 22:25:36 +00:00
if ! exists {
debug . Log ( "unable to find lock %v in the global list of locks, ignoring" , lock )
return
2015-06-27 13:05:20 +00:00
}
2021-10-31 22:25:36 +00:00
lockInfo . cancel ( )
lockInfo . refreshWG . Wait ( )
2015-06-27 13:05:20 +00:00
}
2022-08-26 21:04:59 +00:00
func unlockAll ( code int ) ( int , error ) {
2015-07-12 20:10:01 +00:00
globalLocks . Lock ( )
2021-10-31 22:25:36 +00:00
locks := globalLocks . locks
2016-09-27 20:35:08 +00:00
debug . Log ( "unlocking %d locks" , len ( globalLocks . locks ) )
2021-10-31 22:25:36 +00:00
for _ , lockInfo := range globalLocks . locks {
lockInfo . cancel ( )
}
globalLocks . locks = make ( map [ * restic . Lock ] * lockContext )
globalLocks . Unlock ( )
for _ , lockInfo := range locks {
lockInfo . refreshWG . Wait ( )
2015-06-27 13:05:20 +00:00
}
2022-08-26 21:04:59 +00:00
return code , nil
2015-06-27 13:05:20 +00:00
}
2021-10-31 22:25:36 +00:00
func init ( ) {
globalLocks . locks = make ( map [ * restic . Lock ] * lockContext )
}