2014-11-16 20:13:20 +00:00
|
|
|
// Copyright (C) 2014 The Syncthing Authors.
|
2014-09-29 19:43:32 +00:00
|
|
|
//
|
2015-03-07 20:36:35 +00:00
|
|
|
// This Source Code Form is subject to the terms of the Mozilla Public
|
|
|
|
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
2017-02-09 06:52:18 +00:00
|
|
|
// You can obtain one at https://mozilla.org/MPL/2.0/.
|
2014-06-01 20:50:14 +00:00
|
|
|
|
2014-05-15 03:26:55 +00:00
|
|
|
package model
|
2014-03-02 22:58:14 +00:00
|
|
|
|
2014-03-28 13:36:57 +00:00
|
|
|
import (
|
2023-06-29 13:44:50 +00:00
|
|
|
"context"
|
2022-08-16 08:01:49 +00:00
|
|
|
"errors"
|
2016-10-18 20:00:01 +00:00
|
|
|
"fmt"
|
2019-07-28 08:25:05 +00:00
|
|
|
"path/filepath"
|
2021-07-27 19:27:52 +00:00
|
|
|
"strings"
|
2014-06-21 07:43:12 +00:00
|
|
|
"sync"
|
|
|
|
"time"
|
2019-07-28 08:25:05 +00:00
|
|
|
|
2023-06-29 13:44:50 +00:00
|
|
|
"github.com/prometheus/client_golang/prometheus"
|
2021-07-27 19:27:52 +00:00
|
|
|
"github.com/syncthing/syncthing/lib/events"
|
2019-07-28 08:25:05 +00:00
|
|
|
"github.com/syncthing/syncthing/lib/fs"
|
2021-07-27 19:27:52 +00:00
|
|
|
"github.com/syncthing/syncthing/lib/ur"
|
2014-03-28 13:36:57 +00:00
|
|
|
)
|
2014-03-02 22:58:14 +00:00
|
|
|
|
2016-11-03 21:33:33 +00:00
|
|
|
type Holdable interface {
|
|
|
|
Holders() string
|
2016-10-18 20:00:01 +00:00
|
|
|
}
|
|
|
|
|
2021-07-27 19:27:52 +00:00
|
|
|
func newDeadlockDetector(timeout time.Duration, evLogger events.Logger, fatal func(error)) *deadlockDetector {
|
2016-10-29 23:14:38 +00:00
|
|
|
return &deadlockDetector{
|
2021-07-27 19:27:52 +00:00
|
|
|
warnTimeout: timeout,
|
|
|
|
fatalTimeout: 10 * timeout,
|
|
|
|
lockers: make(map[string]sync.Locker),
|
|
|
|
evLogger: evLogger,
|
|
|
|
fatal: fatal,
|
2016-10-29 23:14:38 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
type deadlockDetector struct {
|
2021-07-27 19:27:52 +00:00
|
|
|
warnTimeout, fatalTimeout time.Duration
|
|
|
|
lockers map[string]sync.Locker
|
|
|
|
evLogger events.Logger
|
|
|
|
fatal func(error)
|
2016-10-29 23:14:38 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func (d *deadlockDetector) Watch(name string, mut sync.Locker) {
|
|
|
|
d.lockers[name] = mut
|
2014-06-21 07:43:12 +00:00
|
|
|
go func() {
|
|
|
|
for {
|
2021-07-27 19:27:52 +00:00
|
|
|
time.Sleep(d.warnTimeout / 4)
|
|
|
|
done := make(chan struct{}, 1)
|
2014-06-21 07:43:12 +00:00
|
|
|
|
|
|
|
go func() {
|
|
|
|
mut.Lock()
|
2016-12-18 18:57:41 +00:00
|
|
|
_ = 1 // empty critical section
|
2014-06-21 07:43:12 +00:00
|
|
|
mut.Unlock()
|
2021-07-27 19:27:52 +00:00
|
|
|
done <- struct{}{}
|
2014-06-21 07:43:12 +00:00
|
|
|
}()
|
|
|
|
|
2021-07-27 19:27:52 +00:00
|
|
|
d.watchInner(name, done)
|
2014-06-21 07:43:12 +00:00
|
|
|
}
|
|
|
|
}()
|
|
|
|
}
|
2019-07-28 08:25:05 +00:00
|
|
|
|
2021-07-27 19:27:52 +00:00
|
|
|
func (d *deadlockDetector) watchInner(name string, done chan struct{}) {
|
|
|
|
warn := time.NewTimer(d.warnTimeout)
|
|
|
|
fatal := time.NewTimer(d.fatalTimeout)
|
|
|
|
defer func() {
|
|
|
|
warn.Stop()
|
|
|
|
fatal.Stop()
|
|
|
|
}()
|
|
|
|
|
|
|
|
select {
|
|
|
|
case <-warn.C:
|
|
|
|
failure := ur.FailureDataWithGoroutines(fmt.Sprintf("potential deadlock detected at %s (short timeout)", name))
|
|
|
|
failure.Extra["timeout"] = d.warnTimeout.String()
|
|
|
|
d.evLogger.Log(events.Failure, failure)
|
|
|
|
case <-done:
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
select {
|
|
|
|
case <-fatal.C:
|
|
|
|
err := fmt.Errorf("potential deadlock detected at %s (long timeout)", name)
|
|
|
|
failure := ur.FailureDataWithGoroutines(err.Error())
|
|
|
|
failure.Extra["timeout"] = d.fatalTimeout.String()
|
|
|
|
others := d.otherHolders()
|
|
|
|
failure.Extra["other-holders"] = others
|
|
|
|
d.evLogger.Log(events.Failure, failure)
|
|
|
|
d.fatal(err)
|
|
|
|
// Give it a minute to shut down gracefully, maybe shutting down
|
|
|
|
// can get out of the deadlock (or it's not really a deadlock).
|
|
|
|
time.Sleep(time.Minute)
|
|
|
|
panic(fmt.Sprintf("%v:\n%v", err, others))
|
|
|
|
case <-done:
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func (d *deadlockDetector) otherHolders() string {
|
|
|
|
var b strings.Builder
|
|
|
|
for otherName, otherMut := range d.lockers {
|
|
|
|
if otherHolder, ok := otherMut.(Holdable); ok {
|
|
|
|
b.WriteString("===" + otherName + "===\n" + otherHolder.Holders() + "\n")
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return b.String()
|
|
|
|
}
|
|
|
|
|
2019-07-28 08:25:05 +00:00
|
|
|
// inWritableDir calls fn(path), while making sure that the directory
|
|
|
|
// containing `path` is writable for the duration of the call.
|
2019-07-31 08:53:35 +00:00
|
|
|
func inWritableDir(fn func(string) error, targetFs fs.Filesystem, path string, ignorePerms bool) error {
|
2019-07-28 08:25:05 +00:00
|
|
|
dir := filepath.Dir(path)
|
|
|
|
info, err := targetFs.Stat(dir)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
if !info.IsDir() {
|
|
|
|
return errors.New("Not a directory: " + path)
|
|
|
|
}
|
2022-03-26 10:07:07 +00:00
|
|
|
|
|
|
|
const permBits = fs.ModePerm | fs.ModeSetuid | fs.ModeSetgid | fs.ModeSticky
|
2022-05-22 11:52:40 +00:00
|
|
|
var parentErr error
|
2023-06-29 13:44:50 +00:00
|
|
|
if mode := info.Mode() & permBits; mode&0o200 == 0 {
|
2019-07-28 08:25:05 +00:00
|
|
|
// A non-writeable directory (for this user; we assume that's the
|
|
|
|
// relevant part). Temporarily change the mode so we can delete the
|
|
|
|
// file or directory inside it.
|
2023-06-29 13:44:50 +00:00
|
|
|
parentErr = targetFs.Chmod(dir, mode|0o700)
|
2022-05-22 11:52:40 +00:00
|
|
|
if parentErr != nil {
|
|
|
|
l.Debugf("Failed to make parent directory writable: %v", parentErr)
|
|
|
|
} else {
|
|
|
|
// Chmod succeeded, we should change the permissions back on the way
|
|
|
|
// out. If we fail we log the error as we have irrevocably messed up
|
|
|
|
// at this point. :( (The operation we were called to wrap has
|
|
|
|
// succeeded or failed on its own so returning an error to the
|
|
|
|
// caller is inappropriate.)
|
|
|
|
defer func() {
|
|
|
|
if err := targetFs.Chmod(dir, mode); err != nil && !fs.IsNotExist(err) {
|
|
|
|
logFn := l.Warnln
|
|
|
|
if ignorePerms {
|
|
|
|
logFn = l.Debugln
|
|
|
|
}
|
|
|
|
logFn("Failed to restore directory permissions after gaining write access:", err)
|
2022-03-26 10:07:07 +00:00
|
|
|
}
|
2022-05-22 11:52:40 +00:00
|
|
|
}()
|
|
|
|
}
|
2019-07-28 08:25:05 +00:00
|
|
|
}
|
|
|
|
|
2022-05-22 11:52:40 +00:00
|
|
|
err = fn(path)
|
|
|
|
if fs.IsPermission(err) && parentErr != nil {
|
|
|
|
err = fmt.Errorf("error after failing to make parent directory writable: %w", err)
|
|
|
|
}
|
|
|
|
return err
|
2019-07-28 08:25:05 +00:00
|
|
|
}
|
2023-06-29 13:44:50 +00:00
|
|
|
|
|
|
|
// addTimeUntilCancelled adds time to the counter for the duration of the
|
|
|
|
// Context. We do this piecemeal so that polling the counter during a long
|
|
|
|
// operation shows a relevant value, instead of the counter just increasing
|
|
|
|
// by a large amount at the end of the operation.
|
|
|
|
func addTimeUntilCancelled(ctx context.Context, counter prometheus.Counter) {
|
|
|
|
t0 := time.Now()
|
2023-06-29 13:56:35 +00:00
|
|
|
defer func() {
|
|
|
|
counter.Add(time.Since(t0).Seconds())
|
|
|
|
}()
|
|
|
|
|
2023-06-29 13:44:50 +00:00
|
|
|
ticker := time.NewTicker(time.Second)
|
|
|
|
defer ticker.Stop()
|
2023-06-29 13:56:35 +00:00
|
|
|
|
2023-06-29 13:44:50 +00:00
|
|
|
for {
|
|
|
|
select {
|
|
|
|
case <-ticker.C:
|
|
|
|
t1 := time.Now()
|
|
|
|
counter.Add(t1.Sub(t0).Seconds())
|
|
|
|
t0 = t1
|
|
|
|
case <-ctx.Done():
|
2023-06-29 13:56:35 +00:00
|
|
|
return
|
2023-06-29 13:44:50 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|