Let suture logging bubble upwards

This commit is contained in:
Jakob Borg 2015-07-11 10:52:57 +10:00
parent d16b04b683
commit 1b837116e6
7 changed files with 50 additions and 31 deletions

2
Godeps/Godeps.json generated
View File

@ -43,7 +43,7 @@
}, },
{ {
"ImportPath": "github.com/thejerf/suture", "ImportPath": "github.com/thejerf/suture",
"Rev": "ff19fb384c3fe30f42717967eaa69da91e5f317c" "Rev": "fc7aaeabdc43fe41c5328efa1479ffea0b820978"
}, },
{ {
"ImportPath": "github.com/vitrun/qart/coding", "ImportPath": "github.com/vitrun/qart/coding",

View File

@ -1,4 +1,4 @@
Copyright (c) 2014 Barracuda Networks, Inc. Copyright (c) 2014-2015 Barracuda Networks, Inc.
Permission is hereby granted, free of charge, to any person obtaining a copy Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal of this software and associated documentation files (the "Software"), to deal

View File

@ -36,6 +36,13 @@ to your Supervisor. Supervisors are also services, so you can create a
tree structure here, depending on the exact combination of restarts tree structure here, depending on the exact combination of restarts
you want to create. you want to create.
As a special case, when adding Supervisors to Supervisors, the "sub"
supervisor will have the "super" supervisor's Log function copied.
This allows you to set one log function on the "top" supervisor, and
have it propagate down to all the sub-supervisors. This also allows
libraries or modules to provide Supervisors without having to commit
their users to a particular logging method.
Finally, as what is probably the last line of your main() function, call Finally, as what is probably the last line of your main() function, call
.Serve() on your top level supervisor. This will start all the services .Serve() on your top level supervisor. This will start all the services
you've defined. you've defined.
@ -126,8 +133,10 @@ type Supervisor struct {
// If you ever come up with some need to get into these, submit a pull // If you ever come up with some need to get into these, submit a pull
// request to make them public and some smidge of justification, and // request to make them public and some smidge of justification, and
// I'll happily do it. // I'll happily do it.
logBadStop func(Service) // But since I've now changed the signature on these once, I'm glad I
logFailure func(service Service, currentFailures float64, failureThreshold float64, restarting bool, error interface{}, stacktrace []byte) // didn't start with them public... :)
logBadStop func(*Supervisor, Service)
logFailure func(supervisor *Supervisor, service Service, currentFailures float64, failureThreshold float64, restarting bool, error interface{}, stacktrace []byte)
logBackoff func(*Supervisor, bool) logBackoff func(*Supervisor, bool)
// avoid a dependency on github.com/thejerf/abtime by just implementing // avoid a dependency on github.com/thejerf/abtime by just implementing
@ -233,10 +242,10 @@ func New(name string, spec Spec) (s *Supervisor) {
s.resumeTimer = make(chan time.Time) s.resumeTimer = make(chan time.Time)
// set up the default logging handlers // set up the default logging handlers
s.logBadStop = func(service Service) { s.logBadStop = func(supervisor *Supervisor, service Service) {
s.log(fmt.Sprintf("Service %s failed to terminate in a timely manner", serviceName(service))) s.log(fmt.Sprintf("%s: Service %s failed to terminate in a timely manner", serviceName(supervisor), serviceName(service)))
} }
s.logFailure = func(service Service, failures float64, threshold float64, restarting bool, err interface{}, st []byte) { s.logFailure = func(supervisor *Supervisor, service Service, failures float64, threshold float64, restarting bool, err interface{}, st []byte) {
var errString string var errString string
e, canError := err.(error) e, canError := err.(error)
@ -246,7 +255,7 @@ func New(name string, spec Spec) (s *Supervisor) {
errString = fmt.Sprintf("%#v", err) errString = fmt.Sprintf("%#v", err)
} }
s.log(fmt.Sprintf("Failed service '%s' (%f failures of %f), restarting: %#v, error: %s, stacktrace: %s", serviceName(service), failures, threshold, restarting, errString, string(st))) s.log(fmt.Sprintf("%s: Failed service '%s' (%f failures of %f), restarting: %#v, error: %s, stacktrace: %s", serviceName(supervisor), serviceName(service), failures, threshold, restarting, errString, string(st)))
} }
s.logBackoff = func(s *Supervisor, entering bool) { s.logBackoff = func(s *Supervisor, entering bool) {
if entering { if entering {
@ -346,12 +355,24 @@ will be started when the supervisor is.
The returned ServiceID may be passed to the Remove method of the Supervisor The returned ServiceID may be passed to the Remove method of the Supervisor
to terminate the service. to terminate the service.
As a special behavior, if the service added is itself a supervisor, the
supervisor being added will copy the Log function from the Supervisor it
is being added to. This allows factoring out providing a Supervisor
from its logging.
*/ */
func (s *Supervisor) Add(service Service) ServiceToken { func (s *Supervisor) Add(service Service) ServiceToken {
if s == nil { if s == nil {
panic("can't add service to nil *suture.Supervisor") panic("can't add service to nil *suture.Supervisor")
} }
if supervisor, isSupervisor := service.(*Supervisor); isSupervisor {
supervisor.logBadStop = s.logBadStop
supervisor.logFailure = s.logFailure
supervisor.logBackoff = s.logBackoff
}
if s.state == notRunning { if s.state == notRunning {
id := s.serviceCounter id := s.serviceCounter
s.serviceCounter++ s.serviceCounter++
@ -492,12 +513,12 @@ func (s *Supervisor) handleFailedService(id serviceID, err interface{}, stacktra
if monitored { if monitored {
if s.state == normal { if s.state == normal {
s.runService(failedService, id) s.runService(failedService, id)
s.logFailure(failedService, s.failures, s.failureThreshold, true, err, stacktrace) s.logFailure(s, failedService, s.failures, s.failureThreshold, true, err, stacktrace)
} else { } else {
// FIXME: When restarting, check that the service still // FIXME: When restarting, check that the service still
// exists (it may have been stopped in the meantime) // exists (it may have been stopped in the meantime)
s.restartQueue = append(s.restartQueue, id) s.restartQueue = append(s.restartQueue, id)
s.logFailure(failedService, s.failures, s.failureThreshold, false, err, stacktrace) s.logFailure(s, failedService, s.failures, s.failureThreshold, false, err, stacktrace)
} }
} }
} }
@ -536,7 +557,7 @@ func (s *Supervisor) removeService(id serviceID) {
case <-successChan: case <-successChan:
// Life is good! // Life is good!
case <-failChan: case <-failChan:
s.logBadStop(service) s.logBadStop(s, service)
} }
}() }()
} }

View File

@ -77,7 +77,7 @@ func TestFailures(t *testing.T) {
// to avoid deadlocks during shutdown, we have to not try to send // to avoid deadlocks during shutdown, we have to not try to send
// things out on channels while we're shutting down (this undoes the // things out on channels while we're shutting down (this undoes the
// logFailure overide about 25 lines down) // logFailure overide about 25 lines down)
s.logFailure = func(Service, float64, float64, bool, interface{}, []byte) {} s.logFailure = func(*Supervisor, Service, float64, float64, bool, interface{}, []byte) {}
s.Stop() s.Stop()
}() }()
s.sync() s.sync()
@ -102,7 +102,7 @@ func TestFailures(t *testing.T) {
failNotify := make(chan bool) failNotify := make(chan bool)
// use this to synchronize on here // use this to synchronize on here
s.logFailure = func(s Service, cf float64, ft float64, r bool, error interface{}, stacktrace []byte) { s.logFailure = func(supervisor *Supervisor, s Service, cf float64, ft float64, r bool, error interface{}, stacktrace []byte) {
failNotify <- r failNotify <- r
} }
@ -276,8 +276,8 @@ func TestDefaultLogging(t *testing.T) {
serviceName(&BarelyService{}) serviceName(&BarelyService{})
s.logBadStop(service) s.logBadStop(s, service)
s.logFailure(service, 1, 1, true, errors.New("test error"), []byte{}) s.logFailure(s, service, 1, 1, true, errors.New("test error"), []byte{})
s.Stop() s.Stop()
} }
@ -289,9 +289,17 @@ func TestNestedSupervisors(t *testing.T) {
super2 := NewSimple("Nested5") super2 := NewSimple("Nested5")
service := NewService("Service5") service := NewService("Service5")
super2.logBadStop = func(*Supervisor, Service) {
panic("Failed to copy logBadStop")
}
super1.Add(super2) super1.Add(super2)
super2.Add(service) super2.Add(service)
// test the functions got copied from super1; if this panics, it didn't
// get copied
super2.logBadStop(super2, service)
go super1.Serve() go super1.Serve()
super1.sync() super1.sync()
@ -340,7 +348,7 @@ func TestStoppingStillWorksWithHungServices(t *testing.T) {
return resumeChan return resumeChan
} }
failNotify := make(chan struct{}) failNotify := make(chan struct{})
s.logBadStop = func(s Service) { s.logBadStop = func(supervisor *Supervisor, s Service) {
failNotify <- struct{}{} failNotify <- struct{}{}
} }
@ -438,7 +446,7 @@ func TestFailingSupervisors(t *testing.T) {
} }
failNotify := make(chan string) failNotify := make(chan string)
// use this to synchronize on here // use this to synchronize on here
s1.logFailure = func(s Service, cf float64, ft float64, r bool, error interface{}, stacktrace []byte) { s1.logFailure = func(supervisor *Supervisor, s Service, cf float64, ft float64, r bool, error interface{}, stacktrace []byte) {
failNotify <- fmt.Sprintf("%s", s) failNotify <- fmt.Sprintf("%s", s)
} }

View File

@ -425,7 +425,9 @@ func upgradeViaRest() error {
func syncthingMain() { func syncthingMain() {
// Create a main service manager. We'll add things to this as we go along. // Create a main service manager. We'll add things to this as we go along.
// We want any logging it does to go through our log system. // We want any logging it does to go through our log system. Other
// suture.Supervisor:s that are Add()ed to mainSvc will inherit this log
// function.
mainSvc := suture.New("main", suture.Spec{ mainSvc := suture.New("main", suture.Spec{
Log: func(line string) { Log: func(line string) {
if debugSuture { if debugSuture {

View File

@ -29,12 +29,6 @@ func NewBroadcast(port int) *Broadcast {
// a while to get solved... // a while to get solved...
FailureThreshold: 2, FailureThreshold: 2,
FailureBackoff: 60 * time.Second, FailureBackoff: 60 * time.Second,
// Only log restarts in debug mode.
Log: func(line string) {
if debug {
l.Debugln(line)
}
},
}), }),
port: port, port: port,
inbox: make(chan []byte), inbox: make(chan []byte),

View File

@ -107,13 +107,7 @@ var (
// for file data without altering the local folder in any way. // for file data without altering the local folder in any way.
func NewModel(cfg *config.Wrapper, id protocol.DeviceID, deviceName, clientName, clientVersion string, ldb *leveldb.DB) *Model { func NewModel(cfg *config.Wrapper, id protocol.DeviceID, deviceName, clientName, clientVersion string, ldb *leveldb.DB) *Model {
m := &Model{ m := &Model{
Supervisor: suture.New("model", suture.Spec{ Supervisor: suture.NewSimple("model"),
Log: func(line string) {
if debug {
l.Debugln(line)
}
},
}),
cfg: cfg, cfg: cfg,
db: ldb, db: ldb,
finder: db.NewBlockFinder(ldb, cfg), finder: db.NewBlockFinder(ldb, cfg),