octoleo/gh-ost
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

WIP

Browse Source
This commit is contained in:
Arthur Schreiber 2022-01-26 11:13:47 +00:00 committed by GitHub
parent d6c83638d7
commit c143c2cc1a
2 changed files with 55 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

77
go/logic/throttler.go
View File

@ -346,31 +346,62 @@ func (this *Throttler) collectGeneralThrottleMetrics() error {
return setThrottle(true, fmt.Sprintf("%s %s", variableName, err), base.NoThrottleReasonHint) return setThrottle(true, fmt.Sprintf("%s %s", variableName, err), base.NoThrottleReasonHint)
} }
if criticalLoadMet && this.migrationContext.CriticalLoadHibernateSeconds > 0 { if criticalLoadMet {
hibernateDuration := time.Duration(this.migrationContext.CriticalLoadHibernateSeconds) * time.Second if this.migrationContext.CriticalLoadIntervalMilliseconds == 0 {
hibernateUntilTime := time.Now().Add(hibernateDuration) if this.migrationContext.CriticalLoadHibernateSeconds > 0 {
atomic.StoreInt64(&this.migrationContext.HibernateUntil, hibernateUntilTime.UnixNano()) hibernateDuration := time.Duration(this.migrationContext.CriticalLoadHibernateSeconds) * time.Second
this.migrationContext.Log.Errorf("critical-load met: %s=%d, >=%d. Will hibernate for the duration of %+v, until %+v", variableName, value, threshold, hibernateDuration, hibernateUntilTime) hibernateUntilTime := time.Now().Add(hibernateDuration)
go func() {
time.Sleep(hibernateDuration)
this.migrationContext.SetThrottleGeneralCheckResult(base.NewThrottleCheckResult(true, "leaving hibernation", base.LeavingHibernationThrottleReasonHint))
atomic.StoreInt64(&this.migrationContext.HibernateUntil, 0)
}()
return nil
}
if criticalLoadMet && this.migrationContext.CriticalLoadIntervalMilliseconds == 0 { // Only go into hibernation if we're not hibernating yet
this.migrationContext.PanicAbort <- fmt.Errorf("critical-load met: %s=%d, >=%d", variableName, value, threshold) if atomic.CompareAndSwapInt64(&this.migrationContext.HibernateUntil, 0, hibernateUntilTime.UnixNano()) {
} this.migrationContext.Log.Errorf("critical-load met: %s=%d, >=%d. Will hibernate for the duration of %+v, until %+v", variableName, value, threshold, hibernateDuration, hibernateUntilTime)
if criticalLoadMet && this.migrationContext.CriticalLoadIntervalMilliseconds > 0 { go func() {
this.migrationContext.Log.Errorf("critical-load met once: %s=%d, >=%d. Will check again in %d millis", variableName, value, threshold, this.migrationContext.CriticalLoadIntervalMilliseconds) time.Sleep(hibernateDuration)
go func() { this.migrationContext.SetThrottleGeneralCheckResult(base.NewThrottleCheckResult(true, "leaving hibernation", base.LeavingHibernationThrottleReasonHint))
timer := time.NewTimer(time.Millisecond * time.Duration(this.migrationContext.CriticalLoadIntervalMilliseconds)) atomic.StoreInt64(&this.migrationContext.HibernateUntil, 0)
<-timer.C }()
if criticalLoadMetAgain, variableName, value, threshold, _ := this.criticalLoadIsMet(); criticalLoadMetAgain { }
this.migrationContext.PanicAbort <- fmt.Errorf("critical-load met again after %d millis: %s=%d, >=%d", this.migrationContext.CriticalLoadIntervalMilliseconds, variableName, value, threshold) } else {
this.migrationContext.PanicAbort <- fmt.Errorf("critical-load met: %s=%d, >=%d", variableName, value, threshold)
} }
}()
return nil
}
if this.migrationContext.CriticalLoadIntervalMilliseconds > 0 {
this.migrationContext.Log.Errorf("critical-load met once: %s=%d, >=%d. Will check again in %d millis", variableName, value, threshold, this.migrationContext.CriticalLoadIntervalMilliseconds)
go func() {
timer := time.NewTimer(time.Millisecond * time.Duration(this.migrationContext.CriticalLoadIntervalMilliseconds))
<-timer.C
// If we've gone into hibernation, don't perform the second check
if atomic.LoadInt64(&this.migrationContext.HibernateUntil) > 0 {
return
}
if criticalLoadMetAgain, variableName, value, threshold, _ := this.criticalLoadIsMet(); criticalLoadMetAgain {
if this.migrationContext.CriticalLoadHibernateSeconds > 0 {
hibernateDuration := time.Duration(this.migrationContext.CriticalLoadHibernateSeconds) * time.Second
hibernateUntilTime := time.Now().Add(hibernateDuration)
// Only go into hibernation if we're not hibernating yet
if atomic.CompareAndSwapInt64(&this.migrationContext.HibernateUntil, 0, hibernateUntilTime.UnixNano()) {
this.migrationContext.Log.Errorf("critical-load met again after %d millis: %s=%d, >=%d. Will hibernate for the duration of %+v, until %+v", this.migrationContext.CriticalLoadIntervalMilliseconds, variableName, value, threshold, hibernateDuration, hibernateUntilTime)
go func() {
time.Sleep(hibernateDuration)
this.migrationContext.SetThrottleGeneralCheckResult(base.NewThrottleCheckResult(true, "leaving hibernation", base.LeavingHibernationThrottleReasonHint))
atomic.StoreInt64(&this.migrationContext.HibernateUntil, 0)
}()
}
} else {
this.migrationContext.PanicAbort <- fmt.Errorf("critical-load met again after %d millis: %s=%d, >=%d", this.migrationContext.CriticalLoadIntervalMilliseconds, variableName, value, threshold)
}
}
}()
}
return nil
} }
// Back to throttle considerations // Back to throttle considerations

2
script/ensure-go-installed
View File

@ -1,7 +1,7 @@
#!/bin/bash #!/bin/bash
PREFERRED_GO_VERSION=go1.16.4 PREFERRED_GO_VERSION=go1.16.4
SUPPORTED_GO_VERSIONS='go1.1[56]' SUPPORTED_GO_VERSIONS='go1.1[567]'
GO_PKG_DARWIN=${PREFERRED_GO_VERSION}.darwin-amd64.pkg GO_PKG_DARWIN=${PREFERRED_GO_VERSION}.darwin-amd64.pkg
GO_PKG_DARWIN_SHA=0f215de06019a054a3da46a0722989986c956d719c7a0a8fc38a5f3c216d6f6b GO_PKG_DARWIN_SHA=0f215de06019a054a3da46a0722989986c956d719c7a0a8fc38a5f3c216d6f6b