Handle panics that percolate up to the graceful module (#11291)

* Handle panics in graceful goroutines

Adds a some deferred functions to handle panics in graceful goroutines

Signed-off-by: Andrew Thornton <art27@cantab.net>

* Handle panic in webhook.Deliver

Signed-off-by: Andrew Thornton <art27@cantab.net>

* Handle panic in mirror.syncMirror

Signed-off-by: Andrew Thornton <art27@cantab.net>

Co-authored-by: Lauris BH <lauris@nix.lv>
Co-authored-by: techknowlogick <techknowlogick@gitea.io>
This commit is contained in:
zeripath 2020-05-15 01:06:00 +01:00 committed by GitHub
parent 4159866528
commit 4a04740daf
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 50 additions and 1 deletions

View file

@ -74,6 +74,12 @@ type RunnableWithShutdownFns func(atShutdown, atTerminate func(context.Context,
func (g *Manager) RunWithShutdownFns(run RunnableWithShutdownFns) { func (g *Manager) RunWithShutdownFns(run RunnableWithShutdownFns) {
g.runningServerWaitGroup.Add(1) g.runningServerWaitGroup.Add(1)
defer g.runningServerWaitGroup.Done() defer g.runningServerWaitGroup.Done()
defer func() {
if err := recover(); err != nil {
log.Critical("PANIC during RunWithShutdownFns: %v\nStacktrace: %s", err, log.Stack(2))
g.doShutdown()
}
}()
run(func(ctx context.Context, atShutdown func()) { run(func(ctx context.Context, atShutdown func()) {
go func() { go func() {
select { select {
@ -103,6 +109,12 @@ type RunnableWithShutdownChan func(atShutdown <-chan struct{}, atTerminate Callb
func (g *Manager) RunWithShutdownChan(run RunnableWithShutdownChan) { func (g *Manager) RunWithShutdownChan(run RunnableWithShutdownChan) {
g.runningServerWaitGroup.Add(1) g.runningServerWaitGroup.Add(1)
defer g.runningServerWaitGroup.Done() defer g.runningServerWaitGroup.Done()
defer func() {
if err := recover(); err != nil {
log.Critical("PANIC during RunWithShutdownChan: %v\nStacktrace: %s", err, log.Stack(2))
g.doShutdown()
}
}()
run(g.IsShutdown(), func(ctx context.Context, atTerminate func()) { run(g.IsShutdown(), func(ctx context.Context, atTerminate func()) {
g.RunAtTerminate(ctx, atTerminate) g.RunAtTerminate(ctx, atTerminate)
}) })
@ -114,6 +126,12 @@ func (g *Manager) RunWithShutdownChan(run RunnableWithShutdownChan) {
func (g *Manager) RunWithShutdownContext(run func(context.Context)) { func (g *Manager) RunWithShutdownContext(run func(context.Context)) {
g.runningServerWaitGroup.Add(1) g.runningServerWaitGroup.Add(1)
defer g.runningServerWaitGroup.Done() defer g.runningServerWaitGroup.Done()
defer func() {
if err := recover(); err != nil {
log.Critical("PANIC during RunWithShutdownContext: %v\nStacktrace: %s", err, log.Stack(2))
g.doShutdown()
}
}()
run(g.ShutdownContext()) run(g.ShutdownContext())
} }
@ -121,18 +139,28 @@ func (g *Manager) RunWithShutdownContext(run func(context.Context)) {
func (g *Manager) RunAtTerminate(ctx context.Context, terminate func()) { func (g *Manager) RunAtTerminate(ctx context.Context, terminate func()) {
g.terminateWaitGroup.Add(1) g.terminateWaitGroup.Add(1)
go func() { go func() {
defer g.terminateWaitGroup.Done()
defer func() {
if err := recover(); err != nil {
log.Critical("PANIC during RunAtTerminate: %v\nStacktrace: %s", err, log.Stack(2))
}
}()
select { select {
case <-g.IsTerminate(): case <-g.IsTerminate():
terminate() terminate()
case <-ctx.Done(): case <-ctx.Done():
} }
g.terminateWaitGroup.Done()
}() }()
} }
// RunAtShutdown creates a go-routine to run the provided function at shutdown // RunAtShutdown creates a go-routine to run the provided function at shutdown
func (g *Manager) RunAtShutdown(ctx context.Context, shutdown func()) { func (g *Manager) RunAtShutdown(ctx context.Context, shutdown func()) {
go func() { go func() {
defer func() {
if err := recover(); err != nil {
log.Critical("PANIC during RunAtShutdown: %v\nStacktrace: %s", err, log.Stack(2))
}
}()
select { select {
case <-g.IsShutdown(): case <-g.IsShutdown():
shutdown() shutdown()
@ -144,6 +172,11 @@ func (g *Manager) RunAtShutdown(ctx context.Context, shutdown func()) {
// RunAtHammer creates a go-routine to run the provided function at shutdown // RunAtHammer creates a go-routine to run the provided function at shutdown
func (g *Manager) RunAtHammer(ctx context.Context, hammer func()) { func (g *Manager) RunAtHammer(ctx context.Context, hammer func()) {
go func() { go func() {
defer func() {
if err := recover(); err != nil {
log.Critical("PANIC during RunAtHammer: %v\nStacktrace: %s", err, log.Stack(2))
}
}()
select { select {
case <-g.IsHammer(): case <-g.IsHammer():
hammer() hammer()

View file

@ -26,6 +26,14 @@ import (
// Deliver deliver hook task // Deliver deliver hook task
func Deliver(t *models.HookTask) error { func Deliver(t *models.HookTask) error {
defer func() {
err := recover()
if err == nil {
return
}
// There was a panic whilst delivering a hook...
log.Error("PANIC whilst trying to deliver webhook[%d] for repo[%d] to %s Panic: %v\nStacktrace: %s", t.ID, t.RepoID, t.URL, err, log.Stack(2))
}()
t.IsDelivered = true t.IsDelivered = true
var req *http.Request var req *http.Request

View file

@ -333,6 +333,14 @@ func SyncMirrors(ctx context.Context) {
func syncMirror(repoID string) { func syncMirror(repoID string) {
log.Trace("SyncMirrors [repo_id: %v]", repoID) log.Trace("SyncMirrors [repo_id: %v]", repoID)
defer func() {
err := recover()
if err == nil {
return
}
// There was a panic whilst syncMirrors...
log.Error("PANIC whilst syncMirrors[%s] Panic: %v\nStacktrace: %s", repoID, err, log.Stack(2))
}()
mirrorQueue.Remove(repoID) mirrorQueue.Remove(repoID)
m, err := models.GetMirrorByRepoID(com.StrTo(repoID).MustInt64()) m, err := models.GetMirrorByRepoID(com.StrTo(repoID).MustInt64())