Skip to content

Commit

Permalink
fix callstack extraction for timed out page calls
Browse files Browse the repository at this point in the history
  • Loading branch information
pk910 committed Aug 31, 2024
1 parent 042e560 commit 33ffee6
Show file tree
Hide file tree
Showing 4 changed files with 22 additions and 46 deletions.
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ require (
github.com/sirupsen/logrus v1.9.3
github.com/stdatiks/jdenticon-go v0.1.0
github.com/tdewolff/minify v2.3.6+incompatible
github.com/timandy/routine v1.1.4
github.com/urfave/negroni v1.0.0
golang.org/x/crypto v0.26.0
golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -412,6 +412,8 @@ github.com/tdewolff/parse v2.3.4+incompatible h1:x05/cnGwIMf4ceLuDMBOdQ1qGniMoxp
github.com/tdewolff/parse v2.3.4+incompatible/go.mod h1:8oBwCsVmUkgHO8M5iCzSIDtpzXOT0WXX9cWhz+bIzJQ=
github.com/tdewolff/test v1.0.9 h1:SswqJCmeN4B+9gEAi/5uqT0qpi1y2/2O47V/1hhGZT0=
github.com/tdewolff/test v1.0.9/go.mod h1:6DAvZliBAAnD7rhVgwaM7DE5/d9NMOAJ09SqYqeK4QE=
github.com/timandy/routine v1.1.4 h1:L9eAli/ROJcW6LhmwZcusYQcdAqxAXGOQhEXLQSNWOA=
github.com/timandy/routine v1.1.4/go.mod h1:siBcl8iIsGmhLCajRGRcy7Y7FVcicNXkr97JODdt9fc=
github.com/tklauser/go-sysconf v0.3.14 h1:g5vzr9iPFFz24v2KZXs/pvpvh8/V9Fw6vQK5ZZb78yU=
github.com/tklauser/go-sysconf v0.3.14/go.mod h1:1ym4lWMLUOhuBOPGtRcJm7tEGX4SCYNEEEtghGG/8uY=
github.com/tklauser/numcpus v0.8.0 h1:Mx4Wwe/FjZLeQsK/6kt2EOepwwSl7SmJrK5bV/dXYgY=
Expand Down
33 changes: 19 additions & 14 deletions services/frontendcache.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import (
"github.com/ethpandaops/dora/cache"
"github.com/ethpandaops/dora/utils"
"github.com/sirupsen/logrus"
"github.com/timandy/routine"
)

type FrontendCacheService struct {
Expand Down Expand Up @@ -119,6 +120,8 @@ func (fc *FrontendCacheService) processPageCall(pageKey string, caching bool, pa
callIdx := fc.pageCallCounter
fc.pageCallCounterMutex.Unlock()

callGoId := int64(0)

go func(callIdx uint64) {
defer func() {
if err := recover(); err != nil {
Expand All @@ -130,6 +133,8 @@ func (fc *FrontendCacheService) processPageCall(pageKey string, caching bool, pa
}
}()

callGoId = routine.Goid()

// check cache
if !utils.Config.Frontend.Debug && caching && fc.getFrontendCache(pageKey, pageData) == nil {
logrus.Debugf("page served from cache: %v", pageKey)
Expand Down Expand Up @@ -169,7 +174,7 @@ func (fc *FrontendCacheService) processPageCall(pageKey string, caching bool, pa
return nil, &FrontendCachePageError{
name: "page timeout",
err: fmt.Errorf("page call %v timeout", callIdx),
stack: fc.extractPageCallStack(callIdx),
stack: fc.extractPageCallStack(callGoId),
}
}
}
Expand All @@ -190,35 +195,35 @@ func (fc *FrontendCacheService) completePageLoad(pageKey string, processingPage
fc.processingMutex.Unlock()
}

func (fc *FrontendCacheService) extractPageCallStack(callIdx uint64) string {
func (fc *FrontendCacheService) extractPageCallStack(callGoid int64) string {
if fc.callStackMutex.TryLock() {
runtime.Stack(fc.callStackBuffer, true)
fc.callStackMutex.Unlock()
}
fc.callStackMutex.RLock()
defer fc.callStackMutex.RUnlock()

callFnName := fmt.Sprintf("processPageCall.func1(0x%x)", callIdx)
scanner := bufio.NewScanner(bytes.NewReader(fc.callStackBuffer))
lastBlock := []string{}
isPageCall := false
stackTrace := []string{}
isRelevantCall := false
for scanner.Scan() {
line := scanner.Text()
if strings.HasPrefix(line, "goroutine ") {
if isPageCall {
if isRelevantCall {
break
}
lastBlock = []string{}
} else {
lastBlock = append(lastBlock, line)

if strings.Contains(line, callFnName) {
isPageCall = true
}
isRelevantCall = strings.HasPrefix(line, fmt.Sprintf("goroutine %v ", callGoid))
}

if isRelevantCall {
stackTrace = append(stackTrace, line)
}
}
if !isPageCall {

if !isRelevantCall {
return "call stack not found"
}
return strings.Join(lastBlock, "\n")

return strings.Join(stackTrace, "\n")
}
32 changes: 0 additions & 32 deletions utils/debug.go

This file was deleted.

0 comments on commit 33ffee6

Please sign in to comment.