Skip to content

Commit

Permalink
add el extra data filter and allow dynamic column selection on filter…
Browse files Browse the repository at this point in the history 
…ed slots page
  • Loading branch information
@pk910
pk910 committed Jun 14, 2024
1 parent 1cfb0af commit c40e3f5
Show file tree
Hide file tree
Showing 4 changed files with 230 additions and 40 deletions.
83 changes: 77 additions & 6 deletions handlers/slots_filtered.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,9 @@ import (
"fmt"
"net/http"
"net/url"
"sort"
"strconv"
"strings"
"time"

"github.com/ethpandaops/dora/dbtypes"
Expand Down Expand Up @@ -34,8 +36,13 @@ func SlotsFiltered(w http.ResponseWriter, r *http.Request) {
if urlArgs.Has("s") {
pageIdx, _ = strconv.ParseUint(urlArgs.Get("s"), 10, 64)
}
var displayColumns string = ""
if urlArgs.Has("d") {
displayColumns = urlArgs.Get("d")
}

var graffiti string
var extradata string
var proposer string
var pname string
var withOrphaned uint64
Expand All @@ -45,6 +52,9 @@ func SlotsFiltered(w http.ResponseWriter, r *http.Request) {
if urlArgs.Has("f.graffiti") {
graffiti = urlArgs.Get("f.graffiti")
}
if urlArgs.Has("f.extra") {
extradata = urlArgs.Get("f.extra")
}
if urlArgs.Has("f.proposer") {
proposer = urlArgs.Get("f.proposer")
}
Expand All @@ -64,7 +74,7 @@ func SlotsFiltered(w http.ResponseWriter, r *http.Request) {
var pageError error
pageError = services.GlobalCallRateLimiter.CheckCallLimit(r, 2)
if pageError == nil {
data.Data, pageError = getFilteredSlotsPageData(pageIdx, pageSize, graffiti, proposer, pname, uint8(withOrphaned), uint8(withMissing))
data.Data, pageError = getFilteredSlotsPageData(pageIdx, pageSize, graffiti, extradata, proposer, pname, uint8(withOrphaned), uint8(withMissing), displayColumns)
}
if pageError != nil {
handlePageError(w, r, pageError)
Expand All @@ -76,11 +86,11 @@ func SlotsFiltered(w http.ResponseWriter, r *http.Request) {
}
}

func getFilteredSlotsPageData(pageIdx uint64, pageSize uint64, graffiti string, proposer string, pname string, withOrphaned uint8, withMissing uint8) (*models.SlotsFilteredPageData, error) {
func getFilteredSlotsPageData(pageIdx uint64, pageSize uint64, graffiti string, extradata string, proposer string, pname string, withOrphaned uint8, withMissing uint8, displayColumns string) (*models.SlotsFilteredPageData, error) {
pageData := &models.SlotsFilteredPageData{}
pageCacheKey := fmt.Sprintf("slots_filtered:%v:%v:%v:%v:%v:%v:%v", pageIdx, pageSize, graffiti, proposer, pname, withOrphaned, withMissing)
pageCacheKey := fmt.Sprintf("slots_filtered:%v:%v:%v:%v:%v:%v:%v:%v:%v", pageIdx, pageSize, graffiti, extradata, proposer, pname, withOrphaned, withMissing, displayColumns)
pageRes, pageErr := services.GlobalFrontendCache.ProcessCachedPage(pageCacheKey, true, pageData, func(_ *services.FrontendCacheProcessingPage) interface{} {
return buildFilteredSlotsPageData(pageIdx, pageSize, graffiti, proposer, pname, withOrphaned, withMissing)
return buildFilteredSlotsPageData(pageIdx, pageSize, graffiti, extradata, proposer, pname, withOrphaned, withMissing, displayColumns)
})
if pageErr == nil && pageRes != nil {
resData, resOk := pageRes.(*models.SlotsFilteredPageData)
Expand All @@ -92,11 +102,14 @@ func getFilteredSlotsPageData(pageIdx uint64, pageSize uint64, graffiti string,
return pageData, pageErr
}

func buildFilteredSlotsPageData(pageIdx uint64, pageSize uint64, graffiti string, proposer string, pname string, withOrphaned uint8, withMissing uint8) *models.SlotsFilteredPageData {
func buildFilteredSlotsPageData(pageIdx uint64, pageSize uint64, graffiti string, extradata string, proposer string, pname string, withOrphaned uint8, withMissing uint8, displayColumns string) *models.SlotsFilteredPageData {
filterArgs := url.Values{}
if graffiti != "" {
filterArgs.Add("f.graffiti", graffiti)
}
if extradata != "" {
filterArgs.Add("f.extra", extradata)
}
if proposer != "" {
filterArgs.Add("f.proposer", proposer)
}
Expand All @@ -110,14 +123,70 @@ func buildFilteredSlotsPageData(pageIdx uint64, pageSize uint64, graffiti string
filterArgs.Add("f.missing", fmt.Sprintf("%v", withMissing))
}

displayMap := map[uint64]bool{}
if displayColumns != "" {
for _, col := range strings.Split(displayColumns, " ") {
colNum, err := strconv.ParseUint(col, 10, 64)
if err != nil {
continue
}
displayMap[colNum] = true
}
}
if len(displayMap) == 0 {
displayMap = map[uint64]bool{
1: true,
2: true,
3: true,
4: true,
5: true,
6: true,
7: true,
8: true,
9: true,
10: true,
11: true,
}
} else {
displayList := make([]uint64, len(displayMap))
displayIdx := 0
for col := range displayMap {
displayList[displayIdx] = col
displayIdx++
}
sort.Slice(displayList, func(a, b int) bool {
return displayList[a] < displayList[b]
})
displayStr := make([]string, len(displayMap))
for idx, col := range displayList {
displayStr[idx] = fmt.Sprintf("%v", col)
}
filterArgs.Add("d", strings.Join(displayStr, " "))
}

pageData := &models.SlotsFilteredPageData{
FilterGraffiti: graffiti,
FilterExtraData: extradata,
FilterProposer: proposer,
FilterProposerName: pname,
FilterWithOrphaned: withOrphaned,
FilterWithMissing: withMissing,

DisplayEpoch: displayMap[1],
DisplaySlot: displayMap[2],
DisplayStatus: displayMap[3],
DisplayTime: displayMap[4],
DisplayProposer: displayMap[5],
DisplayAttestations: displayMap[6],
DisplayDeposits: displayMap[7],
DisplaySlashings: displayMap[8],
DisplayTxCount: displayMap[9],
DisplaySyncAgg: displayMap[10],
DisplayGraffiti: displayMap[11],
DisplayElExtraData: displayMap[12],
DisplayColCount: uint64(len(displayMap)),
}
logrus.Debugf("slots_filtered page called: %v:%v [%v]", pageIdx, pageSize, graffiti)
logrus.Debugf("slots_filtered page called: %v:%v [%v/%v]", pageIdx, pageSize, graffiti, extradata)
if pageIdx == 0 {
pageData.IsDefaultPage = true
}
Expand All @@ -142,6 +211,7 @@ func buildFilteredSlotsPageData(pageIdx uint64, pageSize uint64, graffiti string
pageData.Slots = make([]*models.SlotsFilteredPageDataSlot, 0)
blockFilter := &dbtypes.BlockFilter{
Graffiti: graffiti,
ExtraData: extradata,
ProposerName: pname,
WithOrphaned: withOrphaned,
WithMissing: withMissing,
Expand Down Expand Up @@ -181,6 +251,7 @@ func buildFilteredSlotsPageData(pageIdx uint64, pageSize uint64, graffiti string
slotData.SyncParticipation = float64(dbBlock.Block.SyncParticipation) * 100
slotData.EthTransactionCount = dbBlock.Block.EthTransactionCount
slotData.Graffiti = dbBlock.Block.Graffiti
slotData.ElExtraData = dbBlock.Block.EthBlockExtra
slotData.BlockRoot = dbBlock.Block.Root
if dbBlock.Block.EthBlockNumber != nil {
slotData.WithEthBlock = true
Expand Down
9 changes: 8 additions & 1 deletion services/beaconservice.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -736,6 +736,13 @@ func (bs *ChainService) GetDbBlocksByFilter(filter *dbtypes.BlockFilter, pageIdx
continue
}
}
if filter.ExtraData != "" {
executionExtraData, _ := indexer.GetExecutionExtraData(block.GetBlockBody())
blockExtraData := string(executionExtraData[:])
if !strings.Contains(blockExtraData, filter.ExtraData) {
continue
}
}
proposer := uint64(block.GetHeader().Message.ProposerIndex)
if filter.ProposerIndex != nil {
if proposer != *filter.ProposerIndex {
Expand All @@ -758,7 +765,7 @@ func (bs *ChainService) GetDbBlocksByFilter(filter *dbtypes.BlockFilter, pageIdx
}
}

if filter.WithMissing != 0 && filter.Graffiti == "" && filter.WithOrphaned != 2 {
if filter.WithMissing != 0 && filter.Graffiti == "" && filter.ExtraData == "" && filter.WithOrphaned != 2 {
// add missed blocks
idxHeadSlot := bs.indexer.GetHighestSlot()
idxHeadEpoch := utils.EpochOfSlot(idxHeadSlot)
Expand Down
Loading

0 comments on commit c40e3f5

Please sign in to comment.