code cleanup & added combined datasource approach for voluntary exits…

… page (show most recent voluntary exits from indexer cache)

db/slashings.go

@@ -57,41 +57,6 @@ func InsertSlashings(slashings []*dbtypes.Slashing, tx *sqlx.Tx) error {
 	return nil
 }
 
-func GetSlashings(firstSlot uint64, limit uint32, reason dbtypes.SlashingReason) []*dbtypes.Slashing {
-	var sql strings.Builder
-	args := []any{}
-	fmt.Fprint(&sql, `
-	SELECT
-		slot_number, slot_index, slot_root, orphaned, validator, slasher, reason
-	FROM slashings
-	`)
-	filterOp := "WHERE"
-	if firstSlot > 0 {
-		args = append(args, firstSlot)
-		fmt.Fprintf(&sql, " %v slot_number <= $%v ", filterOp, len(args))
-		filterOp = "AND"
-	}
-	if reason > 0 {
-		args = append(args, reason)
-		fmt.Fprintf(&sql, " %v reason <= $%v ", filterOp, len(args))
-		filterOp = "AND"
-	}
-
-	args = append(args, limit)
-	fmt.Fprintf(&sql, `
-	ORDER BY slot_number DESC, slot_index DESC
-	LIMIT $%v
-	`, len(args))
-
-	slashings := []*dbtypes.Slashing{}
-	err := ReaderDb.Select(&slashings, sql.String(), args...)
-	if err != nil {
-		logger.Errorf("Error while fetching slashings: %v", err)
-		return nil
-	}
-	return slashings
-}
-
 func GetSlashingForValidator(validator uint64) *dbtypes.Slashing {
 	var sql strings.Builder
 	args := []any{

db/voluntary_exits.go

@@ -55,34 +55,6 @@ func InsertVoluntaryExits(voluntaryExits []*dbtypes.VoluntaryExit, tx *sqlx.Tx)
 	return nil
 }
 
-func GetVoluntaryExits(firstSlot uint64, limit uint32) []*dbtypes.VoluntaryExit {
-	var sql strings.Builder
-	args := []any{}
-	fmt.Fprint(&sql, `
-	SELECT
-		slot_number, slot_index, slot_root, orphaned, validator
-	FROM voluntary_exits
-	`)
-	if firstSlot > 0 {
-		args = append(args, firstSlot)
-		fmt.Fprintf(&sql, " WHERE slot_number <= $%v ", len(args))
-	}
-
-	args = append(args, limit)
-	fmt.Fprintf(&sql, `
-	ORDER BY slot_number DESC, slot_index DESC
-	LIMIT $%v
-	`, len(args))
-
-	voluntaryExits := []*dbtypes.VoluntaryExit{}
-	err := ReaderDb.Select(&voluntaryExits, sql.String(), args...)
-	if err != nil {
-		logger.Errorf("Error while fetching voluntary exits: %v", err)
-		return nil
-	}
-	return voluntaryExits
-}
-
 func GetVoluntaryExitForValidator(validator uint64) *dbtypes.VoluntaryExit {
 	var sql strings.Builder
 	args := []any{

handlers/index.go

@@ -300,14 +300,14 @@ func buildIndexPageRecentSlotsData(pageData *models.IndexPageData, firstSlot uin
 			}
 			pageData.RecentSlots = append(pageData.RecentSlots, slotData)
 			blockCount++
-			buildIndexPageSlotGraph(pageData, slotData, &maxOpenFork, openForks)
+			buildIndexPageSlotGraph(slotData, &maxOpenFork, openForks)
 		}
 	}
 	pageData.RecentSlotCount = uint64(blockCount)
 	pageData.ForkTreeWidth = (maxOpenFork * 20) + 20
 }
 
-func buildIndexPageSlotGraph(pageData *models.IndexPageData, slotData *models.IndexPageDataSlots, maxOpenFork *int, openForks map[int][]byte) {
+func buildIndexPageSlotGraph(slotData *models.IndexPageDataSlots, maxOpenFork *int, openForks map[int][]byte) {
 	// fork tree
 	var forkGraphIdx int = -1
 	var freeForkIdx int = -1

handlers/pageData.go

@@ -58,7 +58,7 @@ func InitPageData(w http.ResponseWriter, r *http.Request, active, path, title st
 		ChainConfig:           utils.Config.Chain.Config,
 		Lang:                  "en-US",
 		Debug:                 utils.Config.Frontend.Debug,
-		MainMenuItems:         createMenuItems(active, isMainnet),
+		MainMenuItems:         createMenuItems(active),
 	}
 
 	if utils.Config.Frontend.SiteDescription != "" {
@@ -82,7 +82,7 @@ func InitPageData(w http.ResponseWriter, r *http.Request, active, path, title st
 	return data
 }
 
-func createMenuItems(active string, isMain bool) []types.MainMenuItem {
+func createMenuItems(active string) []types.MainMenuItem {
 	hiddenFor := []string{"confirmation", "login", "register"}
 
 	if utils.SliceContains(hiddenFor, active) {

handlers/voluntary_exits.go

@@ -9,7 +9,6 @@ import (
 
 	v1 "github.com/attestantio/go-eth2-client/api/v1"
 	"github.com/attestantio/go-eth2-client/spec/phase0"
-	"github.com/ethpandaops/dora/db"
 	"github.com/ethpandaops/dora/dbtypes"
 	"github.com/ethpandaops/dora/services"
 	"github.com/ethpandaops/dora/templates"
@@ -160,12 +159,7 @@ func buildFilteredVoluntaryExitsPageData(pageIdx uint64, pageSize uint64, minSlo
 		WithOrphaned:  withOrphaned,
 	}
 
-	offset := (pageIdx - 1) * pageSize
-
-	dbVoluntaryExits, totalRows, err := db.GetVoluntaryExitsFiltered(offset, uint32(pageSize), uint64(finalizedEpoch), voluntaryExitFilter)
-	if err != nil {
-		panic(err)
-	}
+	dbVoluntaryExits, totalRows := services.GlobalBeaconService.GetVoluntaryExitsByFilter(voluntaryExitFilter, pageIdx-1, uint32(pageSize))
 
 	validatorSetRsp := services.GlobalBeaconService.GetCachedValidatorSet()
 	validatorActivityMap, validatorActivityMax := services.GlobalBeaconService.GetValidatorActivity()

indexer/cache.go

@@ -28,6 +28,7 @@ type indexerCache struct {
 	persistEpoch            int64
 	cleanupBlockEpoch       int64
 	cleanupStatsEpoch       int64
+	lastPersistedEpoch      int64
 	slotMap                 map[uint64][]*CacheBlock
 	rootMap                 map[string]*CacheBlock
 	epochStatsMutex         sync.RWMutex
@@ -56,6 +57,7 @@ func newIndexerCache(indexer *Indexer) *indexerCache {
 		persistEpoch:            -1,
 		cleanupBlockEpoch:       -1,
 		cleanupStatsEpoch:       -1,
+		lastPersistedEpoch:      -1,
 		slotMap:                 make(map[uint64][]*CacheBlock),
 		rootMap:                 make(map[string]*CacheBlock),
 		epochStatsMap:           make(map[uint64][]*EpochStats),

indexer/cache_logic.go

@@ -405,21 +405,9 @@ func (cache *indexerCache) processCachePersistence() error {
 						return err
 					}
 
-					err = persistBlockDeposits(block, nil, true, tx)
+					// insert child objects as orphaned (we don't know if they're canonical yet)
+					err = persistBlockChildObjects(block, nil, true, tx)
 					if err != nil {
-						logger.Errorf("error persisting unfinalized deposits: %v", err)
-						return err
-					}
-
-					err = persistBlockVoluntaryExits(block, true, tx)
-					if err != nil {
-						logger.Errorf("error persisting unfinalized voluntary exits: %v", err)
-						return err
-					}
-
-					err = persistBlockSlashings(block, true, tx)
-					if err != nil {
-						logger.Errorf("error persisting unfinalized slashings: %v", err)
 						return err
 					}
 
@@ -453,6 +441,10 @@ func (cache *indexerCache) processCachePersistence() error {
 		}
 	}
 
+	if cache.lastPersistedEpoch < minPersistEpoch {
+		cache.lastPersistedEpoch = minPersistEpoch
+	}
+
 	for _, block := range pruneBlocks {
 		if block.isInUnfinalizedDb {
 			block.block = nil

indexer/indexer.go

@@ -321,6 +321,18 @@ func (indexer *Indexer) GetHighestSlot() uint64 {
 	return uint64(indexer.indexerCache.highestSlot)
 }
 
+func (indexer *Indexer) GetCacheState() (highestSlot int64, finalizedEpoch int64, persistedEpoch int64, processedEpoch int64) {
+	indexer.indexerCache.cacheMutex.RLock()
+	defer indexer.indexerCache.cacheMutex.RUnlock()
+
+	highestSlot = indexer.indexerCache.highestSlot
+	finalizedEpoch = indexer.indexerCache.finalizedEpoch
+	persistedEpoch = indexer.indexerCache.lastPersistedEpoch
+	processedEpoch = indexer.indexerCache.processedEpoch
+
+	return
+}
+
 func (indexer *Indexer) GetHeadForks(readyOnly bool) []*HeadFork {
 	headForks := []*HeadFork{}
 	for _, client := range indexer.consensusClients {

indexer/write_db.go

@@ -68,6 +68,18 @@ func persistBlockData(block *CacheBlock, epochStats *EpochStats, depositIndex *u
 
 	block.isInFinalizedDb = true
 
+	// insert child objects
+	err = persistBlockChildObjects(block, depositIndex, orphaned, tx)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func persistBlockChildObjects(block *CacheBlock, depositIndex *uint64, orphaned bool, tx *sqlx.Tx) error {
+	var err error
+
 	// insert deposits
 	err = persistBlockDeposits(block, depositIndex, orphaned, tx)
 	if err != nil {
@@ -356,7 +368,7 @@ func buildDbDeposits(block *CacheBlock, depositIndex *uint64) []*dbtypes.Deposit
 
 func persistBlockVoluntaryExits(block *CacheBlock, orphaned bool, tx *sqlx.Tx) error {
 	// insert voluntary exits
-	dbVoluntaryExits := buildDbVoluntaryExits(block)
+	dbVoluntaryExits := BuildDbVoluntaryExits(block)
 	if orphaned {
 		for idx := range dbVoluntaryExits {
 			dbVoluntaryExits[idx].Orphaned = true
@@ -373,7 +385,7 @@ func persistBlockVoluntaryExits(block *CacheBlock, orphaned bool, tx *sqlx.Tx) e
 	return nil
 }
 
-func buildDbVoluntaryExits(block *CacheBlock) []*dbtypes.VoluntaryExit {
+func BuildDbVoluntaryExits(block *CacheBlock) []*dbtypes.VoluntaryExit {
 	blockBody := block.GetBlockBody()
 	if blockBody == nil {
 		return nil