change database schema to avoid expensive joins on search views

ethpandaops · Apr 26, 2024 · d2e1056 · d2e1056
1 parent a938c79
commit d2e1056
Show file tree

Hide file tree

Showing 20 changed files with 768 additions and 310 deletions.
diff --git a/db/db.go b/db/db.go
diff --git a/db/schema/pgsql/20240423132923_block-status.sql b/db/schema/pgsql/20240423132923_block-status.sql
@@ -0,0 +1,110 @@
+-- +goose Up
+-- +goose StatementBegin
+
+CREATE TABLE IF NOT EXISTS public.slots
+(
+    slot bigint NOT NULL,
+    proposer bigint NOT NULL,
+    status smallint NOT NULL,
+    root bytea NOT NULL,
+    parent_root bytea NULL,
+    state_root bytea NULL,
+    graffiti bytea NULL,
+    graffiti_text text NULL COLLATE pg_catalog."default",
+    attestation_count integer NULL DEFAULT 0,
+    deposit_count integer NULL DEFAULT 0,
+    exit_count integer NULL DEFAULT 0,
+    withdraw_count integer NULL DEFAULT 0,
+    withdraw_amount bigint NULL DEFAULT 0,
+    attester_slashing_count integer NULL DEFAULT 0,
+    proposer_slashing_count integer NULL DEFAULT 0,
+    bls_change_count integer NULL DEFAULT 0,
+    eth_transaction_count integer NULL DEFAULT 0,
+    eth_block_number bigint NULL,
+    eth_block_hash bytea NULL,
+    eth_block_extra text NULL COLLATE pg_catalog."default",
+    sync_participation real NULL DEFAULT 0,
+    CONSTRAINT slots_pkey PRIMARY KEY (slot, root)
+);
+
+CREATE INDEX IF NOT EXISTS "slots_root_idx"
+    ON public."slots"
+    ("root" ASC NULLS FIRST);
+
+CREATE INDEX IF NOT EXISTS "slots_graffiti_idx"
+    ON public."slots" USING gin 
+    ("graffiti_text" gin_trgm_ops);
+
+CREATE INDEX IF NOT EXISTS "slots_eth_block_extra_idx"
+    ON public."slots" USING gin 
+    ("eth_block_extra" gin_trgm_ops);
+
+CREATE INDEX IF NOT EXISTS "slots_slot_idx"
+    ON public."slots" 
+    ("slot" ASC NULLS LAST);
+
+CREATE INDEX IF NOT EXISTS "slots_state_root_idx"
+    ON public."slots" 
+    ("state_root" ASC NULLS LAST);
+
+CREATE INDEX IF NOT EXISTS "slots_eth_block_number_idx"
+    ON public."slots" 
+    ("eth_block_number" ASC NULLS LAST);
+
+CREATE INDEX IF NOT EXISTS "slots_eth_block_hash_idx"
+    ON public."slots" 
+    ("eth_block_hash" ASC NULLS LAST);
+
+CREATE INDEX IF NOT EXISTS "slots_proposer_idx"
+    ON public."slots" 
+    ("proposer" ASC NULLS LAST);
+
+CREATE INDEX IF NOT EXISTS "slots_status_idx"
+    ON public."slots" 
+    ("status" ASC NULLS LAST);
+
+CREATE INDEX IF NOT EXISTS "slots_parent_root_idx"
+    ON public."slots" 
+    ("parent_root" ASC NULLS LAST);
+
+-- migrate blocks
+
+INSERT INTO "slots" (
+    slot, proposer, status, root, parent_root, state_root, graffiti, graffiti_text,
+    attestation_count, deposit_count, exit_count, withdraw_count, withdraw_amount, attester_slashing_count, 
+    proposer_slashing_count, bls_change_count, eth_transaction_count, eth_block_number, eth_block_hash, sync_participation
+)
+SELECT 
+    slot, proposer, orphaned+1, root, parent_root, state_root, graffiti, graffiti_text,
+    attestation_count, deposit_count, exit_count, withdraw_count, withdraw_amount, attester_slashing_count, 
+    proposer_slashing_count, bls_change_count, eth_transaction_count, eth_block_number, eth_block_hash, sync_participation
+FROM "blocks";
+
+-- migrate missing blocks
+
+INSERT INTO "slots" (
+    slot, proposer, status, root
+)
+SELECT slot, proposer, 0, '0x'
+FROM (
+    SELECT slot, proposer, COALESCE((
+        SELECT 
+            CASE 
+            WHEN blocks.orphaned = 1 THEN 2 
+            WHEN blocks.orphaned = 0 THEN 1 
+            ELSE 0 END
+        FROM blocks
+        WHERE blocks.slot = slot_assignments.slot 
+        AND blocks.proposer = slot_assignments.proposer
+        LIMIT 1
+    ), 0) AS block_status
+    FROM "slot_assignments"
+) AS t1
+WHERE t1.block_status = 0;
+
+
+-- +goose StatementEnd
+-- +goose Down
+-- +goose StatementBegin
+SELECT 'NOT SUPPORTED';
+-- +goose StatementEnd
diff --git a/db/schema/sqlite/20240423132923_block-status.sql b/db/schema/sqlite/20240423132923_block-status.sql
@@ -0,0 +1,111 @@
+-- +goose Up
+-- +goose StatementBegin
+
+CREATE TABLE IF NOT EXISTS slots
+(
+    slot bigint NOT NULL,
+    proposer bigint NOT NULL,
+    status smallint NOT NULL,
+    root BLOB NOT NULL,
+    parent_root BLOB NULL,
+    state_root BLOB NULL,
+    graffiti BLOB NULL,
+    graffiti_text text NULL,
+    attestation_count integer NULL DEFAULT 0,
+    deposit_count integer NULL DEFAULT 0,
+    exit_count integer NULL DEFAULT 0,
+    withdraw_count integer NULL DEFAULT 0,
+    withdraw_amount bigint NULL DEFAULT 0,
+    attester_slashing_count integer NULL DEFAULT 0,
+    proposer_slashing_count integer NULL DEFAULT 0,
+    bls_change_count integer NULL DEFAULT 0,
+    eth_transaction_count integer NULL DEFAULT 0,
+    eth_block_number bigint NULL,
+    eth_block_hash BLOB NULL,
+    eth_block_extra text NULL,
+    sync_participation real NULL DEFAULT 0,
+    CONSTRAINT slots_pkey PRIMARY KEY (slot, root)
+);
+
+CREATE INDEX IF NOT EXISTS "slots_root_idx"
+    ON "slots"
+    ("root" ASC);
+
+CREATE INDEX IF NOT EXISTS "slots_graffiti_idx"
+    ON "slots"  
+    ("graffiti_text" ASC);
+
+CREATE INDEX IF NOT EXISTS "slots_eth_block_extra_idx"
+    ON "slots"  
+    ("eth_block_extra" ASC);
+
+CREATE INDEX IF NOT EXISTS "slots_slot_idx"
+    ON "slots" 
+    ("slot" ASC);
+
+CREATE INDEX IF NOT EXISTS "slots_state_root_idx"
+    ON "slots" 
+    ("state_root" ASC );
+
+CREATE INDEX IF NOT EXISTS "slots_eth_block_number_idx"
+    ON "slots" 
+    ("eth_block_number" ASC );
+
+CREATE INDEX IF NOT EXISTS "slots_eth_block_hash_idx"
+    ON "slots" 
+    ("eth_block_hash" ASC );
+
+CREATE INDEX IF NOT EXISTS "slots_proposer_idx"
+    ON "slots" 
+    ("proposer" ASC );
+
+CREATE INDEX IF NOT EXISTS "slots_status_idx"
+    ON "slots" 
+    ("status" ASC );
+
+CREATE INDEX IF NOT EXISTS "slots_parent_root_idx"
+    ON "slots" 
+    ("parent_root" ASC );
+
+-- migrate blocks
+
+INSERT INTO "slots" (
+    slot, proposer, status, root, parent_root, state_root, graffiti, graffiti_text,
+    attestation_count, deposit_count, exit_count, withdraw_count, withdraw_amount, attester_slashing_count, 
+    proposer_slashing_count, bls_change_count, eth_transaction_count, eth_block_number, eth_block_hash, sync_participation
+)
+SELECT 
+    slot, proposer, orphaned+1, root, parent_root, state_root, graffiti, graffiti_text,
+    attestation_count, deposit_count, exit_count, withdraw_count, withdraw_amount, attester_slashing_count, 
+    proposer_slashing_count, bls_change_count, eth_transaction_count, eth_block_number, eth_block_hash, sync_participation
+FROM "blocks";
+
+-- migrate missing blocks
+
+INSERT INTO "slots" (
+    slot, proposer, status, root
+)
+SELECT slot, proposer, 0, '0x'
+FROM (
+    SELECT slot, proposer, COALESCE((
+        SELECT 
+            CASE 
+            WHEN blocks.orphaned = 1 THEN 2 
+            WHEN blocks.orphaned = 0 THEN 1 
+            ELSE 0 END
+        FROM blocks
+        WHERE blocks.slot = slot_assignments.slot 
+        AND blocks.proposer = slot_assignments.proposer
+        LIMIT 1
+    ), 0) AS block_status
+    FROM "slot_assignments"
+) AS t1
+WHERE t1.block_status = 0;
+
+
+
+-- +goose StatementEnd
+-- +goose Down
+-- +goose StatementBegin
+SELECT 'NOT SUPPORTED';
+-- +goose StatementEnd
diff --git a/dbtypes/dbtypes.go b/dbtypes/dbtypes.go
@@ -10,13 +10,25 @@ type ValidatorName struct {
 	Name  string `db:"name"`
 }
 
-type Block struct {
+type SlotStatus uint8
+
+const (
+	Missing SlotStatus = iota
+	Canonical
+	Orphaned
+)
+
+type SlotHeader struct {
+	Slot     uint64     `db:"slot"`
+	Proposer uint64     `db:"proposer"`
+	Status   SlotStatus `db:"status"`
+}
+
+type Slot struct {
+	SlotHeader
 	Root                  []byte  `db:"root"`
-	Slot                  uint64  `db:"slot"`
 	ParentRoot            []byte  `db:"parent_root"`
 	StateRoot             []byte  `db:"state_root"`
-	Orphaned              uint8   `db:"orphaned"`
-	Proposer              uint64  `db:"proposer"`
 	Graffiti              []byte  `db:"graffiti"`
 	GraffitiText          string  `db:"graffiti_text"`
 	AttestationCount      uint64  `db:"attestation_count"`
@@ -30,14 +42,10 @@ type Block struct {
 	EthTransactionCount   uint64  `db:"eth_transaction_count"`
 	EthBlockNumber        *uint64 `db:"eth_block_number"`
 	EthBlockHash          []byte  `db:"eth_block_hash"`
+	EthBlockExtra         []byte  `db:"eth_block_extra"`
 	SyncParticipation     float32 `db:"sync_participation"`
 }
 
-type BlockOrphanedRef struct {
-	Root     []byte `db:"root"`
-	Orphaned bool   `db:"orphaned"`
-}
-
 type Epoch struct {
 	Epoch                 uint64  `db:"epoch"`
 	ValidatorCount        uint64  `db:"validator_count"`

diff --git a/dbtypes/other.go b/dbtypes/other.go
@@ -1,9 +1,14 @@
 package dbtypes
 
-type AssignedBlock struct {
+type AssignedSlot struct {
 	Slot     uint64 `db:"slot"`
 	Proposer uint64 `db:"proposer"`
-	Block    *Block `db:"block"`
+	Block    *Slot  `db:"block"`
+}
+
+type BlockStatus struct {
+	Root   []byte `db:"root"`
+	Status bool   `db:"status"`
 }
 
 type AssignedBlob struct {
@@ -15,6 +20,7 @@ type AssignedBlob struct {
 
 type BlockFilter struct {
 	Graffiti      string
+	ExtraData     string
 	ProposerIndex *uint64
 	ProposerName  string
 	WithOrphaned  uint8

diff --git a/handlers/epoch.go b/handlers/epoch.go
@@ -90,7 +90,6 @@ func buildEpochPageData(epoch uint64) (*models.EpochPageData, time.Duration) {
 	}
 
 	finalizedEpoch, _ := services.GlobalBeaconService.GetFinalizedEpoch()
-	slotAssignments, syncedEpochs := services.GlobalBeaconService.GetProposerAssignments(epoch, epoch)
 
 	nextEpoch := epoch + 1
 	if nextEpoch > currentEpoch {
@@ -103,13 +102,14 @@ func buildEpochPageData(epoch uint64) (*models.EpochPageData, time.Duration) {
 		PreviousEpoch: epoch - 1,
 		NextEpoch:     nextEpoch,
 		Ts:            utils.EpochToTime(epoch),
-		Synchronized:  syncedEpochs[epoch],
+		Synchronized:  true,
 		Finalized:     finalizedEpoch >= int64(epoch),
 	}
 
 	dbEpochs := services.GlobalBeaconService.GetDbEpochs(epoch, 1)
 	dbEpoch := dbEpochs[0]
 	if dbEpoch != nil {
+		pageData.Synchronized = true
 		pageData.AttestationCount = dbEpoch.AttestationCount
 		pageData.DepositCount = dbEpoch.DepositCount
 		pageData.ExitCount = dbEpoch.ExitCount
@@ -135,19 +135,16 @@ func buildEpochPageData(epoch uint64) (*models.EpochPageData, time.Duration) {
 
 	// load slots
 	pageData.Slots = make([]*models.EpochPageDataSlot, 0)
-	dbSlots := services.GlobalBeaconService.GetDbBlocksForSlots(uint64(lastSlot), uint32(utils.Config.Chain.Config.SlotsPerEpoch), true)
+	dbSlots := services.GlobalBeaconService.GetDbBlocksForSlots(uint64(lastSlot), uint32(utils.Config.Chain.Config.SlotsPerEpoch), true, true)
 	dbIdx := 0
 	dbCnt := len(dbSlots)
 	blockCount := uint64(0)
 	for slotIdx := int64(lastSlot); slotIdx >= int64(firstSlot); slotIdx-- {
 		slot := uint64(slotIdx)
-		haveBlock := false
 		for dbIdx < dbCnt && dbSlots[dbIdx] != nil && dbSlots[dbIdx].Slot == slot {
 			dbSlot := dbSlots[dbIdx]
 			dbIdx++
-			blockStatus := uint8(1)
-			if dbSlot.Orphaned == 1 {
-				blockStatus = 2
+			if dbSlot.Status == 2 {
 				pageData.OrphanedCount++
 			} else {
 				pageData.CanonicalCount++
@@ -157,7 +154,7 @@ func buildEpochPageData(epoch uint64) (*models.EpochPageData, time.Duration) {
 				Slot:                  slot,
 				Epoch:                 utils.EpochOfSlot(slot),
 				Ts:                    utils.SlotToTime(slot),
-				Status:                blockStatus,
+				Status:                uint8(dbSlot.Status),
 				Proposer:              dbSlot.Proposer,
 				ProposerName:          services.GlobalBeaconService.GetValidatorName(dbSlot.Proposer),
 				AttestationCount:      dbSlot.AttestationCount,
@@ -176,26 +173,6 @@ func buildEpochPageData(epoch uint64) (*models.EpochPageData, time.Duration) {
 			}
 			pageData.Slots = append(pageData.Slots, slotData)
 			blockCount++
-			haveBlock = true
-		}
-
-		if !haveBlock {
-			slotData := &models.EpochPageDataSlot{
-				Slot:         slot,
-				Epoch:        epoch,
-				Ts:           utils.SlotToTime(slot),
-				Scheduled:    slot >= currentSlot,
-				Status:       0,
-				Proposer:     slotAssignments[slot],
-				ProposerName: services.GlobalBeaconService.GetValidatorName(slotAssignments[slot]),
-			}
-			if slotData.Scheduled {
-				pageData.ScheduledCount++
-			} else {
-				pageData.MissedCount++
-			}
-			pageData.Slots = append(pageData.Slots, slotData)
-			blockCount++
 		}
 	}
 	pageData.BlockCount = uint64(blockCount)

diff --git a/handlers/forks.go b/handlers/forks.go
@@ -6,6 +6,7 @@ import (
 	"time"
 
 	"github.com/ethpandaops/dora/db"
+	"github.com/ethpandaops/dora/dbtypes"
 	"github.com/ethpandaops/dora/services"
 	"github.com/ethpandaops/dora/templates"
 	"github.com/ethpandaops/dora/types/models"
@@ -70,8 +71,8 @@ func buildForksPageData() (*models.ForksPageData, time.Duration) {
 		}
 		if int64(fork.Slot) < finalizedEpoch*int64(utils.Config.Chain.Config.SlotsPerEpoch) {
 			// check block
-			dbBlock := db.GetBlockByRoot(fork.Root)
-			if dbBlock != nil && dbBlock.Orphaned == 0 {
+			dbBlock := db.GetSlotByRoot(fork.Root)
+			if dbBlock != nil && dbBlock.Status == dbtypes.Canonical {
 				headForks[0].AllClients = append(headForks[0].AllClients, fork.AllClients...)
 				headForks[idx] = nil
 			}