From 29f6f083797607db844891bf161b2aebdd2d5661 Mon Sep 17 00:00:00 2001 From: pk910 Date: Wed, 6 Sep 2023 21:15:11 +0200 Subject: [PATCH] fixed processing for pre-bellatrix blocks (missing execution payload) and ensure payloads for post-bellatrix blocks --- .../20230906202015_nullable-el-blocks.sql | 17 ++++ .../20230906202015_nullable-el-blocks.sql | 77 +++++++++++++++++++ dbtypes/dbtypes.go | 2 +- handlers/epoch.go | 5 +- handlers/index.go | 10 ++- handlers/slots.go | 5 +- handlers/slots_filtered.go | 5 +- handlers/validator.go | 5 +- handlers/validator_slots.go | 5 +- indexer/client.go | 4 + indexer/synchronizer.go | 3 + indexer/write_db.go | 3 +- templates/index/recentBlocks.html | 2 +- types/models/epoch.go | 1 + types/models/indexPage.go | 1 + types/models/slots.go | 1 + types/models/slots_filtered.go | 1 + types/models/validator.go | 15 ++-- types/models/validator_slots.go | 1 + 19 files changed, 145 insertions(+), 18 deletions(-) create mode 100644 db/schema/pgsql/20230906202015_nullable-el-blocks.sql create mode 100644 db/schema/sqlite/20230906202015_nullable-el-blocks.sql diff --git a/db/schema/pgsql/20230906202015_nullable-el-blocks.sql b/db/schema/pgsql/20230906202015_nullable-el-blocks.sql new file mode 100644 index 00000000..d23aa95e --- /dev/null +++ b/db/schema/pgsql/20230906202015_nullable-el-blocks.sql @@ -0,0 +1,17 @@ +-- +goose Up +-- +goose StatementBegin + +ALTER TABLE IF EXISTS public."blocks" + ALTER COLUMN "eth_block_number" DROP DEFAULT; + +ALTER TABLE IF EXISTS public."blocks" + ALTER COLUMN "eth_block_number" DROP NOT NULL; + +ALTER TABLE IF EXISTS public."blocks" + ALTER COLUMN "eth_block_hash" DROP NOT NULL; + +-- +goose StatementEnd +-- +goose Down +-- +goose StatementBegin +SELECT 'NOT SUPPORTED'; +-- +goose StatementEnd diff --git a/db/schema/sqlite/20230906202015_nullable-el-blocks.sql b/db/schema/sqlite/20230906202015_nullable-el-blocks.sql new file mode 100644 index 00000000..0bf81e94 --- /dev/null +++ b/db/schema/sqlite/20230906202015_nullable-el-blocks.sql @@ -0,0 +1,77 @@ +-- +goose Up +-- +goose StatementBegin + +CREATE TABLE IF NOT EXISTS "new_blocks" +( + "root" BLOB NOT NULL UNIQUE, + "slot" BIGINT NOT NULL, + "parent_root" BLOB NOT NULL, + "state_root" BLOB NOT NULL, + "orphaned" INTEGER NOT NULL, + "proposer" BIGINT NOT NULL, + "graffiti" BLOB NOT NULL, + "graffiti_text" TEXT NULL, + "attestation_count" INTEGER NOT NULL DEFAULT 0, + "deposit_count" INTEGER NOT NULL DEFAULT 0, + "exit_count" INTEGER NOT NULL DEFAULT 0, + "withdraw_count" INTEGER NOT NULL DEFAULT 0, + "withdraw_amount" BIGINT NOT NULL DEFAULT 0, + "attester_slashing_count" INTEGER NOT NULL DEFAULT 0, + "proposer_slashing_count" INTEGER NOT NULL DEFAULT 0, + "bls_change_count" INTEGER NOT NULL DEFAULT 0, + "eth_transaction_count" INTEGER NOT NULL DEFAULT 0, + "eth_block_number" BIGINT NULL, + "eth_block_hash" BLOB NULL, + "sync_participation" REAL NOT NULL DEFAULT 0, + CONSTRAINT "blocks_pkey" PRIMARY KEY ("root") +); + +INSERT INTO "new_blocks" ( + root, slot, parent_root, state_root, orphaned, proposer, graffiti, graffiti_text, + attestation_count, deposit_count, exit_count, withdraw_count, withdraw_amount, attester_slashing_count, + proposer_slashing_count, bls_change_count, eth_transaction_count, eth_block_number, eth_block_hash, sync_participation +) +SELECT + root, slot, parent_root, state_root, orphaned, proposer, graffiti, graffiti_text, + attestation_count, deposit_count, exit_count, withdraw_count, withdraw_amount, attester_slashing_count, + proposer_slashing_count, bls_change_count, eth_transaction_count, eth_block_number, eth_block_hash, sync_participation +FROM "blocks"; + + +DROP TABLE "blocks"; +ALTER TABLE "new_blocks" RENAME TO "blocks"; + + +CREATE INDEX IF NOT EXISTS "blocks_graffiti_idx" + ON "blocks" + ("graffiti_text" ASC); + +CREATE INDEX IF NOT EXISTS "blocks_slot_idx" + ON "blocks" + ("slot" ASC); + +CREATE INDEX IF NOT EXISTS "blocks_state_root_idx" + ON "blocks" + ("state_root" ASC); + +CREATE INDEX IF NOT EXISTS "blocks_eth_block_number_idx" + ON "blocks" + ("eth_block_number" ASC); + +CREATE INDEX IF NOT EXISTS "blocks_eth_block_hash_idx" + ON "blocks" + ("eth_block_hash" ASC); + +CREATE INDEX IF NOT EXISTS "blocks_proposer_idx" + ON "blocks" + ("proposer" ASC); + +CREATE INDEX IF NOT EXISTS "blocks_parent_root_idx" + ON "blocks" + ("parent_root" ASC); + +-- +goose StatementEnd +-- +goose Down +-- +goose StatementBegin +SELECT 'NOT SUPPORTED'; +-- +goose StatementEnd diff --git a/dbtypes/dbtypes.go b/dbtypes/dbtypes.go index 7f0c5678..de4aaa68 100644 --- a/dbtypes/dbtypes.go +++ b/dbtypes/dbtypes.go @@ -28,7 +28,7 @@ type Block struct { ProposerSlashingCount uint64 `db:"proposer_slashing_count"` BLSChangeCount uint64 `db:"bls_change_count"` EthTransactionCount uint64 `db:"eth_transaction_count"` - EthBlockNumber uint64 `db:"eth_block_number"` + EthBlockNumber *uint64 `db:"eth_block_number"` EthBlockHash []byte `db:"eth_block_hash"` SyncParticipation float32 `db:"sync_participation"` } diff --git a/handlers/epoch.go b/handlers/epoch.go index 8f11e6e8..c02232a2 100644 --- a/handlers/epoch.go +++ b/handlers/epoch.go @@ -153,10 +153,13 @@ func buildEpochPageData(epoch uint64) (*models.EpochPageData, time.Duration) { AttesterSlashingCount: dbSlot.AttesterSlashingCount, SyncParticipation: float64(dbSlot.SyncParticipation) * 100, EthTransactionCount: dbSlot.EthTransactionCount, - EthBlockNumber: dbSlot.EthBlockNumber, Graffiti: dbSlot.Graffiti, BlockRoot: dbSlot.Root, } + if dbSlot.EthBlockNumber != nil { + slotData.WithEthBlock = true + slotData.EthBlockNumber = *dbSlot.EthBlockNumber + } pageData.Slots = append(pageData.Slots, slotData) blockCount++ haveBlock = true diff --git a/handlers/index.go b/handlers/index.go index 9d8ca650..14343dd5 100644 --- a/handlers/index.go +++ b/handlers/index.go @@ -211,16 +211,20 @@ func buildIndexPageRecentBlocksData(pageData *models.IndexPageData, currentSlot if blockData.Orphaned == 1 { blockStatus = 2 } - pageData.RecentBlocks = append(pageData.RecentBlocks, &models.IndexPageDataBlocks{ + blockModel := &models.IndexPageDataBlocks{ Epoch: utils.EpochOfSlot(blockData.Slot), Slot: blockData.Slot, - EthBlock: blockData.EthBlockNumber, Ts: utils.SlotToTime(blockData.Slot), Proposer: blockData.Proposer, ProposerName: services.GlobalBeaconService.GetValidatorName(blockData.Proposer), Status: uint64(blockStatus), BlockRoot: blockData.Root, - }) + } + if blockData.EthBlockNumber != nil { + blockModel.WithEthBlock = true + blockModel.EthBlock = *blockData.EthBlockNumber + } + pageData.RecentBlocks = append(pageData.RecentBlocks, blockModel) } pageData.RecentBlockCount = uint64(len(pageData.RecentBlocks)) } diff --git a/handlers/slots.go b/handlers/slots.go index 3cf7a17c..4e2fa197 100644 --- a/handlers/slots.go +++ b/handlers/slots.go @@ -149,12 +149,15 @@ func buildSlotsPageData(firstSlot uint64, pageSize uint64) (*models.SlotsPageDat AttesterSlashingCount: dbSlot.AttesterSlashingCount, SyncParticipation: float64(dbSlot.SyncParticipation) * 100, EthTransactionCount: dbSlot.EthTransactionCount, - EthBlockNumber: dbSlot.EthBlockNumber, Graffiti: dbSlot.Graffiti, BlockRoot: dbSlot.Root, ParentRoot: dbSlot.ParentRoot, ForkGraph: make([]*models.SlotsPageDataForkGraph, 0), } + if dbSlot.EthBlockNumber != nil { + slotData.WithEthBlock = true + slotData.EthBlockNumber = *dbSlot.EthBlockNumber + } pageData.Slots = append(pageData.Slots, slotData) blockCount++ haveBlock = true diff --git a/handlers/slots_filtered.go b/handlers/slots_filtered.go index 12afd611..0bb54127 100644 --- a/handlers/slots_filtered.go +++ b/handlers/slots_filtered.go @@ -170,9 +170,12 @@ func buildFilteredSlotsPageData(pageIdx uint64, pageSize uint64, graffiti string slotData.AttesterSlashingCount = dbBlock.Block.AttesterSlashingCount slotData.SyncParticipation = float64(dbBlock.Block.SyncParticipation) * 100 slotData.EthTransactionCount = dbBlock.Block.EthTransactionCount - slotData.EthBlockNumber = dbBlock.Block.EthBlockNumber slotData.Graffiti = dbBlock.Block.Graffiti slotData.BlockRoot = dbBlock.Block.Root + if dbBlock.Block.EthBlockNumber != nil { + slotData.WithEthBlock = true + slotData.EthBlockNumber = *dbBlock.Block.EthBlockNumber + } } pageData.Slots = append(pageData.Slots, slotData) } diff --git a/handlers/validator.go b/handlers/validator.go index 51276965..a1ca42ed 100644 --- a/handlers/validator.go +++ b/handlers/validator.go @@ -170,8 +170,11 @@ func buildValidatorPageData(validatorIndex uint64) (*models.ValidatorPageData, t } if blockData.Block != nil { blockEntry.Graffiti = blockData.Block.Graffiti - blockEntry.EthBlock = blockData.Block.EthBlockNumber blockEntry.BlockRoot = fmt.Sprintf("0x%x", blockData.Block.Root) + if blockData.Block.EthBlockNumber != nil { + blockEntry.WithEthBlock = true + blockEntry.EthBlock = *blockData.Block.EthBlockNumber + } } pageData.RecentBlocks = append(pageData.RecentBlocks, &blockEntry) } diff --git a/handlers/validator_slots.go b/handlers/validator_slots.go index d5b24c22..c01cb042 100644 --- a/handlers/validator_slots.go +++ b/handlers/validator_slots.go @@ -127,9 +127,12 @@ func buildValidatorSlotsPageData(validator uint64, pageIdx uint64, pageSize uint slotData.AttesterSlashingCount = dbBlock.AttesterSlashingCount slotData.SyncParticipation = float64(dbBlock.SyncParticipation) * 100 slotData.EthTransactionCount = dbBlock.EthTransactionCount - slotData.EthBlockNumber = dbBlock.EthBlockNumber slotData.Graffiti = dbBlock.Graffiti slotData.BlockRoot = dbBlock.Root + if dbBlock.EthBlockNumber != nil { + slotData.WithEthBlock = true + slotData.EthBlockNumber = *dbBlock.EthBlockNumber + } } pageData.Slots = append(pageData.Slots, slotData) } diff --git a/indexer/client.go b/indexer/client.go index 44285e12..1484178a 100644 --- a/indexer/client.go +++ b/indexer/client.go @@ -360,6 +360,10 @@ func (client *IndexerClient) ensureBlock(block *CacheBlock, header *rpctypes.Sig logger.WithField("client", client.clientName).Warnf("ensure block %v [0x%x] failed (block): %v", block.Slot, block.Root, err) return err } + if utils.EpochOfSlot(block.Slot) >= utils.Config.Chain.Config.BellatrixForkEpoch && blockRsp.Data.Message.Body.ExecutionPayload == nil { + logger.WithField("client", client.clientName).Warnf("ensure block %v [0x%x] failed (block): execution payload missing for post-bellatix block", block.Slot, block.Root) + return err + } block.block = &blockRsp.Data } // set seen flag diff --git a/indexer/synchronizer.go b/indexer/synchronizer.go index f679e510..2c6f9f17 100644 --- a/indexer/synchronizer.go +++ b/indexer/synchronizer.go @@ -177,6 +177,9 @@ func (sync *synchronizerState) syncEpoch(syncEpoch uint64, lastTry bool, skipCli if err != nil { return false, client, fmt.Errorf("error fetching slot %v block: %v", slot, err) } + if utils.EpochOfSlot(slot) >= utils.Config.Chain.Config.BellatrixForkEpoch && blockRsp.Data.Message.Body.ExecutionPayload == nil { + return false, client, fmt.Errorf("error fetching slot %v block: execution payload missing for post-bellatix block", slot) + } sync.cachedBlocks[slot] = &CacheBlock{ Root: headerRsp.Data.Root, Slot: slot, diff --git a/indexer/write_db.go b/indexer/write_db.go index b92ebee9..abeeae3d 100644 --- a/indexer/write_db.go +++ b/indexer/write_db.go @@ -121,7 +121,8 @@ func buildDbBlock(block *CacheBlock, epochStats *EpochStats) *dbtypes.Block { if executionPayload := blockBody.Message.Body.ExecutionPayload; executionPayload != nil { dbBlock.EthTransactionCount = uint64(len(executionPayload.Transactions)) - dbBlock.EthBlockNumber = uint64(executionPayload.BlockNumber) + blockNumber := uint64(executionPayload.BlockNumber) + dbBlock.EthBlockNumber = &blockNumber dbBlock.EthBlockHash = executionPayload.BlockHash if executionPayload.Withdrawals != nil { diff --git a/templates/index/recentBlocks.html b/templates/index/recentBlocks.html index e442d9f0..6eb38624 100644 --- a/templates/index/recentBlocks.html +++ b/templates/index/recentBlocks.html @@ -29,7 +29,7 @@
{{ formatAddCommas $block.Slot }} {{ end }} - {{ ethBlockLink $block.EthBlock }} + {{ if $block.WithEthBlock }}{{ ethBlockLink $block.EthBlock }}{{ else }}-{{ end }} {{ if eq $block.Slot 0 }} Genesis diff --git a/types/models/epoch.go b/types/models/epoch.go index ba2f4c1f..e7a8d263 100644 --- a/types/models/epoch.go +++ b/types/models/epoch.go @@ -54,6 +54,7 @@ type EpochPageDataSlot struct { SyncParticipation float64 `json:"sync_participation"` EthTransactionCount uint64 `json:"eth_transaction_count"` EthBlockNumber uint64 `json:"eth_block_number"` + WithEthBlock bool `json:"with_eth_block"` Graffiti []byte `json:"graffiti"` BlockRoot []byte `json:"block_root"` } diff --git a/types/models/indexPage.go b/types/models/indexPage.go index 52246f41..fcaa3d3d 100644 --- a/types/models/indexPage.go +++ b/types/models/indexPage.go @@ -58,6 +58,7 @@ type IndexPageDataEpochs struct { type IndexPageDataBlocks struct { Epoch uint64 `json:"epoch"` Slot uint64 `json:"slot"` + WithEthBlock bool `json:"with_eth_block"` EthBlock uint64 `json:"eth_block"` Ts time.Time `json:"ts"` Proposer uint64 `json:"proposer"` diff --git a/types/models/slots.go b/types/models/slots.go index 7774b589..075277e3 100644 --- a/types/models/slots.go +++ b/types/models/slots.go @@ -41,6 +41,7 @@ type SlotsPageDataSlot struct { AttesterSlashingCount uint64 `json:"attester_slashing_count"` SyncParticipation float64 `json:"sync_participation"` EthTransactionCount uint64 `json:"eth_transaction_count"` + WithEthBlock bool `json:"with_eth_block"` EthBlockNumber uint64 `json:"eth_block_number"` Graffiti []byte `json:"graffiti"` BlockRoot []byte `json:"block_root"` diff --git a/types/models/slots_filtered.go b/types/models/slots_filtered.go index 50b8af44..a0c2b3e2 100644 --- a/types/models/slots_filtered.go +++ b/types/models/slots_filtered.go @@ -51,6 +51,7 @@ type SlotsFilteredPageDataSlot struct { AttesterSlashingCount uint64 `json:"attester_slashing_count"` SyncParticipation float64 `json:"sync_participation"` EthTransactionCount uint64 `json:"eth_transaction_count"` + WithEthBlock bool `json:"with_eth_block"` EthBlockNumber uint64 `json:"eth_block_number"` Graffiti []byte `json:"graffiti"` BlockRoot []byte `json:"block_root"` diff --git a/types/models/validator.go b/types/models/validator.go index b85955f5..f6c11a52 100644 --- a/types/models/validator.go +++ b/types/models/validator.go @@ -36,11 +36,12 @@ type ValidatorPageData struct { } type ValidatorPageDataBlocks struct { - Epoch uint64 `json:"epoch"` - Slot uint64 `json:"slot"` - EthBlock uint64 `json:"eth_block"` - Ts time.Time `json:"ts"` - Status uint64 `json:"status"` - BlockRoot string `json:"block_root"` - Graffiti []byte `json:"graffiti"` + Epoch uint64 `json:"epoch"` + Slot uint64 `json:"slot"` + WithEthBlock bool `json:"with_eth_block"` + EthBlock uint64 `json:"eth_block"` + Ts time.Time `json:"ts"` + Status uint64 `json:"status"` + BlockRoot string `json:"block_root"` + Graffiti []byte `json:"graffiti"` } diff --git a/types/models/validator_slots.go b/types/models/validator_slots.go index c65b561c..22b33e1e 100644 --- a/types/models/validator_slots.go +++ b/types/models/validator_slots.go @@ -43,6 +43,7 @@ type ValidatorSlotsPageDataSlot struct { AttesterSlashingCount uint64 `json:"attester_slashing_count"` SyncParticipation float64 `json:"sync_participation"` EthTransactionCount uint64 `json:"eth_transaction_count"` + WithEthBlock bool `json:"with_eth_block"` EthBlockNumber uint64 `json:"eth_block_number"` Graffiti []byte `json:"graffiti"` BlockRoot []byte `json:"block_root"`