fixed processing for pre-bellatrix blocks (missing execution payload)…

… and ensure payloads for post-bellatrix blocks

db/schema/pgsql/20230906202015_nullable-el-blocks.sql

@@ -0,0 +1,17 @@
+-- +goose Up
+-- +goose StatementBegin
+
+ALTER TABLE IF EXISTS public."blocks"
+    ALTER COLUMN "eth_block_number" DROP DEFAULT;
+
+ALTER TABLE IF EXISTS public."blocks"
+    ALTER COLUMN "eth_block_number" DROP NOT NULL;
+
+ALTER TABLE IF EXISTS public."blocks"
+    ALTER COLUMN "eth_block_hash" DROP NOT NULL;
+
+-- +goose StatementEnd
+-- +goose Down
+-- +goose StatementBegin
+SELECT 'NOT SUPPORTED';
+-- +goose StatementEnd

db/schema/sqlite/20230906202015_nullable-el-blocks.sql

@@ -0,0 +1,77 @@
+-- +goose Up
+-- +goose StatementBegin
+
+CREATE TABLE IF NOT EXISTS "new_blocks"
+(
+    "root" BLOB NOT NULL UNIQUE,
+    "slot" BIGINT NOT NULL,
+    "parent_root" BLOB NOT NULL,
+    "state_root" BLOB NOT NULL,
+    "orphaned" INTEGER NOT NULL,
+    "proposer" BIGINT NOT NULL,
+    "graffiti" BLOB NOT NULL,
+    "graffiti_text" TEXT NULL,
+    "attestation_count" INTEGER NOT NULL DEFAULT 0,
+    "deposit_count" INTEGER NOT NULL DEFAULT 0,
+    "exit_count" INTEGER NOT NULL DEFAULT 0,
+    "withdraw_count" INTEGER NOT NULL DEFAULT 0,
+    "withdraw_amount" BIGINT NOT NULL DEFAULT 0,
+    "attester_slashing_count" INTEGER NOT NULL DEFAULT 0,
+    "proposer_slashing_count" INTEGER NOT NULL DEFAULT 0,
+    "bls_change_count" INTEGER NOT NULL DEFAULT 0,
+    "eth_transaction_count" INTEGER NOT NULL DEFAULT 0,
+    "eth_block_number" BIGINT NULL,
+    "eth_block_hash" BLOB NULL,
+    "sync_participation" REAL NOT NULL DEFAULT 0,
+    CONSTRAINT "blocks_pkey" PRIMARY KEY ("root")
+);
+
+INSERT INTO "new_blocks" (
+    root, slot, parent_root, state_root, orphaned, proposer, graffiti, graffiti_text,
+    attestation_count, deposit_count, exit_count, withdraw_count, withdraw_amount, attester_slashing_count, 
+    proposer_slashing_count, bls_change_count, eth_transaction_count, eth_block_number, eth_block_hash, sync_participation
+)
+SELECT 
+    root, slot, parent_root, state_root, orphaned, proposer, graffiti, graffiti_text,
+    attestation_count, deposit_count, exit_count, withdraw_count, withdraw_amount, attester_slashing_count, 
+    proposer_slashing_count, bls_change_count, eth_transaction_count, eth_block_number, eth_block_hash, sync_participation
+FROM "blocks";
+
+
+DROP TABLE "blocks";
+ALTER TABLE "new_blocks" RENAME TO "blocks"; 
+
+
+CREATE INDEX IF NOT EXISTS "blocks_graffiti_idx"
+    ON "blocks"
+    ("graffiti_text" ASC);
+
+CREATE INDEX IF NOT EXISTS "blocks_slot_idx"
+    ON "blocks" 
+    ("slot" ASC);
+
+CREATE INDEX IF NOT EXISTS "blocks_state_root_idx"
+    ON "blocks" 
+    ("state_root" ASC);
+
+CREATE INDEX IF NOT EXISTS "blocks_eth_block_number_idx"
+    ON "blocks" 
+    ("eth_block_number" ASC);
+
+CREATE INDEX IF NOT EXISTS "blocks_eth_block_hash_idx"
+    ON "blocks" 
+    ("eth_block_hash" ASC);
+
+CREATE INDEX IF NOT EXISTS "blocks_proposer_idx"
+    ON "blocks" 
+    ("proposer" ASC);
+
+CREATE INDEX IF NOT EXISTS "blocks_parent_root_idx"
+    ON "blocks" 
+    ("parent_root" ASC);
+
+-- +goose StatementEnd
+-- +goose Down
+-- +goose StatementBegin
+SELECT 'NOT SUPPORTED';
+-- +goose StatementEnd

dbtypes/dbtypes.go

@@ -28,7 +28,7 @@ type Block struct {
 	ProposerSlashingCount uint64  `db:"proposer_slashing_count"`
 	BLSChangeCount        uint64  `db:"bls_change_count"`
 	EthTransactionCount   uint64  `db:"eth_transaction_count"`
-	EthBlockNumber        uint64  `db:"eth_block_number"`
+	EthBlockNumber        *uint64 `db:"eth_block_number"`
 	EthBlockHash          []byte  `db:"eth_block_hash"`
 	SyncParticipation     float32 `db:"sync_participation"`
 }

handlers/epoch.go

@@ -153,10 +153,13 @@ func buildEpochPageData(epoch uint64) (*models.EpochPageData, time.Duration) {
 				AttesterSlashingCount: dbSlot.AttesterSlashingCount,
 				SyncParticipation:     float64(dbSlot.SyncParticipation) * 100,
 				EthTransactionCount:   dbSlot.EthTransactionCount,
-				EthBlockNumber:        dbSlot.EthBlockNumber,
 				Graffiti:              dbSlot.Graffiti,
 				BlockRoot:             dbSlot.Root,
 			}
+			if dbSlot.EthBlockNumber != nil {
+				slotData.WithEthBlock = true
+				slotData.EthBlockNumber = *dbSlot.EthBlockNumber
+			}
 			pageData.Slots = append(pageData.Slots, slotData)
 			blockCount++
 			haveBlock = true

handlers/index.go

@@ -211,16 +211,20 @@ func buildIndexPageRecentBlocksData(pageData *models.IndexPageData, currentSlot
 		if blockData.Orphaned == 1 {
 			blockStatus = 2
 		}
-		pageData.RecentBlocks = append(pageData.RecentBlocks, &models.IndexPageDataBlocks{
+		blockModel := &models.IndexPageDataBlocks{
 			Epoch:        utils.EpochOfSlot(blockData.Slot),
 			Slot:         blockData.Slot,
-			EthBlock:     blockData.EthBlockNumber,
 			Ts:           utils.SlotToTime(blockData.Slot),
 			Proposer:     blockData.Proposer,
 			ProposerName: services.GlobalBeaconService.GetValidatorName(blockData.Proposer),
 			Status:       uint64(blockStatus),
 			BlockRoot:    blockData.Root,
-		})
+		}
+		if blockData.EthBlockNumber != nil {
+			blockModel.WithEthBlock = true
+			blockModel.EthBlock = *blockData.EthBlockNumber
+		}
+		pageData.RecentBlocks = append(pageData.RecentBlocks, blockModel)
 	}
 	pageData.RecentBlockCount = uint64(len(pageData.RecentBlocks))
 }

handlers/slots.go

@@ -149,12 +149,15 @@ func buildSlotsPageData(firstSlot uint64, pageSize uint64) (*models.SlotsPageDat
 				AttesterSlashingCount: dbSlot.AttesterSlashingCount,
 				SyncParticipation:     float64(dbSlot.SyncParticipation) * 100,
 				EthTransactionCount:   dbSlot.EthTransactionCount,
-				EthBlockNumber:        dbSlot.EthBlockNumber,
 				Graffiti:              dbSlot.Graffiti,
 				BlockRoot:             dbSlot.Root,
 				ParentRoot:            dbSlot.ParentRoot,
 				ForkGraph:             make([]*models.SlotsPageDataForkGraph, 0),
 			}
+			if dbSlot.EthBlockNumber != nil {
+				slotData.WithEthBlock = true
+				slotData.EthBlockNumber = *dbSlot.EthBlockNumber
+			}
 			pageData.Slots = append(pageData.Slots, slotData)
 			blockCount++
 			haveBlock = true

handlers/slots_filtered.go

@@ -170,9 +170,12 @@ func buildFilteredSlotsPageData(pageIdx uint64, pageSize uint64, graffiti string
 			slotData.AttesterSlashingCount = dbBlock.Block.AttesterSlashingCount
 			slotData.SyncParticipation = float64(dbBlock.Block.SyncParticipation) * 100
 			slotData.EthTransactionCount = dbBlock.Block.EthTransactionCount
-			slotData.EthBlockNumber = dbBlock.Block.EthBlockNumber
 			slotData.Graffiti = dbBlock.Block.Graffiti
 			slotData.BlockRoot = dbBlock.Block.Root
+			if dbBlock.Block.EthBlockNumber != nil {
+				slotData.WithEthBlock = true
+				slotData.EthBlockNumber = *dbBlock.Block.EthBlockNumber
+			}
 		}
 		pageData.Slots = append(pageData.Slots, slotData)
 	}

handlers/validator.go

@@ -170,8 +170,11 @@ func buildValidatorPageData(validatorIndex uint64) (*models.ValidatorPageData, t
 		}
 		if blockData.Block != nil {
 			blockEntry.Graffiti = blockData.Block.Graffiti
-			blockEntry.EthBlock = blockData.Block.EthBlockNumber
 			blockEntry.BlockRoot = fmt.Sprintf("0x%x", blockData.Block.Root)
+			if blockData.Block.EthBlockNumber != nil {
+				blockEntry.WithEthBlock = true
+				blockEntry.EthBlock = *blockData.Block.EthBlockNumber
+			}
 		}
 		pageData.RecentBlocks = append(pageData.RecentBlocks, &blockEntry)
 	}

handlers/validator_slots.go

@@ -127,9 +127,12 @@ func buildValidatorSlotsPageData(validator uint64, pageIdx uint64, pageSize uint
 			slotData.AttesterSlashingCount = dbBlock.AttesterSlashingCount
 			slotData.SyncParticipation = float64(dbBlock.SyncParticipation) * 100
 			slotData.EthTransactionCount = dbBlock.EthTransactionCount
-			slotData.EthBlockNumber = dbBlock.EthBlockNumber
 			slotData.Graffiti = dbBlock.Graffiti
 			slotData.BlockRoot = dbBlock.Root
+			if dbBlock.EthBlockNumber != nil {
+				slotData.WithEthBlock = true
+				slotData.EthBlockNumber = *dbBlock.EthBlockNumber
+			}
 		}
 		pageData.Slots = append(pageData.Slots, slotData)
 	}

indexer/client.go

@@ -360,6 +360,10 @@ func (client *IndexerClient) ensureBlock(block *CacheBlock, header *rpctypes.Sig
 			logger.WithField("client", client.clientName).Warnf("ensure block %v [0x%x] failed (block): %v", block.Slot, block.Root, err)
 			return err
 		}
+		if utils.EpochOfSlot(block.Slot) >= utils.Config.Chain.Config.BellatrixForkEpoch && blockRsp.Data.Message.Body.ExecutionPayload == nil {
+			logger.WithField("client", client.clientName).Warnf("ensure block %v [0x%x] failed (block): execution payload missing for post-bellatix block", block.Slot, block.Root)
+			return err
+		}
 		block.block = &blockRsp.Data
 	}
 	// set seen flag

indexer/synchronizer.go

@@ -177,6 +177,9 @@ func (sync *synchronizerState) syncEpoch(syncEpoch uint64, lastTry bool, skipCli
 			if err != nil {
 				return false, client, fmt.Errorf("error fetching slot %v block: %v", slot, err)
 			}
+			if utils.EpochOfSlot(slot) >= utils.Config.Chain.Config.BellatrixForkEpoch && blockRsp.Data.Message.Body.ExecutionPayload == nil {
+				return false, client, fmt.Errorf("error fetching slot %v block: execution payload missing for post-bellatix block", slot)
+			}
 			sync.cachedBlocks[slot] = &CacheBlock{
 				Root:   headerRsp.Data.Root,
 				Slot:   slot,

indexer/write_db.go

@@ -121,7 +121,8 @@ func buildDbBlock(block *CacheBlock, epochStats *EpochStats) *dbtypes.Block {
 
 	if executionPayload := blockBody.Message.Body.ExecutionPayload; executionPayload != nil {
 		dbBlock.EthTransactionCount = uint64(len(executionPayload.Transactions))
-		dbBlock.EthBlockNumber = uint64(executionPayload.BlockNumber)
+		blockNumber := uint64(executionPayload.BlockNumber)
+		dbBlock.EthBlockNumber = &blockNumber
 		dbBlock.EthBlockHash = executionPayload.BlockHash
 
 		if executionPayload.Withdrawals != nil {

templates/index/recentBlocks.html

@@ -29,7 +29,7 @@ <h5 class="card-title d-flex justify-content-between align-items-center" style="
                   {{ else }}
                     <td><a href="/slot/{{ $block.Slot }}">{{ formatAddCommas $block.Slot }}</a></td>
                   {{ end }}
-                  <td>{{ ethBlockLink $block.EthBlock }}</td>
+                  <td>{{ if $block.WithEthBlock }}{{ ethBlockLink $block.EthBlock }}{{ else }}-{{ end }}</td>
                   <td>
                     {{ if eq $block.Slot 0 }}
                       <span class="badge rounded-pill text-bg-info">Genesis</span>

types/models/epoch.go

@@ -54,6 +54,7 @@ type EpochPageDataSlot struct {
 	SyncParticipation     float64   `json:"sync_participation"`
 	EthTransactionCount   uint64    `json:"eth_transaction_count"`
 	EthBlockNumber        uint64    `json:"eth_block_number"`
+	WithEthBlock          bool      `json:"with_eth_block"`
 	Graffiti              []byte    `json:"graffiti"`
 	BlockRoot             []byte    `json:"block_root"`
 }

types/models/indexPage.go

@@ -58,6 +58,7 @@ type IndexPageDataEpochs struct {
 type IndexPageDataBlocks struct {
 	Epoch        uint64    `json:"epoch"`
 	Slot         uint64    `json:"slot"`
+	WithEthBlock bool      `json:"with_eth_block"`
 	EthBlock     uint64    `json:"eth_block"`
 	Ts           time.Time `json:"ts"`
 	Proposer     uint64    `json:"proposer"`

types/models/slots.go

@@ -41,6 +41,7 @@ type SlotsPageDataSlot struct {
 	AttesterSlashingCount uint64                    `json:"attester_slashing_count"`
 	SyncParticipation     float64                   `json:"sync_participation"`
 	EthTransactionCount   uint64                    `json:"eth_transaction_count"`
+	WithEthBlock          bool                      `json:"with_eth_block"`
 	EthBlockNumber        uint64                    `json:"eth_block_number"`
 	Graffiti              []byte                    `json:"graffiti"`
 	BlockRoot             []byte                    `json:"block_root"`

types/models/slots_filtered.go

@@ -51,6 +51,7 @@ type SlotsFilteredPageDataSlot struct {
 	AttesterSlashingCount uint64    `json:"attester_slashing_count"`
 	SyncParticipation     float64   `json:"sync_participation"`
 	EthTransactionCount   uint64    `json:"eth_transaction_count"`
+	WithEthBlock          bool      `json:"with_eth_block"`
 	EthBlockNumber        uint64    `json:"eth_block_number"`
 	Graffiti              []byte    `json:"graffiti"`
 	BlockRoot             []byte    `json:"block_root"`

types/models/validator.go

@@ -36,11 +36,12 @@ type ValidatorPageData struct {
 }
 
 type ValidatorPageDataBlocks struct {
-	Epoch     uint64    `json:"epoch"`
-	Slot      uint64    `json:"slot"`
-	EthBlock  uint64    `json:"eth_block"`
-	Ts        time.Time `json:"ts"`
-	Status    uint64    `json:"status"`
-	BlockRoot string    `json:"block_root"`
-	Graffiti  []byte    `json:"graffiti"`
+	Epoch        uint64    `json:"epoch"`
+	Slot         uint64    `json:"slot"`
+	WithEthBlock bool      `json:"with_eth_block"`
+	EthBlock     uint64    `json:"eth_block"`
+	Ts           time.Time `json:"ts"`
+	Status       uint64    `json:"status"`
+	BlockRoot    string    `json:"block_root"`
+	Graffiti     []byte    `json:"graffiti"`
 }

types/models/validator_slots.go

@@ -43,6 +43,7 @@ type ValidatorSlotsPageDataSlot struct {
 	AttesterSlashingCount uint64    `json:"attester_slashing_count"`
 	SyncParticipation     float64   `json:"sync_participation"`
 	EthTransactionCount   uint64    `json:"eth_transaction_count"`
+	WithEthBlock          bool      `json:"with_eth_block"`
 	EthBlockNumber        uint64    `json:"eth_block_number"`
 	Graffiti              []byte    `json:"graffiti"`
 	BlockRoot             []byte    `json:"block_root"`