Add a 2 second timeout to fetching eth1data (#5583)

* Add a 2 second timeout to fetching eth1data
* goimports
* fix
* use package const
* Merge refs/heads/master into graceful-failure-eth1data
* fix test
* Merge refs/heads/master into graceful-failure-eth1data
* Merge refs/heads/master into graceful-failure-eth1data
* Merge refs/heads/master into graceful-failure-eth1data
* Merge refs/heads/master into graceful-failure-eth1data

beacon-chain/rpc/validator/proposer.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"math/big"
 	"math/rand"
+	"time"
 
 	"github.com/pkg/errors"
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
@@ -30,6 +31,8 @@ import (
 // eth1DataNotification is a latch to stop flooding logs with the same warning.
 var eth1DataNotification bool
 
+const eth1dataTimeout = 2 * time.Second
+
 // GetBlock is called by a proposer during its assigned slot to request a block to sign
 // by passing in the slot and the signed randao reveal of the slot.
 func (vs *Server) GetBlock(ctx context.Context, req *ethpb.BlockRequest) (*ethpb.BeaconBlock, error) {
@@ -154,6 +157,9 @@ func (vs *Server) ProposeBlock(ctx context.Context, blk *ethpb.SignedBeaconBlock
 //  - Subtract that eth1block.number by ETH1_FOLLOW_DISTANCE.
 //  - This is the eth1block to use for the block proposal.
 func (vs *Server) eth1Data(ctx context.Context, slot uint64) (*ethpb.Eth1Data, error) {
+	ctx, cancel := context.WithTimeout(ctx, eth1dataTimeout)
+	defer cancel()
+
 	if vs.MockEth1Votes {
 		return vs.mockETH1DataVote(ctx, slot)
 	}
@@ -169,11 +175,13 @@ func (vs *Server) eth1Data(ctx context.Context, slot uint64) (*ethpb.Eth1Data, e
 	// Look up most recent block up to timestamp
 	blockNumber, err := vs.Eth1BlockFetcher.BlockNumberByTimestamp(ctx, eth1VotingPeriodStartTime)
 	if err != nil {
-		return nil, errors.Wrap(err, "could not get block number from timestamp")
+		log.WithError(err).Error("Failed to get block number from timestamp")
+		return vs.randomETH1DataVote(ctx)
 	}
 	eth1Data, err := vs.defaultEth1DataResponse(ctx, blockNumber)
 	if err != nil {
-		return nil, errors.Wrap(err, "could not get eth1 data from block number")
+		log.WithError(err).Error("Failed to get eth1 data from block number")
+		return vs.randomETH1DataVote(ctx)
 	}
 
 	return eth1Data, nil
@@ -362,6 +370,9 @@ func (vs *Server) canonicalEth1Data(ctx context.Context, beaconState *stateTrie.
 // hash of eth1 block hash that is FOLLOW_DISTANCE back from its
 // latest block.
 func (vs *Server) defaultEth1DataResponse(ctx context.Context, currentHeight *big.Int) (*ethpb.Eth1Data, error) {
+	if ctx.Err() != nil {
+		return nil, ctx.Err()
+	}
 	eth1FollowDistance := int64(params.BeaconConfig().Eth1FollowDistance)
 	ancestorHeight := big.NewInt(0).Sub(currentHeight, big.NewInt(eth1FollowDistance))
 	blockHash, err := vs.Eth1BlockFetcher.BlockHashByHeight(ctx, ancestorHeight)

beacon-chain/rpc/validator/proposer_test.go

@@ -5,7 +5,6 @@ import (
 	"context"
 	"math/big"
 	"reflect"
-	"strings"
 	"testing"
 
 	"github.com/gogo/protobuf/proto"
@@ -1130,9 +1129,8 @@ func TestEth1Data_EmptyVotesFetchBlockHashFailure(t *testing.T) {
 		BlockReceiver:     &mock.ChainService{State: beaconState},
 		HeadFetcher:       &mock.ChainService{State: beaconState},
 	}
-	want := "could not fetch ETH1_FOLLOW_DISTANCE ancestor"
-	if _, err := proposerServer.eth1Data(context.Background(), beaconState.Slot()+1); err == nil || !strings.Contains(err.Error(), want) {
-		t.Errorf("Expected error %v, received %v", want, err)
+	if _, err := proposerServer.eth1Data(context.Background(), beaconState.Slot()+1); err != nil {
+		t.Errorf("A failed request should not have returned an error, got %v", err)
 	}
 }