Add healthz check to E2E (#5426)

* Add healthz check to E2E * Log fixes * Merge branch 'master' of https://github.com/prysmaticlabs/prysm into e2e-healthz * Merge branch 'master' of https://github.com/prysmaticlabs/prysm into e2e-healthz * Merge branch 'master' into e2e-healthz * Merge branch 'master' into e2e-healthz * Merge branch 'master' into e2e-healthz * Merge branch 'master' into e2e-healthz
prysmaticlabs · Apr 15, 2020 · c70103b · c70103b
1 parent 0ecd2a6
commit c70103b
Show file tree

Hide file tree

Showing 7 changed files with 52 additions and 1 deletion.
diff --git a/endtoend/components/beacon_node.go b/endtoend/components/beacon_node.go
@@ -48,7 +48,7 @@ func StartNewBeaconNode(t *testing.T, config *types.E2EConfig, index int, enr st
 		fmt.Sprintf("--min-sync-peers=%d", e2e.TestParams.BeaconNodeCount-1),
 		fmt.Sprintf("--p2p-udp-port=%d", e2e.TestParams.BeaconNodeRPCPort+index+10),
 		fmt.Sprintf("--p2p-tcp-port=%d", e2e.TestParams.BeaconNodeRPCPort+index+20),
-		fmt.Sprintf("--monitoring-port=%d", e2e.TestParams.BeaconNodeRPCPort+index+30),
+		fmt.Sprintf("--monitoring-port=%d", e2e.TestParams.BeaconNodeMetricsPort+index),
 		fmt.Sprintf("--grpc-gateway-port=%d", e2e.TestParams.BeaconNodeRPCPort+index+40),
 		fmt.Sprintf("--contract-deployment-block=%d", 0),
 		fmt.Sprintf("--rpc-max-page-size=%d", params.BeaconConfig().MinGenesisActiveValidatorCount),

diff --git a/endtoend/evaluators/BUILD.bazel b/endtoend/evaluators/BUILD.bazel
@@ -13,6 +13,7 @@ go_library(
     visibility = ["//endtoend:__subpackages__"],
     deps = [
         "//beacon-chain/core/helpers:go_default_library",
+        "//endtoend/params:go_default_library",
         "//endtoend/types:go_default_library",
         "//shared/bytesutil:go_default_library",
         "//shared/params:go_default_library",

diff --git a/endtoend/evaluators/node.go b/endtoend/evaluators/node.go
@@ -4,10 +4,13 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"io/ioutil"
+	"net/http"
 
 	ptypes "github.com/gogo/protobuf/types"
 	"github.com/pkg/errors"
 	eth "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	e2e "github.com/prysmaticlabs/prysm/endtoend/params"
 	"github.com/prysmaticlabs/prysm/endtoend/types"
 	"google.golang.org/grpc"
 )
@@ -19,6 +22,13 @@ var PeersConnect = types.Evaluator{
 	Evaluation: peersConnect,
 }
 
+// HealthzCheck pings healthz and errors if it doesn't have the expected OK status.
+var HealthzCheck = types.Evaluator{
+	Name:       "healthz_check_epoch_%d",
+	Policy:     afterNthEpoch(0),
+	Evaluation: healthzCheck,
+}
+
 // FinishedSyncing returns whether the beacon node with the given rpc port has finished syncing.
 var FinishedSyncing = types.Evaluator{
 	Name:       "finished_syncing",
@@ -40,6 +50,42 @@ func onEpoch(epoch uint64) func(uint64) bool {
 	}
 }
 
+func healthzCheck(conns ...*grpc.ClientConn) error {
+	count := len(conns)
+	for i := 0; i < count; i++ {
+		resp, err := http.Get(fmt.Sprintf("http://localhost:%d/healthz", e2e.TestParams.BeaconNodeMetricsPort+i))
+		if err != nil {
+			return errors.Wrapf(err, "could not connect to beacon node %d", i)
+		}
+		if resp.StatusCode != http.StatusOK {
+			body, err := ioutil.ReadAll(resp.Body)
+			if err != nil {
+				return err
+			}
+			return fmt.Errorf("expected status code OK for beacon node %d, received %v with body %s", i, resp.StatusCode, body)
+		}
+		if err := resp.Body.Close(); err != nil {
+			return err
+		}
+
+		resp, err = http.Get(fmt.Sprintf("http://localhost:%d/healthz", e2e.TestParams.ValidatorMetricsPort+i))
+		if err != nil {
+			return errors.Wrapf(err, "could not connect to validator client %d", i)
+		}
+		if resp.StatusCode != http.StatusOK {
+			body, err := ioutil.ReadAll(resp.Body)
+			if err != nil {
+				return err
+			}
+			return fmt.Errorf("expected status code OK for validator client %d, received %v with body %s", i, resp.StatusCode, body)
+		}
+		if err := resp.Body.Close(); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 func peersConnect(conns ...*grpc.ClientConn) error {
 	if len(conns) == 1 {
 		return nil

diff --git a/endtoend/minimal_antiflake_e2e_1_test.go b/endtoend/minimal_antiflake_e2e_1_test.go
@@ -22,6 +22,7 @@ func TestEndToEnd_AntiFlake_MinimalConfig_1(t *testing.T) {
 		TestSlasher:    false,
 		Evaluators: []types.Evaluator{
 			ev.PeersConnect,
+			ev.HealthzCheck,
 			ev.ValidatorsAreActive,
 			ev.ValidatorsParticipating,
 		},

diff --git a/endtoend/minimal_antiflake_e2e_2_test.go b/endtoend/minimal_antiflake_e2e_2_test.go
@@ -22,6 +22,7 @@ func TestEndToEnd_AntiFlake_MinimalConfig_2(t *testing.T) {
 		TestSlasher:    false,
 		Evaluators: []types.Evaluator{
 			ev.PeersConnect,
+			ev.HealthzCheck,
 			ev.ValidatorsAreActive,
 			ev.ValidatorsParticipating,
 		},

diff --git a/endtoend/minimal_e2e_test.go b/endtoend/minimal_e2e_test.go
@@ -33,6 +33,7 @@ func TestEndToEnd_MinimalConfig(t *testing.T) {
 		TestSlasher:    true,
 		Evaluators: []types.Evaluator{
 			ev.PeersConnect,
+			ev.HealthzCheck,
 			ev.ValidatorsAreActive,
 			ev.ValidatorsParticipating,
 			ev.FinalizationOccurs,

diff --git a/endtoend/minimal_slashing_e2e_test.go b/endtoend/minimal_slashing_e2e_test.go
@@ -22,6 +22,7 @@ func TestEndToEnd_Slashing_MinimalConfig(t *testing.T) {
 		TestSlasher:    true,
 		Evaluators: []types.Evaluator{
 			ev.PeersConnect,
+			ev.HealthzCheck,
 			ev.ValidatorsSlashed,
 			ev.SlashedValidatorsLoseBalance,
 			ev.InjectDoubleVote,