E2E: retry setup of data integrity check (#6710)

We have seen some test failures on builds where the cluster rejects connection attempts, even though it has previously already responded to health checks. The setup of the data integrity check is one of the few places where we do not retry requests on failures. I looked at the logs and it seems the requests are made only a few seconds after the cluster reached ready state so I am wondering if we should be more lenient here and allow for retries. 

Also track the internal http service in e2e test checks and fix a potential resource leak in the test client.

test/e2e/test/elasticsearch/checks_data.go

@@ -91,8 +91,8 @@ func (dc *DataIntegrityCheck) Init() error {
 		return err
 	}
 	// delete the index but ignore errors (e.g. if it did not exist yet)
-	resp, err := esClient.Request(context.Background(), indexDeletion)
-	if err == nil {
+	resp, _ := esClient.Request(context.Background(), indexDeletion)
+	if resp != nil {
 		resp.Body.Close()
 	}
 

test/e2e/test/elasticsearch/checks_k8s.go

@@ -359,6 +359,7 @@ func CheckServices(b Builder, k *test.K8sClient) test.Step {
 		Test: test.Eventually(func() error {
 			for _, s := range []string{
 				// we intentionally hardcode the names here to catch any accidental breaking change
+				b.Elasticsearch.Name + "-es-internal-http",
 				b.Elasticsearch.Name + "-es-http",
 				b.Elasticsearch.Name + "-es-transport",
 			} {
@@ -385,8 +386,9 @@ func CheckServicesEndpoints(b Builder, k *test.K8sClient) test.Step {
 			expectedEs := b.GetExpectedElasticsearch()
 			for endpointName, addrCount := range map[string]int{
 				// we intentionally hardcode the names here to catch any accidental breaking change
-				b.Elasticsearch.Name + "-es-http":      int(expectedEs.Spec.NodeCount()),
-				b.Elasticsearch.Name + "-es-transport": int(expectedEs.Spec.NodeCount()),
+				b.Elasticsearch.Name + "-es-internal-http": int(expectedEs.Spec.NodeCount()),
+				b.Elasticsearch.Name + "-es-http":          int(expectedEs.Spec.NodeCount()),
+				b.Elasticsearch.Name + "-es-transport":     int(expectedEs.Spec.NodeCount()),
 			} {
 				if addrCount == 0 {
 					continue // maybe no Kibana

test/e2e/test/elasticsearch/steps_mutation.go

@@ -109,7 +109,9 @@ func (b Builder) MutationTestSteps(k *test.K8sClient) test.StepList {
 			Name: "Add some data to the cluster before starting the mutation",
 			Test: func(t *testing.T) {
 				dataIntegrityCheck = NewDataIntegrityCheck(k, b)
-				require.NoError(t, dataIntegrityCheck.Init())
+				test.Eventually(func() error {
+					return dataIntegrityCheck.Init()
+				})(t)
 			},
 		},
 		test.Step{