fix(e2e): TestSync_SingleBlockProducer not flaky

- refactor `compareBlocksByNumber`

tests/stress/helpers.go

@@ -7,6 +7,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"strings"
 	"testing"
 	"time"
 
@@ -73,52 +74,34 @@ func compareChainHeadsWithRetry(ctx context.Context, nodes node.Nodes,
 // compareBlocksByNumber calls getBlockByNumber for each node in the array
 // it returns a map of block hashes to node key names, and an error if the hashes don't all match
 func compareBlocksByNumber(ctx context.Context, t *testing.T, nodes node.Nodes,
-	num string) (hashToKeys map[common.Hash][]string) {
-	type resultContainer struct {
-		hash    common.Hash
-		nodeKey string
-		err     error
-	}
-	results := make(chan resultContainer)
+	num string) (nodeKeys []string) {
+	t.Helper()
 
+	blockHashes := make(map[common.Hash]struct{}, 1)
 	for _, n := range nodes {
-		go func(node node.Node) {
-			result := resultContainer{
-				nodeKey: node.GetKey(),
+		const retryWait = time.Second
+		err := retry.UntilOK(ctx, retryWait, func() (ok bool) {
+			hash, err := rpc.GetBlockHash(ctx, n.GetRPCPort(), num)
+			if err != nil {
+				const blockDoesNotExistString = "cannot find node with number greater than highest in blocktree"
+				if strings.Contains(err.Error(), blockDoesNotExistString) {
+					return false // retry after retryWait has elapsed.
+				}
+				require.NoError(t, err) // fail the test now.
 			}
 
-			const retryWait = 200 * time.Millisecond
-			result.err = retry.UntilNoError(ctx, retryWait, func() (err error) {
-				result.hash, err = rpc.GetBlockHash(ctx, node.GetRPCPort(), num)
-				return err
-			})
-
-			results <- result
-		}(n)
-	}
-
-	var err error
-	hashToKeys = make(map[common.Hash][]string, len(nodes))
-	for range nodes {
-		result := <-results
-		if err != nil {
-			continue // one failed, we don't care anymore
-		}
-
-		if result.err != nil {
-			err = result.err
-			continue
-		}
-
-		hashToKeys[result.hash] = append(hashToKeys[result.hash], result.nodeKey)
+			blockHashes[hash] = struct{}{}
+			nodeKeys = append(nodeKeys, n.GetKey())
+			return true
+		})
+		require.NoError(t, err)
 	}
 
-	require.NoError(t, err)
-	require.Lenf(t, hashToKeys, 1,
+	require.Lenf(t, blockHashes, 1,
 		"expected 1 block found for number %s but got %d block(s)",
-		num, len(hashToKeys))
+		num, len(blockHashes))
 
-	return hashToKeys
+	return nodeKeys
 }
 
 // compareFinalizedHeads calls getFinalizedHeadByRound for each node in the array

tests/stress/stress_test.go

@@ -10,7 +10,6 @@ import (
 	"math/rand"
 	"os"
 	"path/filepath"
-	"strconv"
 	"strings"
 	"testing"
 	"time"
@@ -101,10 +100,8 @@ func TestRestartNode(t *testing.T) {
 }
 
 func TestSync_SingleBlockProducer(t *testing.T) {
-	numNodes := 4
-	utils.Logger.Patch(log.SetLevel(log.Info))
+	const numNodes = 4
 
-	// start block producing node first
 	basePath := t.TempDir()
 	genesisPath := libutils.GetDevGenesisSpecPathTest(t)
 	configNoGrandpa := config.CreateNoGrandpa(t)
@@ -115,41 +112,25 @@ func TestSync_SingleBlockProducer(t *testing.T) {
 		node.SetConfig(configNoGrandpa),
 		node.SetBabeLead(true))
 
-	ctx, cancel := context.WithCancel(context.Background())
-	babeLeadNode.InitAndStartTest(ctx, t, cancel)
-
 	configNoAuthority := config.CreateNotAuthority(t)
-
-	// wait and start rest of nodes - if they all start at the same time the first round usually doesn't complete since
-	// all nodes vote for different blocks.
-	time.Sleep(time.Second * 15)
-
-	nodes := node.MakeNodes(t, numNodes-1,
+	noAuthorityNodes := node.MakeNodes(t, numNodes-1,
 		node.SetGenesis(genesisPath),
 		node.SetConfig(configNoAuthority))
-	nodes.InitAndStartTest(ctx, t, cancel)
-	nodes = append(nodes, babeLeadNode)
-
-	time.Sleep(time.Second * 30)
-
-	numCmps := 10
 
-	for i := 0; i < numCmps; i++ {
-		time.Sleep(3 * time.Second)
-		t.Log("comparing...", i)
+	nodes := make(node.Nodes, 0, numNodes)
+	nodes = append(nodes, babeLeadNode)
+	nodes = append(nodes, noAuthorityNodes...)
 
-		const comparisonTimeout = 5 * time.Second
-		compareCtx, cancel := context.WithTimeout(ctx, comparisonTimeout)
+	ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
 
-		hashes := compareBlocksByNumber(compareCtx, t, nodes, strconv.Itoa(i))
+	nodes.InitAndStartTest(ctx, t, cancel)
 
-		cancel()
+	const blockNumbers = 10
+	for blockNumber := 0; blockNumber < blockNumbers; blockNumber++ {
+		t.Logf("comparing block number %d...", blockNumber)
 
-		// there will only be one key in the mapping
-		for _, nodesWithHash := range hashes {
-			// allow 1 node to potentially not have synced. this is due to the need to increase max peer count
-			require.GreaterOrEqual(t, len(nodesWithHash), numNodes-1)
-		}
+		nodeKeys := compareBlocksByNumber(ctx, t, nodes, fmt.Sprint(blockNumber))
+		require.Equal(t, len(nodeKeys), numNodes)
 	}
 }
 
@@ -212,7 +193,7 @@ func TestSync_MultipleEpoch(t *testing.T) {
 		const compareTimeout = 5 * time.Second
 		compareCtx, cancel := context.WithTimeout(ctx, compareTimeout)
 
-		_ = compareBlocksByNumber(compareCtx, t, nodes, strconv.Itoa(int(i)))
+		_ = compareBlocksByNumber(compareCtx, t, nodes, fmt.Sprint(i))
 
 		cancel()
 	}
@@ -260,7 +241,7 @@ func TestSync_SingleSyncingNode(t *testing.T) {
 		const compareTimeout = 5 * time.Second
 		compareCtx, cancel := context.WithTimeout(ctx, compareTimeout)
 
-		_ = compareBlocksByNumber(compareCtx, t, nodes, strconv.Itoa(i))
+		_ = compareBlocksByNumber(compareCtx, t, nodes, fmt.Sprint(i))
 
 		cancel()
 	}
@@ -467,7 +448,7 @@ func TestSync_Restart(t *testing.T) {
 		const compareTimeout = 5 * time.Second
 		compareCtx, cancel := context.WithTimeout(mainCtx, compareTimeout)
 
-		_ = compareBlocksByNumber(compareCtx, t, nodes, strconv.Itoa(i))
+		_ = compareBlocksByNumber(compareCtx, t, nodes, fmt.Sprint(i))
 
 		cancel()
 
@@ -863,7 +844,7 @@ func TestStress_SecondarySlotProduction(t *testing.T) {
 				fmt.Printf("%d iteration\n", i)
 
 				getBlockHashCtx, cancel := context.WithTimeout(ctx, time.Second)
-				hash, err := rpc.GetBlockHash(getBlockHashCtx, nodes[0].GetRPCPort(), fmt.Sprintf("%d", i))
+				hash, err := rpc.GetBlockHash(getBlockHashCtx, nodes[0].GetRPCPort(), fmt.Sprint(i))
 				cancel()
 
 				require.NoError(t, err)