diff --git a/tests/stress/helpers.go b/tests/stress/helpers.go index 4dc7ed2f597..4bbe7ad3fda 100644 --- a/tests/stress/helpers.go +++ b/tests/stress/helpers.go @@ -7,6 +7,7 @@ import ( "context" "errors" "fmt" + "strings" "testing" "time" @@ -73,52 +74,34 @@ func compareChainHeadsWithRetry(ctx context.Context, nodes node.Nodes, // compareBlocksByNumber calls getBlockByNumber for each node in the array // it returns a map of block hashes to node key names, and an error if the hashes don't all match func compareBlocksByNumber(ctx context.Context, t *testing.T, nodes node.Nodes, - num string) (hashToKeys map[common.Hash][]string) { - type resultContainer struct { - hash common.Hash - nodeKey string - err error - } - results := make(chan resultContainer) + num string) (nodeKeys []string) { + t.Helper() + blockHashes := make(map[common.Hash]struct{}, 1) for _, n := range nodes { - go func(node node.Node) { - result := resultContainer{ - nodeKey: node.GetKey(), + const retryWait = time.Second + err := retry.UntilOK(ctx, retryWait, func() (ok bool) { + hash, err := rpc.GetBlockHash(ctx, n.GetRPCPort(), num) + if err != nil { + const blockDoesNotExistString = "cannot find node with number greater than highest in blocktree" + if strings.Contains(err.Error(), blockDoesNotExistString) { + return false // retry after retryWait has elapsed. + } + require.NoError(t, err) // fail the test now. } - const retryWait = 200 * time.Millisecond - result.err = retry.UntilNoError(ctx, retryWait, func() (err error) { - result.hash, err = rpc.GetBlockHash(ctx, node.GetRPCPort(), num) - return err - }) - - results <- result - }(n) - } - - var err error - hashToKeys = make(map[common.Hash][]string, len(nodes)) - for range nodes { - result := <-results - if err != nil { - continue // one failed, we don't care anymore - } - - if result.err != nil { - err = result.err - continue - } - - hashToKeys[result.hash] = append(hashToKeys[result.hash], result.nodeKey) + blockHashes[hash] = struct{}{} + nodeKeys = append(nodeKeys, n.GetKey()) + return true + }) + require.NoError(t, err) } - require.NoError(t, err) - require.Lenf(t, hashToKeys, 1, + require.Lenf(t, blockHashes, 1, "expected 1 block found for number %s but got %d block(s)", - num, len(hashToKeys)) + num, len(blockHashes)) - return hashToKeys + return nodeKeys } // compareFinalizedHeads calls getFinalizedHeadByRound for each node in the array diff --git a/tests/stress/stress_test.go b/tests/stress/stress_test.go index c8cfb976c24..fe15e53c78e 100644 --- a/tests/stress/stress_test.go +++ b/tests/stress/stress_test.go @@ -10,7 +10,6 @@ import ( "math/rand" "os" "path/filepath" - "strconv" "strings" "testing" "time" @@ -101,10 +100,8 @@ func TestRestartNode(t *testing.T) { } func TestSync_SingleBlockProducer(t *testing.T) { - numNodes := 4 - utils.Logger.Patch(log.SetLevel(log.Info)) + const numNodes = 4 - // start block producing node first basePath := t.TempDir() genesisPath := libutils.GetDevGenesisSpecPathTest(t) configNoGrandpa := config.CreateNoGrandpa(t) @@ -115,41 +112,25 @@ func TestSync_SingleBlockProducer(t *testing.T) { node.SetConfig(configNoGrandpa), node.SetBabeLead(true)) - ctx, cancel := context.WithCancel(context.Background()) - babeLeadNode.InitAndStartTest(ctx, t, cancel) - configNoAuthority := config.CreateNotAuthority(t) - - // wait and start rest of nodes - if they all start at the same time the first round usually doesn't complete since - // all nodes vote for different blocks. - time.Sleep(time.Second * 15) - - nodes := node.MakeNodes(t, numNodes-1, + noAuthorityNodes := node.MakeNodes(t, numNodes-1, node.SetGenesis(genesisPath), node.SetConfig(configNoAuthority)) - nodes.InitAndStartTest(ctx, t, cancel) - nodes = append(nodes, babeLeadNode) - - time.Sleep(time.Second * 30) - - numCmps := 10 - for i := 0; i < numCmps; i++ { - time.Sleep(3 * time.Second) - t.Log("comparing...", i) + nodes := make(node.Nodes, 0, numNodes) + nodes = append(nodes, babeLeadNode) + nodes = append(nodes, noAuthorityNodes...) - const comparisonTimeout = 5 * time.Second - compareCtx, cancel := context.WithTimeout(ctx, comparisonTimeout) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) - hashes := compareBlocksByNumber(compareCtx, t, nodes, strconv.Itoa(i)) + nodes.InitAndStartTest(ctx, t, cancel) - cancel() + const blockNumbers = 10 + for blockNumber := 0; blockNumber < blockNumbers; blockNumber++ { + t.Logf("comparing block number %d...", blockNumber) - // there will only be one key in the mapping - for _, nodesWithHash := range hashes { - // allow 1 node to potentially not have synced. this is due to the need to increase max peer count - require.GreaterOrEqual(t, len(nodesWithHash), numNodes-1) - } + nodeKeys := compareBlocksByNumber(ctx, t, nodes, fmt.Sprint(blockNumber)) + require.Equal(t, len(nodeKeys), numNodes) } } @@ -212,7 +193,7 @@ func TestSync_MultipleEpoch(t *testing.T) { const compareTimeout = 5 * time.Second compareCtx, cancel := context.WithTimeout(ctx, compareTimeout) - _ = compareBlocksByNumber(compareCtx, t, nodes, strconv.Itoa(int(i))) + _ = compareBlocksByNumber(compareCtx, t, nodes, fmt.Sprint(i)) cancel() } @@ -260,7 +241,7 @@ func TestSync_SingleSyncingNode(t *testing.T) { const compareTimeout = 5 * time.Second compareCtx, cancel := context.WithTimeout(ctx, compareTimeout) - _ = compareBlocksByNumber(compareCtx, t, nodes, strconv.Itoa(i)) + _ = compareBlocksByNumber(compareCtx, t, nodes, fmt.Sprint(i)) cancel() } @@ -467,7 +448,7 @@ func TestSync_Restart(t *testing.T) { const compareTimeout = 5 * time.Second compareCtx, cancel := context.WithTimeout(mainCtx, compareTimeout) - _ = compareBlocksByNumber(compareCtx, t, nodes, strconv.Itoa(i)) + _ = compareBlocksByNumber(compareCtx, t, nodes, fmt.Sprint(i)) cancel() @@ -863,7 +844,7 @@ func TestStress_SecondarySlotProduction(t *testing.T) { fmt.Printf("%d iteration\n", i) getBlockHashCtx, cancel := context.WithTimeout(ctx, time.Second) - hash, err := rpc.GetBlockHash(getBlockHashCtx, nodes[0].GetRPCPort(), fmt.Sprintf("%d", i)) + hash, err := rpc.GetBlockHash(getBlockHashCtx, nodes[0].GetRPCPort(), fmt.Sprint(i)) cancel() require.NoError(t, err)