diff --git a/tests/fixture/tmpnet/network.go b/tests/fixture/tmpnet/network.go index ceff7bcff982..3ca8318efa85 100644 --- a/tests/fixture/tmpnet/network.go +++ b/tests/fixture/tmpnet/network.go @@ -441,6 +441,23 @@ func (n *Network) Restart(ctx context.Context, w io.Writer) error { return err } for _, node := range n.Nodes { + // Ensure the node reuses the same API port across restarts to ensure + // consistent labeling of metrics. Otherwise prometheus's automatic + // addition of the `instance` label (host:port) results in + // segmentation of results for a given node every time the port + // changes on restart. This segmentation causes graphs on the grafana + // dashboards to display multiple series per graph for a given node, + // one for each port that the node used. + // + // There is a non-zero chance of the port being allocatted to a + // different process and the node subsequently being unable to start, + // but the alternative is having to update the grafana dashboards + // query-by-query to ensure that node metrics ignore the instance + // label. + if err := node.SaveAPIPort(); err != nil { + return err + } + if err := node.Stop(ctx); err != nil { return fmt.Errorf("failed to stop node %s: %w", node.NodeID, err) } diff --git a/tests/fixture/tmpnet/node.go b/tests/fixture/tmpnet/node.go index 10f80371cbf4..2d20ce098587 100644 --- a/tests/fixture/tmpnet/node.go +++ b/tests/fixture/tmpnet/node.go @@ -9,6 +9,7 @@ import ( "errors" "fmt" "io" + "net" "net/http" "os" "path/filepath" @@ -214,13 +215,14 @@ func (n *Node) Stop(ctx context.Context) error { // Sets networking configuration for the node. // Convenience method for setting networking flags. func (n *Node) SetNetworkingConfig(bootstrapIDs []string, bootstrapIPs []string) { - var ( - // Use dynamic port allocation. - httpPort uint16 = 0 - stakingPort uint16 = 0 - ) - n.Flags[config.HTTPPortKey] = httpPort - n.Flags[config.StakingPortKey] = stakingPort + if _, ok := n.Flags[config.HTTPPortKey]; !ok { + // Default to dynamic port allocation + n.Flags[config.HTTPPortKey] = 0 + } + if _, ok := n.Flags[config.StakingPortKey]; !ok { + // Default to dynamic port allocation + n.Flags[config.StakingPortKey] = 0 + } n.Flags[config.BootstrapIDsKey] = strings.Join(bootstrapIDs, ",") n.Flags[config.BootstrapIPsKey] = strings.Join(bootstrapIPs, ",") } @@ -348,3 +350,16 @@ func (n *Node) EnsureNodeID() error { return nil } + +// Saves the currently allocated API port to the node's configuration +// for use across restarts. Reusing the port ensures consistent +// labeling of metrics. +func (n *Node) SaveAPIPort() error { + hostPort := strings.TrimPrefix(n.URI, "http://") + _, port, err := net.SplitHostPort(hostPort) + if err != nil { + return err + } + n.Flags[config.HTTPPortKey] = port + return nil +}