e2e: add cluster version test with rolling start servers

tests/e2e/cluster_test.go

@@ -20,6 +20,7 @@ import (
 	"net/url"
 	"os"
 	"strings"
+	"time"
 
 	"go.etcd.io/etcd/etcdserver"
 )
@@ -134,6 +135,8 @@ type etcdProcessClusterConfig struct {
 	enableV2            bool
 	initialCorruptCheck bool
 	authTokenOpts       string
+
+	rollingStart bool
 }
 
 // newEtcdProcessCluster launches a new cluster from etcd processes, returning
@@ -155,8 +158,14 @@ func newEtcdProcessCluster(cfg *etcdProcessClusterConfig) (*etcdProcessCluster,
 		epc.procs[i] = proc
 	}
 
-	if err := epc.Start(); err != nil {
-		return nil, err
+	if cfg.rollingStart {
+		if err := epc.RollingStart(); err != nil {
+			return nil, err
+		}
+	} else {
+		if err := epc.Start(); err != nil {
+			return nil, err
+		}
 	}
 	return epc, nil
 }
@@ -347,6 +356,10 @@ func (epc *etcdProcessCluster) Start() error {
 	return epc.start(func(ep etcdProcess) error { return ep.Start() })
 }
 
+func (epc *etcdProcessCluster) RollingStart() error {
+	return epc.rollingStart(func(ep etcdProcess) error { return ep.Start() })
+}
+
 func (epc *etcdProcessCluster) Restart() error {
 	return epc.start(func(ep etcdProcess) error { return ep.Restart() })
 }
@@ -365,6 +378,22 @@ func (epc *etcdProcessCluster) start(f func(ep etcdProcess) error) error {
 	return nil
 }
 
+func (epc *etcdProcessCluster) rollingStart(f func(ep etcdProcess) error) error {
+	readyC := make(chan error, len(epc.procs))
+	for i := range epc.procs {
+		go func(n int) { readyC <- f(epc.procs[n]) }(i)
+		// make sure the servers do not start at the same time
+		time.Sleep(time.Second)
+	}
+	for range epc.procs {
+		if err := <-readyC; err != nil {
+			epc.Close()
+			return err
+		}
+	}
+	return nil
+}
+
 func (epc *etcdProcessCluster) Stop() (err error) {
 	for _, p := range epc.procs {
 		if p == nil {

tests/e2e/ctl_v3_test.go

@@ -30,42 +30,72 @@ import (
 func TestCtlV3Version(t *testing.T) { testCtl(t, versionTest) }
 
 func TestClusterVersion(t *testing.T) {
-	binary := binDir + "/etcd"
-	if !fileutil.Exist(binary) {
-		t.Skipf("%q does not exist", binary)
+	tests := []struct {
+		name         string
+		rollingStart bool
+	}{
+		{
+			name:         "When start servers at the same time",
+			rollingStart: false,
+		},
+		{
+			name:         "When start servers one by one",
+			rollingStart: true,
+		},
 	}
-	defer testutil.AfterTest(t)
-	cfg := configNoTLS
-	cfg.execPath = binary
-	cfg.snapshotCount = 3
-	cfg.baseScheme = "unix" // to avoid port conflict
 
-	epc, err := newEtcdProcessCluster(&cfg)
-	if err != nil {
-		t.Fatalf("could not start etcd process cluster (%v)", err)
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			binary := binDir + "/etcd"
+			if !fileutil.Exist(binary) {
+				t.Skipf("%q does not exist", binary)
+			}
+			defer testutil.AfterTest(t)
+			cfg := configNoTLS
+			cfg.execPath = binary
+			cfg.snapshotCount = 3
+			cfg.baseScheme = "unix" // to avoid port conflict
+			cfg.rollingStart = tt.rollingStart
+
+			epc, err := newEtcdProcessCluster(&cfg)
+			if err != nil {
+				t.Fatalf("could not start etcd process cluster (%v)", err)
+			}
+			defer func() {
+				if errC := epc.Close(); errC != nil {
+					t.Fatalf("error closing etcd processes (%v)", errC)
+				}
+			}()
+
+			ctx := ctlCtx{
+				t:   t,
+				cfg: cfg,
+				epc: epc,
+			}
+			cv := version.Cluster(version.Version)
+			clusterVersionTest(ctx, `"etcdcluster":"`+cv)
+		})
 	}
-	defer func() {
-		if errC := epc.Close(); errC != nil {
-			t.Fatalf("error closing etcd processes (%v)", errC)
-		}
-	}()
-	cv := version.Cluster(version.Version)
+}
+
+func versionTest(cx ctlCtx) {
+	if err := ctlV3Version(cx); err != nil {
+		cx.t.Fatalf("versionTest ctlV3Version error (%v)", err)
+	}
+}
+
+func clusterVersionTest(cx ctlCtx, expected string) {
+	var err error
 	for i := 0; i < 7; i++ {
-		if err = cURLGet(epc, cURLReq{endpoint: "/version", expected: `"etcdcluster":"` + cv}); err != nil {
-			t.Logf("#%d: v3 is not ready yet (%v)", i, err)
+		if err = cURLGet(cx.epc, cURLReq{endpoint: "/version", expected: expected}); err != nil {
+			cx.t.Logf("#%d: v3 is not ready yet (%v)", i, err)
 			time.Sleep(time.Second)
 			continue
 		}
 		break
 	}
 	if err != nil {
-		t.Fatalf("failed cluster version test expected %v got (%v)", cv, err)
-	}
-}
-
-func versionTest(cx ctlCtx) {
-	if err := ctlV3Version(cx); err != nil {
-		cx.t.Fatalf("versionTest ctlV3Version error (%v)", err)
+		cx.t.Fatalf("failed cluster version test expected %v got (%v)", expected, err)
 	}
 }