diff --git a/kola/tests/kubeadm/kubeadm.go b/kola/tests/kubeadm/kubeadm.go index 1ae4ce4d5..2a3de99ca 100644 --- a/kola/tests/kubeadm/kubeadm.go +++ b/kola/tests/kubeadm/kubeadm.go @@ -40,27 +40,35 @@ var ( "flannel", "cilium", } - // params are used to render script templates - // Release is the kubernetes release version we want to use - // ReleaseVersion is the version of the kubelet service and kubeadm dropin - // TODO: when a new version of kubernetes will be tested, it would be nice - // to have a map[string]Release with Release struct holding the parameter below - params = map[string]interface{}{ - // TODO: it's actually the CLI version - // we should pass the CLI and cilium version - // https://github.com/cilium/cilium-cli/issues/118 - "CiliumVersion": "v0.8.3", - "CNIVersion": "v0.8.7", - "CRIctlVersion": "v1.17.0", - "ReleaseVersion": "v0.4.0", - "Release": "v1.21.0", - "DownloadDir": "/opt/bin", - "PodSubnet": "192.168.0.0/17", - "KubeadmSum": "0673408403a3474c868ae86109f11f9114bca7ddce204be0d169316fb3ce0edefa4b2a472ba9b8308e423e6b927d4098ac36296405570f444f39551fb1c4bbb4", - "KubeletSum": "530689c0cc32ef1830f7ae26ac10995f815043d48a905141e23a34a5e61522c4ee2ff46953648c47c5592d7c2ffa40ce90469a697f36f68475b8da5abd73f9f5", - "CRIctlSum": "e258f4607a89b8d44c700036e636dd42cc3e2ed27a3bb13beef736f80f64f10b7974c01259a66131d3f7b44ed0c61b1ca0ea91597c416a9c095c432de5112d44", - "CNISum": "8f2cbee3b5f94d59f919054dccfe99a8e3db5473b553d91da8af4763e811138533e05df4dbeab16b3f774852b4184a7994968f5e036a3f531ad1ac4620d10ede", - "KubectlSum": "9557d298146ef62ffbcf05b3591bf1ce74f345628370447a4f614b5f64e367b5bfa8e397cc4755da9ea38f1ba04c95c65c313e735550ffc3b03c197e936c3e11", + // testConfig holds params for various kubernetes releases + // and the nested params are used to render script templates + testConfig = map[string]map[string]interface{}{ + "v1.22.0": map[string]interface{}{ + "CiliumVersion": "v0.8.3", + "CNIVersion": "v0.8.7", + "CRIctlVersion": "v1.17.0", + "ReleaseVersion": "v0.4.0", + "DownloadDir": "/opt/bin", + "PodSubnet": "192.168.0.0/17", + "KubeadmSum": "339e13ad840cbeab906e416f321467ab6c91cc4b66e5ad4db6f8d41a974146cf8226727edbcf686854a0803246e316158f028de7e753197cdcd2d99a604afbfd", + "KubeletSum": "1b5d530e62f0198aa7af09371ba799d135b54b9a4513981fa09b786ca5fdc98819345112b5c3a68834f6171e9b4438075cf7ec77c2c575b8e3c56b8eb15d2a86", + "CRIctlSum": "e258f4607a89b8d44c700036e636dd42cc3e2ed27a3bb13beef736f80f64f10b7974c01259a66131d3f7b44ed0c61b1ca0ea91597c416a9c095c432de5112d44", + "CNISum": "8f2cbee3b5f94d59f919054dccfe99a8e3db5473b553d91da8af4763e811138533e05df4dbeab16b3f774852b4184a7994968f5e036a3f531ad1ac4620d10ede", + "KubectlSum": "a93b2ca067629cb1fe9cbf1af1a195c12126488ed321e3652200d4dbfee9a577865647b7ef6bb673e1bdf08f03108b5dcb4b05812a649a0de5c7c9efc1407810", + }, + "v1.21.0": map[string]interface{}{ + "CiliumVersion": "v0.8.3", + "CNIVersion": "v0.8.7", + "CRIctlVersion": "v1.17.0", + "ReleaseVersion": "v0.4.0", + "DownloadDir": "/opt/bin", + "PodSubnet": "192.168.0.0/17", + "KubeadmSum": "0673408403a3474c868ae86109f11f9114bca7ddce204be0d169316fb3ce0edefa4b2a472ba9b8308e423e6b927d4098ac36296405570f444f39551fb1c4bbb4", + "KubeletSum": "530689c0cc32ef1830f7ae26ac10995f815043d48a905141e23a34a5e61522c4ee2ff46953648c47c5592d7c2ffa40ce90469a697f36f68475b8da5abd73f9f5", + "CRIctlSum": "e258f4607a89b8d44c700036e636dd42cc3e2ed27a3bb13beef736f80f64f10b7974c01259a66131d3f7b44ed0c61b1ca0ea91597c416a9c095c432de5112d44", + "CNISum": "8f2cbee3b5f94d59f919054dccfe99a8e3db5473b553d91da8af4763e811138533e05df4dbeab16b3f774852b4184a7994968f5e036a3f531ad1ac4620d10ede", + "KubectlSum": "9557d298146ef62ffbcf05b3591bf1ce74f345628370447a4f614b5f64e367b5bfa8e397cc4755da9ea38f1ba04c95c65c313e735550ffc3b03c197e936c3e11", + }, } plog = capnslog.NewPackageLogger("github.com/coreos/mantle", "kola/tests/kubeadm") etcdConfig = conf.ContainerLinuxConfig(` @@ -75,29 +83,33 @@ systemd: ) func init() { - for _, CNI := range CNIs { - // we need to copy the value of `CNI` to pass it - // to the `Run` method, otherwise only the last - // value of `CNI` (`cilium`) will be used in the registered - // test - cni := CNI - register.Register(®ister.Test{ - Name: fmt.Sprintf("kubeadm.%s.base", cni), - Distros: []string{"cl"}, - ExcludePlatforms: []string{"esx"}, - Run: func(c cluster.TestCluster) { - kubeadmBaseTest(c, cni) - }, - }) + for version, params := range testConfig { + for _, CNI := range CNIs { + // ugly but required to remove the reference between params and the params + // actually used by the test. + testParams := make(map[string]interface{}) + for k, v := range params { + testParams[k] = v + } + testParams["CNI"] = CNI + testParams["Release"] = version + + register.Register(®ister.Test{ + Name: fmt.Sprintf("kubeadm.%s.%s.base", version, CNI), + Distros: []string{"cl"}, + ExcludePlatforms: []string{"esx"}, + Run: func(c cluster.TestCluster) { + kubeadmBaseTest(c, testParams) + }, + }) + } } } // kubeadmBaseTest asserts that the cluster is up and running -func kubeadmBaseTest(c cluster.TestCluster, CNI string) { - board := kola.QEMUOptions.Board - params["Arch"] = strings.SplitN(board, "-", 2)[0] - params["CNI"] = CNI - kubectl, err := setup(c) +func kubeadmBaseTest(c cluster.TestCluster, params map[string]interface{}) { + params["Arch"] = strings.SplitN(kola.QEMUOptions.Board, "-", 2)[0] + kubectl, err := setup(c, params) if err != nil { c.Fatalf("unable to setup cluster: %v", err) } @@ -166,7 +178,7 @@ func render(s string, p map[string]interface{}, b bool) (*bytes.Buffer, error) { // cluster is composed by etcd node, worker and master node // it returns master node in order to have direct access on node // with kubectl installed and setup -func setup(c cluster.TestCluster) (platform.Machine, error) { +func setup(c cluster.TestCluster, params map[string]interface{}) (platform.Machine, error) { plog.Infof("creating etcd node") etcdNode, err := c.NewMachine(etcdConfig) diff --git a/kola/tests/kubeadm/templates.go b/kola/tests/kubeadm/templates.go index 6943a6f07..fa9a55bd1 100644 --- a/kola/tests/kubeadm/templates.go +++ b/kola/tests/kubeadm/templates.go @@ -261,12 +261,22 @@ curl --retry-delay 1 \ -sSL \ "https://github.com/raw/kubernetes/release/${RELEASE_VERSION}/cmd/kubepkg/templates/latest/deb/kubeadm/10-kubeadm.conf" | sed "s:/usr/bin:${DOWNLOAD_DIR}:g" > /etc/systemd/system/kubelet.service.d/10-kubeadm.conf - + + +# we get the node cgroup driver +# in order to pass the params to the +# kubelet config for both controller and worker +cgroup=$(docker info | awk '/Cgroup Driver/ { print $3}') + # we create the kubeadm config # plugin-volume-dir and flex-volume-plugin-dir are required since /usr is read-only mounted # etcd is also defined as external. The provided one has some issues with docker and selinux # (permission denied with /var/lib/etcd) so it can't boot properly cat << EOF > kubeadm-config.yaml +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration +cgroupDriver: ${cgroup} +--- apiVersion: kubeadm.k8s.io/v1beta2 kind: InitConfiguration nodeRegistration: @@ -356,6 +366,10 @@ controlPlane: nodeRegistration: kubeletExtraArgs: volume-plugin-dir: "/opt/libexec/kubernetes/kubelet-plugins/volume/exec/" +--- +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration +cgroupDriver: ${cgroup} EOF ` diff --git a/kola/tests/kubeadm/testdata/master-calico-script.sh b/kola/tests/kubeadm/testdata/master-calico-script.sh index 311c78d4e..70d2933d5 100644 --- a/kola/tests/kubeadm/testdata/master-calico-script.sh +++ b/kola/tests/kubeadm/testdata/master-calico-script.sh @@ -33,12 +33,22 @@ curl --retry-delay 1 \ -sSL \ "https://github.com/raw/kubernetes/release/${RELEASE_VERSION}/cmd/kubepkg/templates/latest/deb/kubeadm/10-kubeadm.conf" | sed "s:/usr/bin:${DOWNLOAD_DIR}:g" > /etc/systemd/system/kubelet.service.d/10-kubeadm.conf - + + +# we get the node cgroup driver +# in order to pass the params to the +# kubelet config for both controller and worker +cgroup=$(docker info | awk '/Cgroup Driver/ { print $3}') + # we create the kubeadm config # plugin-volume-dir and flex-volume-plugin-dir are required since /usr is read-only mounted # etcd is also defined as external. The provided one has some issues with docker and selinux # (permission denied with /var/lib/etcd) so it can't boot properly cat << EOF > kubeadm-config.yaml +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration +cgroupDriver: ${cgroup} +--- apiVersion: kubeadm.k8s.io/v1beta2 kind: InitConfiguration nodeRegistration: @@ -117,4 +127,8 @@ controlPlane: nodeRegistration: kubeletExtraArgs: volume-plugin-dir: "/opt/libexec/kubernetes/kubelet-plugins/volume/exec/" +--- +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration +cgroupDriver: ${cgroup} EOF diff --git a/kola/tests/kubeadm/testdata/master-cilium-script.sh b/kola/tests/kubeadm/testdata/master-cilium-script.sh index 9ee6aac76..6cfe7fb7c 100644 --- a/kola/tests/kubeadm/testdata/master-cilium-script.sh +++ b/kola/tests/kubeadm/testdata/master-cilium-script.sh @@ -33,12 +33,22 @@ curl --retry-delay 1 \ -sSL \ "https://github.com/raw/kubernetes/release/${RELEASE_VERSION}/cmd/kubepkg/templates/latest/deb/kubeadm/10-kubeadm.conf" | sed "s:/usr/bin:${DOWNLOAD_DIR}:g" > /etc/systemd/system/kubelet.service.d/10-kubeadm.conf - + + +# we get the node cgroup driver +# in order to pass the params to the +# kubelet config for both controller and worker +cgroup=$(docker info | awk '/Cgroup Driver/ { print $3}') + # we create the kubeadm config # plugin-volume-dir and flex-volume-plugin-dir are required since /usr is read-only mounted # etcd is also defined as external. The provided one has some issues with docker and selinux # (permission denied with /var/lib/etcd) so it can't boot properly cat << EOF > kubeadm-config.yaml +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration +cgroupDriver: ${cgroup} +--- apiVersion: kubeadm.k8s.io/v1beta2 kind: InitConfiguration nodeRegistration: @@ -102,4 +112,8 @@ controlPlane: nodeRegistration: kubeletExtraArgs: volume-plugin-dir: "/opt/libexec/kubernetes/kubelet-plugins/volume/exec/" +--- +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration +cgroupDriver: ${cgroup} EOF diff --git a/kola/tests/kubeadm/testdata/master-flannel-script.sh b/kola/tests/kubeadm/testdata/master-flannel-script.sh index a25ca49a0..38a119d71 100644 --- a/kola/tests/kubeadm/testdata/master-flannel-script.sh +++ b/kola/tests/kubeadm/testdata/master-flannel-script.sh @@ -33,12 +33,22 @@ curl --retry-delay 1 \ -sSL \ "https://github.com/raw/kubernetes/release/${RELEASE_VERSION}/cmd/kubepkg/templates/latest/deb/kubeadm/10-kubeadm.conf" | sed "s:/usr/bin:${DOWNLOAD_DIR}:g" > /etc/systemd/system/kubelet.service.d/10-kubeadm.conf - + + +# we get the node cgroup driver +# in order to pass the params to the +# kubelet config for both controller and worker +cgroup=$(docker info | awk '/Cgroup Driver/ { print $3}') + # we create the kubeadm config # plugin-volume-dir and flex-volume-plugin-dir are required since /usr is read-only mounted # etcd is also defined as external. The provided one has some issues with docker and selinux # (permission denied with /var/lib/etcd) so it can't boot properly cat << EOF > kubeadm-config.yaml +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration +cgroupDriver: ${cgroup} +--- apiVersion: kubeadm.k8s.io/v1beta2 kind: InitConfiguration nodeRegistration: @@ -99,4 +109,8 @@ controlPlane: nodeRegistration: kubeletExtraArgs: volume-plugin-dir: "/opt/libexec/kubernetes/kubelet-plugins/volume/exec/" +--- +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration +cgroupDriver: ${cgroup} EOF