diff --git a/kinder/ci/tools/update-workflows/config.yaml b/kinder/ci/tools/update-workflows/config.yaml index 07b3d21e..fb516be4 100644 --- a/kinder/ci/tools/update-workflows/config.yaml +++ b/kinder/ci/tools/update-workflows/config.yaml @@ -248,3 +248,15 @@ jobGroups: - ./templates/workflows/encryption-algorithm-tasks.yaml jobs: - kubernetesVersion: latest + +- name: control-plane-local-kubelet-mode + testInfraJobSpec: + targetFile: kubeadm-kinder-control-plane-local-kubelet-mode.yaml + template: ./templates/testinfra/kubeadm-kinder-control-plane-local-kubelet-mode.yaml + kinderWorkflowSpec: + targetFile: control-plane-local-kubelet-mode-{{ .KubernetesVersion }}.yaml + template: ./templates/workflows/control-plane-local-kubelet-mode.yaml + additionalFiles: + - ./templates/workflows/control-plane-local-kubelet-mode-tasks.yaml + jobs: + - kubernetesVersion: latest diff --git a/kinder/ci/tools/update-workflows/templates/testinfra/kubeadm-kinder-control-plane-local-kubelet-mode.yaml b/kinder/ci/tools/update-workflows/templates/testinfra/kubeadm-kinder-control-plane-local-kubelet-mode.yaml new file mode 100644 index 00000000..36c0d65d --- /dev/null +++ b/kinder/ci/tools/update-workflows/templates/testinfra/kubeadm-kinder-control-plane-local-kubelet-mode.yaml @@ -0,0 +1,42 @@ +- name: ci-kubernetes-e2e-kubeadm-kinder-control-plane-kubelet-local-mode-{{ dashVer .KubernetesVersion }} + cluster: eks-prow-build-cluster + interval: {{ .JobInterval }} + decorate: true + labels: + preset-dind-enabled: "true" + preset-kind-volume-mounts: "true" + annotations: + testgrid-dashboards: sig-cluster-lifecycle-kubeadm + testgrid-tab-name: kubeadm-kinder-control-plane-kubelet-local-mode-{{ dashVer .KubernetesVersion }} + testgrid-alert-email: sig-cluster-lifecycle-kubeadm-alerts@kubernetes.io + description: "OWNER: sig-cluster-lifecycle (kinder); Uses kubeadm/kinder to create a cluster using the ControlPlaneKubeletLocalMode feature gate and run the kubeadm-e2e and the conformance suites" + testgrid-num-columns-recent: "20" +{{ .AlertAnnotations }} + decoration_config: + timeout: 60m + extra_refs: + - org: kubernetes + repo: kubernetes + base_ref: {{ branchFor .KubernetesVersion }} + path_alias: k8s.io/kubernetes + - org: kubernetes + repo: kubeadm + base_ref: main + path_alias: k8s.io/kubeadm + spec: + containers: + - image: gcr.io/k8s-staging-test-infra/kubekins-e2e:{{ .TestInfraImage }}-{{ imageVer .KubernetesVersion }} + command: + - runner.sh + - "../kubeadm/kinder/ci/kinder-run.sh" + args: + - {{ .WorkflowFile }} + securityContext: + privileged: true + resources: + limits: + memory: "9000Mi" + cpu: 2000m + requests: + memory: "9000Mi" + cpu: 2000m diff --git a/kinder/ci/tools/update-workflows/templates/workflows/control-plane-local-kubelet-mode-tasks.yaml b/kinder/ci/tools/update-workflows/templates/workflows/control-plane-local-kubelet-mode-tasks.yaml new file mode 100644 index 00000000..a0b4f790 --- /dev/null +++ b/kinder/ci/tools/update-workflows/templates/workflows/control-plane-local-kubelet-mode-tasks.yaml @@ -0,0 +1,248 @@ +# IMPORTANT! this workflow is imported by control-plane-local-kubelet-mode-* workflows. +version: 1 +summary: | + This workflow implements a sequence of tasks used test the proper functioning + of the ControlPlaneKubeletLocalMode feature gate. +vars: + # vars defines default values for variable used by tasks in this workflow; + # those values might be overridden when importing this files. + kubernetesVersion: v1.31.0 + upgradeVersion: v1.31.0 + controlPlaneNodes: 3 + workerNodes: 2 + baseImage: kindest/base:v20221102-76f15095 # has containerd + image: kindest/node:test + clusterName: kinder-control-plane-local-kubelet-mode + kubeadmVerbosity: 6 +tasks: +- name: pull-base-image + description: | + pulls kindest/base image with docker in docker and all the prerequisites necessary for running kind(er) + cmd: docker + args: + - pull + - "{{ .vars.baseImage }}" +- name: add-kubernetes-versions + description: | + creates a node-image-variant by adding a Kubernetes version + cmd: kinder + args: + - build + - node-image-variant + - --base-image={{ .vars.baseImage }} + - --image={{ .vars.image }} + - --with-init-artifacts={{ .vars.kubernetesVersion }} + - --with-upgrade-artifacts={{ .vars.upgradeVersion }} + - --loglevel=debug + timeout: 15m +- name: create-cluster + description: | + create a set of nodes ready for hosting the Kubernetes cluster + cmd: kinder + args: + - create + - cluster + - --name={{ .vars.clusterName }} + - --image={{ .vars.image }} + - --control-plane-nodes={{ .vars.controlPlaneNodes }} + - --worker-nodes={{ .vars.workerNodes }} + - --loglevel=debug + timeout: 5m +- name: init + description: | + Initializes the Kubernetes cluster with version "initVersion" + by starting the boostrap control-plane nodes + cmd: kinder + args: + - do + - kubeadm-init + - --name={{ .vars.clusterName }} + - --loglevel=debug + - --kubeadm-verbosity={{ .vars.kubeadmVerbosity }} + - --kubeadm-feature-gate="ControlPlaneKubeletLocalMode=true" + - --copy-certs=auto + timeout: 5m +- name: post-init + description: | + Run commands after kubeadm init is called on a primary CP node to checks if + the kubelet's kubeconfig file points to the local apiserver. + cmd: /bin/bash + args: + - -c + - | + set -x + IP_ADDRESS="$(docker inspect --format='{{ "{{" }} .NetworkSettings.IPAddress {{ "}}" }}' {{ .vars.clusterName }}-control-plane-1)" + CMD="docker exec {{ .vars.clusterName }}-control-plane-1" + + # Ensure kubelet.conf points to the local IP. + ${CMD} grep "server: https://${IP_ADDRESS}:6443" /etc/kubernetes/kubelet.conf || exit 1 + + # Ensure exit status of 0 + exit 0 + timeout: 5m +- name: join + description: | + Join the other nodes to the Kubernetes cluster + cmd: kinder + args: + - do + - kubeadm-join + - --name={{ .vars.clusterName }} + - --loglevel=debug + - --kubeadm-verbosity={{ .vars.kubeadmVerbosity }} + - --copy-certs=auto + timeout: 10m +- name: post-join + description: | + Run commands after kubeadm join is called on all joined CP node to checks if + the kubelet's kubeconfig file points to the local apiserver. + cmd: /bin/bash + args: + - -c + - | + set -x + IP_ADDRESS="$(docker inspect --format='{{ "{{" }} .NetworkSettings.IPAddress {{ "}}" }}' {{ .vars.clusterName }}-control-plane-2)" + CMD="docker exec {{ .vars.clusterName }}-control-plane-2" + + # Ensure kubelet.conf points to the local IP. + ${CMD} grep "server: https://${IP_ADDRESS}:6443" /etc/kubernetes/kubelet.conf || exit 1 + + IP_ADDRESS="$(docker inspect --format='{{ "{{" }} .NetworkSettings.IPAddress {{ "}}" }}' {{ .vars.clusterName }}-control-plane-3)" + CMD="docker exec {{ .vars.clusterName }}-control-plane-3" + + # Ensure kubelet.conf points to the local IP. + ${CMD} grep "server: https://${IP_ADDRESS}:6443" /etc/kubernetes/kubelet.conf || exit 1 + + # Ensure exit status of 0 + exit 0 + timeout: 5m +- name: pre-upgrade + description: | + Run commands before kubeadm upgrade is called on all joined CP node to replace + the server in the kubelet's kubeconfig to point to the load balancer. + cmd: /bin/bash + args: + - -c + - | + set -x + + LOAD_BALANCER_IP_ADDRESS=$(docker inspect --format='{{ "{{" }} .NetworkSettings.IPAddress {{ "}}" }}' {{ .vars.clusterName }}-lb) + + CMD="docker exec {{ .vars.clusterName }}-control-plane-1" + ${CMD} sed -i 's@server: https://.*:6443@server: https://'${LOAD_BALANCER_IP_ADDRESS}':6443@g' /etc/kubernetes/kubelet.conf || exit 1 + + CMD="docker exec {{ .vars.clusterName }}-control-plane-2" + ${CMD} sed -i 's@server: https://.*:6443@server: https://'${LOAD_BALANCER_IP_ADDRESS}':6443@g' /etc/kubernetes/kubelet.conf || exit 1 + + CMD="docker exec {{ .vars.clusterName }}-control-plane-3" + ${CMD} sed -i 's@server: https://.*:6443@server: https://'${LOAD_BALANCER_IP_ADDRESS}':6443@g' /etc/kubernetes/kubelet.conf || exit 1 + + # Ensure exit status of 0 + exit 0 + timeout: 5m +- name: upgrade + description: | + upgrades the cluster to Kubernetes "upgradeVersion" + cmd: kinder + args: + - do + - kubeadm-upgrade + - --upgrade-version={{ .vars.upgradeVersion }} + - --name={{ .vars.clusterName }} + - --loglevel=debug + - --kubeadm-verbosity={{ .vars.kubeadmVerbosity }} + timeout: 15m +- name: post-upgrade + description: | + Run commands after kubeadm upgrade is called on all joined CP node to checks if + the kubelet's kubeconfig file points to the local apiserver. + cmd: /bin/bash + args: + - -c + - | + set -x + + IP_ADDRESS="$(docker inspect --format='{{ "{{" }} .NetworkSettings.IPAddress {{ "}}" }}' {{ .vars.clusterName }}-control-plane-1)" + CMD="docker exec {{ .vars.clusterName }}-control-plane-1" + ${CMD} grep "server: https://${IP_ADDRESS}:6443" /etc/kubernetes/kubelet.conf || exit 1 + + IP_ADDRESS="$(docker inspect --format='{{ "{{" }} .NetworkSettings.IPAddress {{ "}}" }}' {{ .vars.clusterName }}-control-plane-2)" + CMD="docker exec {{ .vars.clusterName }}-control-plane-2" + ${CMD} grep "server: https://${IP_ADDRESS}:6443" /etc/kubernetes/kubelet.conf || exit 1 + + IP_ADDRESS="$(docker inspect --format='{{ "{{" }} .NetworkSettings.IPAddress {{ "}}" }}' {{ .vars.clusterName }}-control-plane-3)" + CMD="docker exec {{ .vars.clusterName }}-control-plane-3" + ${CMD} grep "server: https://${IP_ADDRESS}:6443" /etc/kubernetes/kubelet.conf || exit 1 + + # Ensure exit status of 0 + exit 0 + timeout: 5m +- name: cluster-info + description: | + Runs cluster-info + cmd: kinder + args: + - do + - cluster-info + - --name={{ .vars.clusterName }} + - --loglevel=debug +- name: e2e-kubeadm + description: | + Runs kubeadm e2e tests + cmd: kinder + args: + - test + - e2e-kubeadm + - --test-flags=--report-dir={{ .env.ARTIFACTS }} --report-prefix=e2e-kubeadm + - --name={{ .vars.clusterName }} + - --loglevel=debug + timeout: 10m +- name: e2e + description: | + Runs Kubernetes e2e test (conformance) + cmd: kinder + args: + - test + - e2e + - --test-flags=--report-dir={{ .env.ARTIFACTS }} --report-prefix=e2e + - --parallel + - --name={{ .vars.clusterName }} + - --loglevel=debug + timeout: 35m +- name: get-logs + description: | + Collects all the test logs + cmd: kinder + args: + - export + - logs + - --loglevel=debug + - --name={{ .vars.clusterName }} + - "{{ .env.ARTIFACTS }}" + force: true + timeout: 5m + # kind export log is know to be flaky, so we are temporary ignoring errors in order + # to make the test pass in case everything else passed + # see https://github.com/kubernetes-sigs/kind/issues/456 + ignoreError: true +- name: reset + description: | + Exec kubeadm reset + cmd: kinder + args: + - do + - kubeadm-reset + - --name={{ .vars.clusterName }} + - --loglevel=debug + - --kubeadm-verbosity={{ .vars.kubeadmVerbosity }} + force: true +- name: delete + description: | + Deletes the cluster + cmd: kinder + args: + - delete + - cluster + - --name={{ .vars.clusterName }} + - --loglevel=debug + force: true diff --git a/kinder/ci/tools/update-workflows/templates/workflows/control-plane-local-kubelet-mode.yaml b/kinder/ci/tools/update-workflows/templates/workflows/control-plane-local-kubelet-mode.yaml new file mode 100644 index 00000000..a9d1038c --- /dev/null +++ b/kinder/ci/tools/update-workflows/templates/workflows/control-plane-local-kubelet-mode.yaml @@ -0,0 +1,11 @@ +version: 1 +summary: | + This workflow tests the proper functioning of the {{ .KubernetesVersion }} version of both kubeadm and Kubernetes using + the ControlPlaneKubeletLocalMode feature gate. + test grid > https://testgrid.k8s.io/sig-cluster-lifecycle-kubeadm#kubeadm-kinder-control-plane-local-kubelet-mode{{ dashVer .KubernetesVersion }} + config > https://git.k8s.io/test-infra/config/jobs/kubernetes/sig-cluster-lifecycle/{{ .TargetFile }} +vars: + kubernetesVersion: "\{\{ resolve `ci/{{ ciLabelFor .KubernetesVersion }}` \}\}" + upgradeVersion: "\{\{ resolve `ci/{{ ciLabelFor .KubernetesVersion }}` \}\}" +tasks: +- import: control-plane-local-kubelet-mode-tasks.yaml diff --git a/kinder/ci/workflows/control-plane-local-kubelet-mode-latest.yaml b/kinder/ci/workflows/control-plane-local-kubelet-mode-latest.yaml new file mode 100644 index 00000000..4ecc6dfb --- /dev/null +++ b/kinder/ci/workflows/control-plane-local-kubelet-mode-latest.yaml @@ -0,0 +1,12 @@ +# AUTOGENERATED by https://git.k8s.io/kubeadm/kinder/ci/tools/update-workflows +version: 1 +summary: | + This workflow tests the proper functioning of the latest version of both kubeadm and Kubernetes using + the ControlPlaneKubeletLocalMode feature gate. + test grid > https://testgrid.k8s.io/sig-cluster-lifecycle-kubeadm#kubeadm-kinder-control-plane-local-kubelet-modelatest + config > https://git.k8s.io/test-infra/config/jobs/kubernetes/sig-cluster-lifecycle/kubeadm-kinder-control-plane-local-kubelet-mode.yaml +vars: + kubernetesVersion: "{{ resolve `ci/latest` }}" + upgradeVersion: "{{ resolve `ci/latest` }}" +tasks: +- import: control-plane-local-kubelet-mode-tasks.yaml diff --git a/kinder/ci/workflows/control-plane-local-kubelet-mode-tasks.yaml b/kinder/ci/workflows/control-plane-local-kubelet-mode-tasks.yaml new file mode 100644 index 00000000..620bf745 --- /dev/null +++ b/kinder/ci/workflows/control-plane-local-kubelet-mode-tasks.yaml @@ -0,0 +1,249 @@ +# AUTOGENERATED by https://git.k8s.io/kubeadm/kinder/ci/tools/update-workflows +# IMPORTANT! this workflow is imported by control-plane-local-kubelet-mode-* workflows. +version: 1 +summary: | + This workflow implements a sequence of tasks used test the proper functioning + of the ControlPlaneKubeletLocalMode feature gate. +vars: + # vars defines default values for variable used by tasks in this workflow; + # those values might be overridden when importing this files. + kubernetesVersion: v1.31.0 + upgradeVersion: v1.31.0 + controlPlaneNodes: 3 + workerNodes: 2 + baseImage: kindest/base:v20221102-76f15095 # has containerd + image: kindest/node:test + clusterName: kinder-control-plane-local-kubelet-mode + kubeadmVerbosity: 6 +tasks: +- name: pull-base-image + description: | + pulls kindest/base image with docker in docker and all the prerequisites necessary for running kind(er) + cmd: docker + args: + - pull + - "{{ .vars.baseImage }}" +- name: add-kubernetes-versions + description: | + creates a node-image-variant by adding a Kubernetes version + cmd: kinder + args: + - build + - node-image-variant + - --base-image={{ .vars.baseImage }} + - --image={{ .vars.image }} + - --with-init-artifacts={{ .vars.kubernetesVersion }} + - --with-upgrade-artifacts={{ .vars.upgradeVersion }} + - --loglevel=debug + timeout: 15m +- name: create-cluster + description: | + create a set of nodes ready for hosting the Kubernetes cluster + cmd: kinder + args: + - create + - cluster + - --name={{ .vars.clusterName }} + - --image={{ .vars.image }} + - --control-plane-nodes={{ .vars.controlPlaneNodes }} + - --worker-nodes={{ .vars.workerNodes }} + - --loglevel=debug + timeout: 5m +- name: init + description: | + Initializes the Kubernetes cluster with version "initVersion" + by starting the boostrap control-plane nodes + cmd: kinder + args: + - do + - kubeadm-init + - --name={{ .vars.clusterName }} + - --loglevel=debug + - --kubeadm-verbosity={{ .vars.kubeadmVerbosity }} + - --kubeadm-feature-gate="ControlPlaneKubeletLocalMode=true" + - --copy-certs=auto + timeout: 5m +- name: post-init + description: | + Run commands after kubeadm init is called on a primary CP node to checks if + the kubelet's kubeconfig file points to the local apiserver. + cmd: /bin/bash + args: + - -c + - | + set -x + IP_ADDRESS="$(docker inspect --format='{{ "{{" }} .NetworkSettings.IPAddress {{ "}}" }}' {{ .vars.clusterName }}-control-plane-1)" + CMD="docker exec {{ .vars.clusterName }}-control-plane-1" + + # Ensure kubelet.conf points to the local IP. + ${CMD} grep "server: https://${IP_ADDRESS}:6443" /etc/kubernetes/kubelet.conf || exit 1 + + # Ensure exit status of 0 + exit 0 + timeout: 5m +- name: join + description: | + Join the other nodes to the Kubernetes cluster + cmd: kinder + args: + - do + - kubeadm-join + - --name={{ .vars.clusterName }} + - --loglevel=debug + - --kubeadm-verbosity={{ .vars.kubeadmVerbosity }} + - --copy-certs=auto + timeout: 10m +- name: post-join + description: | + Run commands after kubeadm join is called on all joined CP node to checks if + the kubelet's kubeconfig file points to the local apiserver. + cmd: /bin/bash + args: + - -c + - | + set -x + IP_ADDRESS="$(docker inspect --format='{{ "{{" }} .NetworkSettings.IPAddress {{ "}}" }}' {{ .vars.clusterName }}-control-plane-2)" + CMD="docker exec {{ .vars.clusterName }}-control-plane-2" + + # Ensure kubelet.conf points to the local IP. + ${CMD} grep "server: https://${IP_ADDRESS}:6443" /etc/kubernetes/kubelet.conf || exit 1 + + IP_ADDRESS="$(docker inspect --format='{{ "{{" }} .NetworkSettings.IPAddress {{ "}}" }}' {{ .vars.clusterName }}-control-plane-3)" + CMD="docker exec {{ .vars.clusterName }}-control-plane-3" + + # Ensure kubelet.conf points to the local IP. + ${CMD} grep "server: https://${IP_ADDRESS}:6443" /etc/kubernetes/kubelet.conf || exit 1 + + # Ensure exit status of 0 + exit 0 + timeout: 5m +- name: pre-upgrade + description: | + Run commands before kubeadm upgrade is called on all joined CP node to replace + the server in the kubelet's kubeconfig to point to the load balancer. + cmd: /bin/bash + args: + - -c + - | + set -x + + LOAD_BALANCER_IP_ADDRESS=$(docker inspect --format='{{ "{{" }} .NetworkSettings.IPAddress {{ "}}" }}' {{ .vars.clusterName }}-lb) + + CMD="docker exec {{ .vars.clusterName }}-control-plane-1" + ${CMD} sed -i 's@server: https://.*:6443@server: https://'${LOAD_BALANCER_IP_ADDRESS}':6443@g' /etc/kubernetes/kubelet.conf || exit 1 + + CMD="docker exec {{ .vars.clusterName }}-control-plane-2" + ${CMD} sed -i 's@server: https://.*:6443@server: https://'${LOAD_BALANCER_IP_ADDRESS}':6443@g' /etc/kubernetes/kubelet.conf || exit 1 + + CMD="docker exec {{ .vars.clusterName }}-control-plane-3" + ${CMD} sed -i 's@server: https://.*:6443@server: https://'${LOAD_BALANCER_IP_ADDRESS}':6443@g' /etc/kubernetes/kubelet.conf || exit 1 + + # Ensure exit status of 0 + exit 0 + timeout: 5m +- name: upgrade + description: | + upgrades the cluster to Kubernetes "upgradeVersion" + cmd: kinder + args: + - do + - kubeadm-upgrade + - --upgrade-version={{ .vars.upgradeVersion }} + - --name={{ .vars.clusterName }} + - --loglevel=debug + - --kubeadm-verbosity={{ .vars.kubeadmVerbosity }} + timeout: 15m +- name: post-upgrade + description: | + Run commands after kubeadm upgrade is called on all joined CP node to checks if + the kubelet's kubeconfig file points to the local apiserver. + cmd: /bin/bash + args: + - -c + - | + set -x + + IP_ADDRESS="$(docker inspect --format='{{ "{{" }} .NetworkSettings.IPAddress {{ "}}" }}' {{ .vars.clusterName }}-control-plane-1)" + CMD="docker exec {{ .vars.clusterName }}-control-plane-1" + ${CMD} grep "server: https://${IP_ADDRESS}:6443" /etc/kubernetes/kubelet.conf || exit 1 + + IP_ADDRESS="$(docker inspect --format='{{ "{{" }} .NetworkSettings.IPAddress {{ "}}" }}' {{ .vars.clusterName }}-control-plane-2)" + CMD="docker exec {{ .vars.clusterName }}-control-plane-2" + ${CMD} grep "server: https://${IP_ADDRESS}:6443" /etc/kubernetes/kubelet.conf || exit 1 + + IP_ADDRESS="$(docker inspect --format='{{ "{{" }} .NetworkSettings.IPAddress {{ "}}" }}' {{ .vars.clusterName }}-control-plane-3)" + CMD="docker exec {{ .vars.clusterName }}-control-plane-3" + ${CMD} grep "server: https://${IP_ADDRESS}:6443" /etc/kubernetes/kubelet.conf || exit 1 + + # Ensure exit status of 0 + exit 0 + timeout: 5m +- name: cluster-info + description: | + Runs cluster-info + cmd: kinder + args: + - do + - cluster-info + - --name={{ .vars.clusterName }} + - --loglevel=debug +- name: e2e-kubeadm + description: | + Runs kubeadm e2e tests + cmd: kinder + args: + - test + - e2e-kubeadm + - --test-flags=--report-dir={{ .env.ARTIFACTS }} --report-prefix=e2e-kubeadm + - --name={{ .vars.clusterName }} + - --loglevel=debug + timeout: 10m +- name: e2e + description: | + Runs Kubernetes e2e test (conformance) + cmd: kinder + args: + - test + - e2e + - --test-flags=--report-dir={{ .env.ARTIFACTS }} --report-prefix=e2e + - --parallel + - --name={{ .vars.clusterName }} + - --loglevel=debug + timeout: 35m +- name: get-logs + description: | + Collects all the test logs + cmd: kinder + args: + - export + - logs + - --loglevel=debug + - --name={{ .vars.clusterName }} + - "{{ .env.ARTIFACTS }}" + force: true + timeout: 5m + # kind export log is know to be flaky, so we are temporary ignoring errors in order + # to make the test pass in case everything else passed + # see https://github.com/kubernetes-sigs/kind/issues/456 + ignoreError: true +- name: reset + description: | + Exec kubeadm reset + cmd: kinder + args: + - do + - kubeadm-reset + - --name={{ .vars.clusterName }} + - --loglevel=debug + - --kubeadm-verbosity={{ .vars.kubeadmVerbosity }} + force: true +- name: delete + description: | + Deletes the cluster + cmd: kinder + args: + - delete + - cluster + - --name={{ .vars.clusterName }} + - --loglevel=debug + force: true