From fa5ac66dc1dd192965e0e5761c6849afeb1eb28f Mon Sep 17 00:00:00 2001 From: "Lubomir I. Ivanov" Date: Mon, 1 Jul 2024 15:19:40 +0300 Subject: [PATCH] kinder: use "--config" for kubeadm "reset" and "upgrade" When executing "kubeadm reset" or "kubeadm upgrade", if the kubeadm version is 1.31 (or pre-release) pass the "--config" flag with the v1beta4 kinds. Else, continue to call the flags for a given command. Additionally: - Call "kubeadm upgrade diff" on the primary CP node after calling "upgrade plan". - Fallback to the /kinder/upgrade/version during upgrade if the provided version is different. --- kinder/pkg/cluster/manager/actions/actions.go | 4 +- .../cluster/manager/actions/kubeadm-config.go | 41 ++++- .../cluster/manager/actions/kubeadm-reset.go | 26 ++- .../manager/actions/kubeadm-upgrade.go | 162 +++++++++++++----- kinder/pkg/kubeadm/componentpatches.go | 30 +++- kinder/pkg/kubeadm/config.go | 25 +++ 6 files changed, 228 insertions(+), 60 deletions(-) diff --git a/kinder/pkg/cluster/manager/actions/actions.go b/kinder/pkg/cluster/manager/actions/actions.go index 2d1c8189..64305b53 100644 --- a/kinder/pkg/cluster/manager/actions/actions.go +++ b/kinder/pkg/cluster/manager/actions/actions.go @@ -40,7 +40,7 @@ var actionRegistry = map[string]func(*status.Cluster, *RunOptions) error{ "kubeadm-config": func(c *status.Cluster, flags *RunOptions) error { // Nb. this action is invoked automatically at kubeadm init/join time, but it is possible // to invoke it separately as well - return KubeadmConfig(c, flags.kubeadmConfigVersion, flags.copyCertsMode, flags.discoveryMode, flags.featureGate, flags.encryptionAlgorithm, c.K8sNodes().EligibleForActions()...) + return KubeadmConfig(c, flags.kubeadmConfigVersion, flags.copyCertsMode, flags.discoveryMode, flags.featureGate, flags.encryptionAlgorithm, flags.upgradeVersion, c.K8sNodes().EligibleForActions()...) }, "kubeadm-init": func(c *status.Cluster, flags *RunOptions) error { return KubeadmInit(c, flags.usePhases, flags.copyCertsMode, flags.kubeadmConfigVersion, flags.patchesDir, flags.ignorePreflightErrors, flags.featureGate, flags.encryptionAlgorithm, flags.wait, flags.vLevel) @@ -49,7 +49,7 @@ var actionRegistry = map[string]func(*status.Cluster, *RunOptions) error{ return KubeadmJoin(c, flags.usePhases, flags.copyCertsMode, flags.discoveryMode, flags.kubeadmConfigVersion, flags.patchesDir, flags.ignorePreflightErrors, flags.wait, flags.vLevel) }, "kubeadm-upgrade": func(c *status.Cluster, flags *RunOptions) error { - return KubeadmUpgrade(c, flags.upgradeVersion, flags.patchesDir, flags.featureGate, flags.wait, flags.vLevel) + return KubeadmUpgrade(c, flags.upgradeVersion, flags.patchesDir, flags.wait, flags.vLevel) }, "kubeadm-reset": func(c *status.Cluster, flags *RunOptions) error { return KubeadmReset(c, flags.vLevel) diff --git a/kinder/pkg/cluster/manager/actions/kubeadm-config.go b/kinder/pkg/cluster/manager/actions/kubeadm-config.go index 25095809..f9cf094c 100644 --- a/kinder/pkg/cluster/manager/actions/kubeadm-config.go +++ b/kinder/pkg/cluster/manager/actions/kubeadm-config.go @@ -23,7 +23,9 @@ import ( "github.com/pkg/errors" log "github.com/sirupsen/logrus" + "k8s.io/apimachinery/pkg/util/version" "k8s.io/client-go/tools/clientcmd" + "k8s.io/kubeadm/kinder/pkg/cluster/status" "k8s.io/kubeadm/kinder/pkg/constants" "k8s.io/kubeadm/kinder/pkg/cri/nodes" @@ -42,7 +44,7 @@ type kubeadmConfigOptions struct { // to invoke it separately as well. func KubeadmInitConfig(c *status.Cluster, kubeadmConfigVersion string, copyCertsMode CopyCertsMode, featureGate, encryptionAlgorithm string, nodes ...*status.Node) error { // defaults everything not relevant for the Init Config - return KubeadmConfig(c, kubeadmConfigVersion, copyCertsMode, TokenDiscovery, featureGate, encryptionAlgorithm, nodes...) + return KubeadmConfig(c, kubeadmConfigVersion, copyCertsMode, TokenDiscovery, featureGate, encryptionAlgorithm, nil, nodes...) } // KubeadmJoinConfig action writes the JoinConfiguration into /kind/kubeadm.conf file on all the K8s nodes in the cluster. @@ -50,13 +52,23 @@ func KubeadmInitConfig(c *status.Cluster, kubeadmConfigVersion string, copyCerts // to invoke it separately as well. func KubeadmJoinConfig(c *status.Cluster, kubeadmConfigVersion string, copyCertsMode CopyCertsMode, discoveryMode DiscoveryMode, nodes ...*status.Node) error { // defaults everything not relevant for the join Config - return KubeadmConfig(c, kubeadmConfigVersion, copyCertsMode, discoveryMode, "" /* feature-gates */, "" /* encryptionAlgorithm */, nodes...) + return KubeadmConfig(c, kubeadmConfigVersion, copyCertsMode, discoveryMode, "", "", nil, nodes...) +} + +// KubeadmUpgradeConfig action writes the UpgradeConfiguration into /kind/kubeadm.conf file on all the K8s nodes in the cluster. +func KubeadmUpgradeConfig(c *status.Cluster, upgradeVersion *version.Version, nodes ...*status.Node) error { + return KubeadmConfig(c, "", "", "", "", "", upgradeVersion, nodes...) +} + +// KubeadmResetConfig action writes the UpgradeConfiguration into /kind/kubeadm.conf file on all the K8s nodes in the cluster. +func KubeadmResetConfig(c *status.Cluster, nodes ...*status.Node) error { + return KubeadmConfig(c, "", "", "", "", "", nil, nodes...) } // KubeadmConfig action writes the /kind/kubeadm.conf file on all the K8s nodes in the cluster. // Please note that this action is automatically executed at create time, but it is possible // to invoke it separately as well. -func KubeadmConfig(c *status.Cluster, kubeadmConfigVersion string, copyCertsMode CopyCertsMode, discoveryMode DiscoveryMode, featureGate, encryptionAlgorithm string, nodes ...*status.Node) error { +func KubeadmConfig(c *status.Cluster, kubeadmConfigVersion string, copyCertsMode CopyCertsMode, discoveryMode DiscoveryMode, featureGate, encryptionAlgorithm string, upgradeVersion *version.Version, nodes ...*status.Node) error { cp1 := c.BootstrapControlPlane() // get installed kubernetes version from the node image @@ -95,6 +107,19 @@ func KubeadmConfig(c *status.Cluster, kubeadmConfigVersion string, copyCertsMode featureGateValue = split[1] } + if copyCertsMode == "" { + copyCertsMode = CopyCertsModeAuto + } + + if discoveryMode == "" { + discoveryMode = TokenDiscovery + } + + // Use a placeholder upgrade version for non-upgrade actions. + if upgradeVersion == nil { + upgradeVersion = version.MustParseSemantic("v1.0.0") + } + // create configData with all the configurations supported by the kubeadm config template implemented in kind configData := kubeadm.ConfigData{ ClusterName: c.Name(), @@ -109,6 +134,7 @@ func KubeadmConfig(c *status.Cluster, kubeadmConfigVersion string, copyCertsMode FeatureGateName: featureGateName, FeatureGateValue: featureGateValue, EncryptionAlgorithm: encryptionAlgorithm, + UpgradeVersion: fmt.Sprintf("v%s", upgradeVersion.String()), } // create configOptions with all the kinder flags that impact on the kubeadm config generation @@ -179,7 +205,10 @@ func writeKubeadmConfig(c *status.Cluster, n *status.Node, data kubeadm.ConfigDa return errors.Wrap(err, "failed to generate kubeadm config content") } - log.Debugf("generated config:\n%s", kubeadmConfig) + log.Debug("generating config...") + if log.GetLevel() == log.DebugLevel { + fmt.Print(kubeadmConfig) + } // copy the config to the node if err := n.WriteFile(constants.KubeadmConfigPath, []byte(kubeadmConfig)); err != nil { @@ -329,6 +358,8 @@ func getKubeadmConfig(c *status.Cluster, n *status.Node, data kubeadm.ConfigData return selectYamlFramentByKind(patched, "ClusterConfiguration", "InitConfiguration", + "UpgradeConfiguration", + "ResetConfiguration", "KubeletConfiguration", "KubeProxyConfiguration"), nil } @@ -336,6 +367,8 @@ func getKubeadmConfig(c *status.Cluster, n *status.Node, data kubeadm.ConfigData // otherwise select only the JoinConfiguration return selectYamlFramentByKind(patched, "JoinConfiguration", + "UpgradeCOnfiguration", + "ResetConfiguration", ), nil } diff --git a/kinder/pkg/cluster/manager/actions/kubeadm-reset.go b/kinder/pkg/cluster/manager/actions/kubeadm-reset.go index 17421a49..961baf8e 100644 --- a/kinder/pkg/cluster/manager/actions/kubeadm-reset.go +++ b/kinder/pkg/cluster/manager/actions/kubeadm-reset.go @@ -19,16 +19,36 @@ package actions import ( "fmt" + "github.com/pkg/errors" + "k8s.io/kubeadm/kinder/pkg/cluster/status" + "k8s.io/kubeadm/kinder/pkg/constants" + "k8s.io/kubeadm/kinder/pkg/kubeadm" ) // KubeadmReset executes the kubeadm reset workflow func KubeadmReset(c *status.Cluster, vLevel int) error { //TODO: implements kubeadm reset with phases for _, n := range c.K8sNodes().EligibleForActions() { - if err := n.Command( - "kubeadm", "reset", "--force", fmt.Sprintf("--v=%d", vLevel), - ).RunWithEcho(); err != nil { + flags := []string{"reset", fmt.Sprintf("--v=%d", vLevel)} + + // After upgrade, the 'kubeadm version' should return the version of the kubeadm used + // to perform the upgrade. Use this version to determine if v1beta4 is enabled. If yes, + // use ResetConfiguration with a 'force: true', else just use the '--force' flag. + v, err := n.KubeadmVersion() + if err != nil { + return errors.Wrap(err, "could not obtain the kubeadm version before calling 'kubeadm reset'") + } + if kubeadm.GetKubeadmConfigVersion(v) == "v1beta4" { + if err := KubeadmResetConfig(c, n); err != nil { + return errors.Wrap(err, "could not write kubeadm config before calling 'kubeadm reset'") + } + flags = append(flags, "--config", constants.KubeadmConfigPath) + } else { + flags = append(flags, "--force") + } + + if err := n.Command("kubeadm", flags...).RunWithEcho(); err != nil { return err } } diff --git a/kinder/pkg/cluster/manager/actions/kubeadm-upgrade.go b/kinder/pkg/cluster/manager/actions/kubeadm-upgrade.go index 71d91412..ed6e80bf 100644 --- a/kinder/pkg/cluster/manager/actions/kubeadm-upgrade.go +++ b/kinder/pkg/cluster/manager/actions/kubeadm-upgrade.go @@ -22,11 +22,13 @@ import ( "time" "github.com/pkg/errors" + log "github.com/sirupsen/logrus" - K8sVersion "k8s.io/apimachinery/pkg/util/version" + "k8s.io/apimachinery/pkg/util/version" "k8s.io/kubeadm/kinder/pkg/cluster/status" "k8s.io/kubeadm/kinder/pkg/constants" "k8s.io/kubeadm/kinder/pkg/cri/nodes" + "k8s.io/kubeadm/kinder/pkg/kubeadm" ) // KubeadmUpgrade executes the kubeadm upgrade workflow, including also deployment of new @@ -35,30 +37,68 @@ import ( // // The implementation assumes that the kubeadm/kubelet/kubectl binaries and all the necessary images // for the new kubernetes version are available in the /kinder/upgrade/{version} folder. -func KubeadmUpgrade(c *status.Cluster, upgradeVersion *K8sVersion.Version, patchesDir string, featureGate string, wait time.Duration, vLevel int) (err error) { +func KubeadmUpgrade(c *status.Cluster, upgradeVersion *version.Version, patchesDir string, wait time.Duration, vLevel int) (err error) { if upgradeVersion == nil { return errors.New("kubeadm-upgrade actions requires the --upgrade-version parameter to be set") } - preloadUpgradeImages(c, upgradeVersion) nodeList := c.K8sNodes().EligibleForActions() for _, n := range nodeList { + preloadNodeUpgradeImages(n, upgradeVersion) + if err := copyPatchesToNode(n, patchesDir); err != nil { return err } + // Check if the upgrade version provided on the CLI is different from what is on the node image. + // If there is a difference print a warning and fallback to what is on the node image. + // This is useful in debug scenarios where the ci/latest version label changed during + // debugging a particular workflow. + versionPath := filepath.Join("/kinder", "upgrade", "version") + out, err := n.Command("cat", versionPath).Silent().RunAndCapture() + if err != nil { + return errors.Wrapf(err, "could not compare %s file before upgrade", versionPath) + } + if len(out) != 1 { + return errors.Errorf("expected %s to have 1 line, got %d", versionPath, len(out)) + } + nodeVersion := version.MustParseSemantic(out[0]) + cmp, err := nodeVersion.Compare(upgradeVersion.String()) + if err != nil { + return errors.Wrapf(err, "cannot compare %s to provided upgrade version", versionPath) + } + if cmp != 0 { + log.Warnf("provided upgrade version is %s, but the node has %s, using the node version", + upgradeVersion, nodeVersion) + upgradeVersion = nodeVersion + } + if err := upgradeKubeadmBinary(n, upgradeVersion); err != nil { return err } + // prepares the kubeadm config on this node + if err := KubeadmUpgradeConfig(c, upgradeVersion, n); err != nil { + return err + } + + v, err := n.KubeadmVersion() + if err != nil { + errors.Wrap(err, "could not obtain the kubeadm version before calling kubeadm reset") + } + kubeadmConfigVersion := kubeadm.GetKubeadmConfigVersion(v) + if n.Name() == c.BootstrapControlPlane().Name() { - if err := kubeadmUpgradePlan(c, n, upgradeVersion, featureGate, vLevel); err != nil { + if err := kubeadmUpgradePlan(c, n, kubeadmConfigVersion, upgradeVersion, vLevel); err != nil { + return err + } + if err := kubeadmUpgradeDiff(c, n, kubeadmConfigVersion, upgradeVersion, vLevel); err != nil { return err } - err = kubeadmUpgradeApply(c, n, upgradeVersion, patchesDir, featureGate, wait, vLevel) + err = kubeadmUpgradeApply(c, n, kubeadmConfigVersion, upgradeVersion, patchesDir, wait, vLevel) } else { - err = kubeadmUpgradeNode(c, n, upgradeVersion, patchesDir, wait, vLevel) + err = kubeadmUpgradeNode(c, n, kubeadmConfigVersion, upgradeVersion, patchesDir, wait, vLevel) } if err != nil { return err @@ -74,41 +114,39 @@ func KubeadmUpgrade(c *status.Cluster, upgradeVersion *K8sVersion.Version, patch return nil } -func preloadUpgradeImages(c *status.Cluster, upgradeVersion *K8sVersion.Version) { +func preloadNodeUpgradeImages(n *status.Node, upgradeVersion *version.Version) { srcFolder := filepath.Join("/kinder", "upgrade", fmt.Sprintf("v%s", upgradeVersion)) // load images cached on the node into CRI engine // this should be executed on all nodes before running kubeadm upgrade apply in order to // get everything in place when kubeadm creates pre-pull daemonsets (if not, this might be blocking in case of // images not available on public registry, like e.g. pre-release images) - for _, n := range c.K8sNodes() { - n.Infof("pre-loading images required for the upgrade") - nodeCRI, err := n.CRI() - if err != nil { - fmt.Printf("error detecting CRI: %v", err) - continue - } + n.Infof("pre-loading images required for the upgrade") + nodeCRI, err := n.CRI() + if err != nil { + fmt.Printf("error detecting CRI: %v", err) + return + } - actionHelper, err := nodes.NewActionHelper(nodeCRI) - if err != nil { - fmt.Printf("error creating the action helper: %v", err) - continue - } + actionHelper, err := nodes.NewActionHelper(nodeCRI) + if err != nil { + fmt.Printf("error creating the action helper: %v", err) + return + } - if err := actionHelper.PreLoadUpgradeImages(n, srcFolder); err != nil { - fmt.Printf("error PreLoadUpgradeImages: %v", err) - continue - } + if err := actionHelper.PreLoadUpgradeImages(n, srcFolder); err != nil { + fmt.Printf("error PreLoadUpgradeImages: %v", err) + return + } - // checks pre-loaded images available on the node (this will report missing images, if any) - if err := checkImagesForVersion(n, upgradeVersion.String()); err != nil { - fmt.Printf("error ReportImages: %v", err) - continue - } + // checks pre-loaded images available on the node (this will report missing images, if any) + if err := checkImagesForVersion(n, upgradeVersion.String()); err != nil { + fmt.Printf("error ReportImages: %v", err) + return } } -func upgradeKubeadmBinary(n *status.Node, upgradeVersion *K8sVersion.Version) error { +func upgradeKubeadmBinary(n *status.Node, upgradeVersion *version.Version) error { n.Infof("upgrade kubeadm binary") srcFolder := filepath.Join("/kinder", "upgrade", fmt.Sprintf("v%s", upgradeVersion)) @@ -123,15 +161,37 @@ func upgradeKubeadmBinary(n *status.Node, upgradeVersion *K8sVersion.Version) er return nil } -func kubeadmUpgradePlan(c *status.Cluster, cp1 *status.Node, upgradeVersion *K8sVersion.Version, featureGate string, vLevel int) error { +func kubeadmUpgradeDiff(c *status.Cluster, cp1 *status.Node, configVersion string, upgradeVersion *version.Version, vLevel int) error { + diffArgs := []string{ + "upgrade", "diff", fmt.Sprintf("--v=%d", vLevel), + } + + if configVersion == "v1beta4" { + diffArgs = append(diffArgs, "--config", constants.KubeadmConfigPath) + } else { + diffArgs = append(diffArgs, fmt.Sprintf("v%s", upgradeVersion)) + } + + if err := cp1.Command( + "kubeadm", diffArgs..., + ).RunWithEcho(); err != nil { + return err + } + return nil +} + +func kubeadmUpgradePlan(c *status.Cluster, cp1 *status.Node, configVersion string, upgradeVersion *version.Version, vLevel int) error { planArgs := []string{ - "upgrade", "plan", fmt.Sprintf("v%s", upgradeVersion), - "--allow-experimental-upgrades", "--allow-release-candidate-upgrades", - fmt.Sprintf("--v=%d", vLevel), + "upgrade", "plan", fmt.Sprintf("--v=%d", vLevel), } - if len(featureGate) > 0 { - planArgs = append(planArgs, fmt.Sprintf("--feature-gates=%s", featureGate)) + + if configVersion == "v1beta4" { + planArgs = append(planArgs, "--config", constants.KubeadmConfigPath) + } else { + planArgs = append(planArgs, "--allow-experimental-upgrades", "--allow-release-candidate-upgrades", + fmt.Sprintf("v%s", upgradeVersion)) } + if err := cp1.Command( "kubeadm", planArgs..., ).RunWithEcho(); err != nil { @@ -140,16 +200,20 @@ func kubeadmUpgradePlan(c *status.Cluster, cp1 *status.Node, upgradeVersion *K8s return nil } -func kubeadmUpgradeApply(c *status.Cluster, cp1 *status.Node, upgradeVersion *K8sVersion.Version, patchesDir string, featureGate string, wait time.Duration, vLevel int) error { +func kubeadmUpgradeApply(c *status.Cluster, cp1 *status.Node, configVersion string, upgradeVersion *version.Version, patchesDir string, wait time.Duration, vLevel int) error { applyArgs := []string{ - "upgrade", "apply", "-f", fmt.Sprintf("v%s", upgradeVersion), fmt.Sprintf("--v=%d", vLevel), - } - if patchesDir != "" { - applyArgs = append(applyArgs, "--patches", constants.PatchesDir) + "upgrade", "apply", fmt.Sprintf("--v=%d", vLevel), } - if len(featureGate) > 0 { - applyArgs = append(applyArgs, fmt.Sprintf("--feature-gates=%s", featureGate)) + + if configVersion == "v1beta4" { + applyArgs = append(applyArgs, "--config", constants.KubeadmConfigPath) + } else { + if patchesDir != "" { + applyArgs = append(applyArgs, fmt.Sprintf("--patches=%s", constants.PatchesDir)) + } + applyArgs = append(applyArgs, "-f", fmt.Sprintf("v%s", upgradeVersion.String())) } + if err := cp1.Command( "kubeadm", applyArgs..., ).RunWithEcho(); err != nil { @@ -163,7 +227,7 @@ func kubeadmUpgradeApply(c *status.Cluster, cp1 *status.Node, upgradeVersion *K8 return nil } -func kubeadmUpgradeNode(c *status.Cluster, n *status.Node, upgradeVersion *K8sVersion.Version, patchesDir string, wait time.Duration, vLevel int) error { +func kubeadmUpgradeNode(c *status.Cluster, n *status.Node, configVersion string, upgradeVersion *version.Version, patchesDir string, wait time.Duration, vLevel int) error { // waitKubeletHasRBAC waits for the kubelet to have access to the expected config map // please note that this is a temporary workaround for a problem we are observing on upgrades while // executing node upgrades immediately after control-plane upgrade. @@ -175,9 +239,15 @@ func kubeadmUpgradeNode(c *status.Cluster, n *status.Node, upgradeVersion *K8sVe nodeArgs := []string{ "upgrade", "node", fmt.Sprintf("--v=%d", vLevel), } - if patchesDir != "" { - nodeArgs = append(nodeArgs, fmt.Sprintf("--patches=%s", constants.PatchesDir)) + + if configVersion == "v1beta4" { + nodeArgs = append(nodeArgs, "--config", constants.KubeadmConfigPath) + } else { + if patchesDir != "" { + nodeArgs = append(nodeArgs, fmt.Sprintf("--patches=%s", constants.PatchesDir)) + } } + if err := n.Command( "kubeadm", nodeArgs..., ).RunWithEcho(); err != nil { @@ -193,7 +263,7 @@ func kubeadmUpgradeNode(c *status.Cluster, n *status.Node, upgradeVersion *K8sVe return nil } -func upgradeKubeletKubectl(c *status.Cluster, n *status.Node, upgradeVersion *K8sVersion.Version, wait time.Duration) error { +func upgradeKubeletKubectl(c *status.Cluster, n *status.Node, upgradeVersion *version.Version, wait time.Duration) error { n.Infof("upgrade kubelet and kubectl binaries") srcFolder := filepath.Join("/kinder", "upgrade", fmt.Sprintf("v%s", upgradeVersion)) diff --git a/kinder/pkg/kubeadm/componentpatches.go b/kinder/pkg/kubeadm/componentpatches.go index 3180ea9c..2816a48f 100644 --- a/kinder/pkg/kubeadm/componentpatches.go +++ b/kinder/pkg/kubeadm/componentpatches.go @@ -28,21 +28,29 @@ import ( // to use patches directory. func GetPatchesDirectoryPatches(kubeadmConfigVersion string) ([]string, error) { log.Debugf("Preparing patches directory for kubeadm config %s", kubeadmConfigVersion) - var patchInit, patchJoin string + var patchInit, patchJoin, patchUpgradeApply, patchUpgradeNode string switch kubeadmConfigVersion { case "v1beta3": patchInit = patchesDirectoryPatchInitv1beta3 patchJoin = patchesDirectoryPatchJoinv1beta3 + return []string{ + fmt.Sprintf(patchInit, constants.PatchesDir), + fmt.Sprintf(patchJoin, constants.PatchesDir), + }, nil case "v1beta4": patchInit = patchesDirectoryPatchInitv1beta4 patchJoin = patchesDirectoryPatchJoinv1beta4 + patchUpgradeApply = patchesDirectoryPatchUpgradeApplyv1beta4 + patchUpgradeNode = patchesDirectoryPatchUpgradeNodev1beta4 + return []string{ + fmt.Sprintf(patchInit, constants.PatchesDir), + fmt.Sprintf(patchJoin, constants.PatchesDir), + fmt.Sprintf(patchUpgradeApply, constants.PatchesDir), + fmt.Sprintf(patchUpgradeNode, constants.PatchesDir), + }, nil default: return []string{}, errors.Errorf("unknown kubeadm config version: %s", kubeadmConfigVersion) } - return []string{ - fmt.Sprintf(patchInit, constants.PatchesDir), - fmt.Sprintf(patchJoin, constants.PatchesDir), - }, nil } const patchesDirectoryPatchInitv1beta3 = `apiVersion: kubeadm.k8s.io/v1beta3 @@ -64,3 +72,15 @@ const patchesDirectoryPatchJoinv1beta4 = `apiVersion: kubeadm.k8s.io/v1beta4 kind: JoinConfiguration patches: directory: %s` + +const patchesDirectoryPatchUpgradeApplyv1beta4 = `apiVersion: kubeadm.k8s.io/v1beta4 +kind: UpgradeConfiguration +apply: + patches: + directory: %s` + +const patchesDirectoryPatchUpgradeNodev1beta4 = `apiVersion: kubeadm.k8s.io/v1beta4 +kind: UpgradeConfiguration +node: + patches: + directory: %s` diff --git a/kinder/pkg/kubeadm/config.go b/kinder/pkg/kubeadm/config.go index 18f106d5..df105915 100644 --- a/kinder/pkg/kubeadm/config.go +++ b/kinder/pkg/kubeadm/config.go @@ -100,6 +100,8 @@ type ConfigData struct { FeatureGateValue string // The encryption algorithm EncryptionAlgorithm string + // UpgradeVersion is the version passed to kubeadm upgrade + UpgradeVersion string // DerivedConfigData is populated by Derive() // These auto-generated fields are available to Config templates, // but not meant to be set by hand @@ -195,6 +197,29 @@ discovery: token: "{{ .Token }}" unsafeSkipCAVerification: true --- +apiVersion: kubeadm.k8s.io/v1beta4 +kind: UpgradeConfiguration +plan: + kubernetesVersion: {{.UpgradeVersion}} + allowExperimentalUpgrades: true + allowRCUpgrades: true +node: + patches: + directory: "/kinder/patches" +diff: + kubernetesVersion: {{.UpgradeVersion}} +apply: + kubernetesVersion: {{.UpgradeVersion}} + allowExperimentalUpgrades: true + allowRCUpgrades: true + forceUpgrade: true + patches: + directory: "/kinder/patches" +--- +apiVersion: kubeadm.k8s.io/v1beta4 +kind: ResetConfiguration +force: true +--- apiVersion: kubelet.config.k8s.io/v1beta1 kind: KubeletConfiguration # configure ipv6 addresses in IPv6 mode