kinder: use "--config" for kubeadm "reset" and "upgrade"

When executing "kubeadm reset" or "kubeadm upgrade",
if the kubeadm version is 1.31 (or pre-release) pass
the "--config" flag with the v1beta4 kinds. Else,
continue to call the flags for a given command.

Additionally:
- Call "kubeadm upgrade diff" on the primary CP
node after calling "upgrade plan".
- Fallback to the /kinder/upgrade/version during
upgrade if the provided version is different.

kinder/pkg/cluster/manager/actions/actions.go

@@ -40,7 +40,7 @@ var actionRegistry = map[string]func(*status.Cluster, *RunOptions) error{
 	"kubeadm-config": func(c *status.Cluster, flags *RunOptions) error {
 		// Nb. this action is invoked automatically at kubeadm init/join time, but it is possible
 		// to invoke it separately as well
-		return KubeadmConfig(c, flags.kubeadmConfigVersion, flags.copyCertsMode, flags.discoveryMode, flags.featureGate, flags.encryptionAlgorithm, c.K8sNodes().EligibleForActions()...)
+		return KubeadmConfig(c, flags.kubeadmConfigVersion, flags.copyCertsMode, flags.discoveryMode, flags.featureGate, flags.encryptionAlgorithm, flags.upgradeVersion, c.K8sNodes().EligibleForActions()...)
 	},
 	"kubeadm-init": func(c *status.Cluster, flags *RunOptions) error {
 		return KubeadmInit(c, flags.usePhases, flags.copyCertsMode, flags.kubeadmConfigVersion, flags.patchesDir, flags.ignorePreflightErrors, flags.featureGate, flags.encryptionAlgorithm, flags.wait, flags.vLevel)
@@ -49,7 +49,7 @@ var actionRegistry = map[string]func(*status.Cluster, *RunOptions) error{
 		return KubeadmJoin(c, flags.usePhases, flags.copyCertsMode, flags.discoveryMode, flags.kubeadmConfigVersion, flags.patchesDir, flags.ignorePreflightErrors, flags.wait, flags.vLevel)
 	},
 	"kubeadm-upgrade": func(c *status.Cluster, flags *RunOptions) error {
-		return KubeadmUpgrade(c, flags.upgradeVersion, flags.patchesDir, flags.featureGate, flags.wait, flags.vLevel)
+		return KubeadmUpgrade(c, flags.upgradeVersion, flags.patchesDir, flags.wait, flags.vLevel)
 	},
 	"kubeadm-reset": func(c *status.Cluster, flags *RunOptions) error {
 		return KubeadmReset(c, flags.vLevel)

kinder/pkg/cluster/manager/actions/kubeadm-config.go

@@ -23,7 +23,9 @@ import (
 	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
 
+	"k8s.io/apimachinery/pkg/util/version"
 	"k8s.io/client-go/tools/clientcmd"
+
 	"k8s.io/kubeadm/kinder/pkg/cluster/status"
 	"k8s.io/kubeadm/kinder/pkg/constants"
 	"k8s.io/kubeadm/kinder/pkg/cri/nodes"
@@ -42,21 +44,31 @@ type kubeadmConfigOptions struct {
 // to invoke it separately as well.
 func KubeadmInitConfig(c *status.Cluster, kubeadmConfigVersion string, copyCertsMode CopyCertsMode, featureGate, encryptionAlgorithm string, nodes ...*status.Node) error {
 	// defaults everything not relevant for the Init Config
-	return KubeadmConfig(c, kubeadmConfigVersion, copyCertsMode, TokenDiscovery, featureGate, encryptionAlgorithm, nodes...)
+	return KubeadmConfig(c, kubeadmConfigVersion, copyCertsMode, TokenDiscovery, featureGate, encryptionAlgorithm, nil, nodes...)
 }
 
 // KubeadmJoinConfig action writes the JoinConfiguration into /kind/kubeadm.conf file on all the K8s nodes in the cluster.
 // Please note that this action is automatically executed at create time, but it is possible
 // to invoke it separately as well.
 func KubeadmJoinConfig(c *status.Cluster, kubeadmConfigVersion string, copyCertsMode CopyCertsMode, discoveryMode DiscoveryMode, nodes ...*status.Node) error {
 	// defaults everything not relevant for the join Config
-	return KubeadmConfig(c, kubeadmConfigVersion, copyCertsMode, discoveryMode, "" /* feature-gates */, "" /* encryptionAlgorithm */, nodes...)
+	return KubeadmConfig(c, kubeadmConfigVersion, copyCertsMode, discoveryMode, "", "", nil, nodes...)
+}
+
+// KubeadmUpgradeConfig action writes the UpgradeConfiguration into /kind/kubeadm.conf file on all the K8s nodes in the cluster.
+func KubeadmUpgradeConfig(c *status.Cluster, upgradeVersion *version.Version, nodes ...*status.Node) error {
+	return KubeadmConfig(c, "", "", "", "", "", upgradeVersion, nodes...)
+}
+
+// KubeadmResetConfig action writes the UpgradeConfiguration into /kind/kubeadm.conf file on all the K8s nodes in the cluster.
+func KubeadmResetConfig(c *status.Cluster, nodes ...*status.Node) error {
+	return KubeadmConfig(c, "", "", "", "", "", nil, nodes...)
 }
 
 // KubeadmConfig action writes the /kind/kubeadm.conf file on all the K8s nodes in the cluster.
 // Please note that this action is automatically executed at create time, but it is possible
 // to invoke it separately as well.
-func KubeadmConfig(c *status.Cluster, kubeadmConfigVersion string, copyCertsMode CopyCertsMode, discoveryMode DiscoveryMode, featureGate, encryptionAlgorithm string, nodes ...*status.Node) error {
+func KubeadmConfig(c *status.Cluster, kubeadmConfigVersion string, copyCertsMode CopyCertsMode, discoveryMode DiscoveryMode, featureGate, encryptionAlgorithm string, upgradeVersion *version.Version, nodes ...*status.Node) error {
 	cp1 := c.BootstrapControlPlane()
 
 	// get installed kubernetes version from the node image
@@ -95,6 +107,19 @@ func KubeadmConfig(c *status.Cluster, kubeadmConfigVersion string, copyCertsMode
 		featureGateValue = split[1]
 	}
 
+	if copyCertsMode == "" {
+		copyCertsMode = CopyCertsModeAuto
+	}
+
+	if discoveryMode == "" {
+		discoveryMode = TokenDiscovery
+	}
+
+	// Use a placeholder upgrade version for non-upgrade actions.
+	if upgradeVersion == nil {
+		upgradeVersion = version.MustParseSemantic("v1.0.0")
+	}
+
 	// create configData with all the configurations supported by the kubeadm config template implemented in kind
 	configData := kubeadm.ConfigData{
 		ClusterName:          c.Name(),
@@ -109,6 +134,7 @@ func KubeadmConfig(c *status.Cluster, kubeadmConfigVersion string, copyCertsMode
 		FeatureGateName:      featureGateName,
 		FeatureGateValue:     featureGateValue,
 		EncryptionAlgorithm:  encryptionAlgorithm,
+		UpgradeVersion:       fmt.Sprintf("v%s", upgradeVersion.String()),
 	}
 
 	// create configOptions with all the kinder flags that impact on the kubeadm config generation
@@ -179,7 +205,10 @@ func writeKubeadmConfig(c *status.Cluster, n *status.Node, data kubeadm.ConfigDa
 		return errors.Wrap(err, "failed to generate kubeadm config content")
 	}
 
-	log.Debugf("generated config:\n%s", kubeadmConfig)
+	log.Debug("generating config...")
+	if log.GetLevel() == log.DebugLevel {
+		fmt.Print(kubeadmConfig)
+	}
 
 	// copy the config to the node
 	if err := n.WriteFile(constants.KubeadmConfigPath, []byte(kubeadmConfig)); err != nil {
@@ -329,13 +358,17 @@ func getKubeadmConfig(c *status.Cluster, n *status.Node, data kubeadm.ConfigData
 		return selectYamlFramentByKind(patched,
 			"ClusterConfiguration",
 			"InitConfiguration",
+			"UpgradeConfiguration",
+			"ResetConfiguration",
 			"KubeletConfiguration",
 			"KubeProxyConfiguration"), nil
 	}
 
 	// otherwise select only the JoinConfiguration
 	return selectYamlFramentByKind(patched,
 		"JoinConfiguration",
+		"UpgradeCOnfiguration",
+		"ResetConfiguration",
 	), nil
 }
 

kinder/pkg/cluster/manager/actions/kubeadm-reset.go

@@ -19,16 +19,36 @@ package actions
 import (
 	"fmt"
 
+	"github.com/pkg/errors"
+
 	"k8s.io/kubeadm/kinder/pkg/cluster/status"
+	"k8s.io/kubeadm/kinder/pkg/constants"
+	"k8s.io/kubeadm/kinder/pkg/kubeadm"
 )
 
 // KubeadmReset executes the kubeadm reset workflow
 func KubeadmReset(c *status.Cluster, vLevel int) error {
 	//TODO: implements kubeadm reset with phases
 	for _, n := range c.K8sNodes().EligibleForActions() {
-		if err := n.Command(
-			"kubeadm", "reset", "--force", fmt.Sprintf("--v=%d", vLevel),
-		).RunWithEcho(); err != nil {
+		flags := []string{"reset", fmt.Sprintf("--v=%d", vLevel)}
+
+		// After upgrade, the 'kubeadm version' should return the version of the kubeadm used
+		// to perform the upgrade. Use this version to determine if v1beta4 is enabled. If yes,
+		// use ResetConfiguration with a 'force: true', else just use the '--force' flag.
+		v, err := n.KubeadmVersion()
+		if err != nil {
+			return errors.Wrap(err, "could not obtain the kubeadm version before calling 'kubeadm reset'")
+		}
+		if kubeadm.GetKubeadmConfigVersion(v) == "v1beta4" {
+			if err := KubeadmResetConfig(c, n); err != nil {
+				return errors.Wrap(err, "could not write kubeadm config before calling 'kubeadm reset'")
+			}
+			flags = append(flags, "--config", constants.KubeadmConfigPath)
+		} else {
+			flags = append(flags, "--force")
+		}
+
+		if err := n.Command("kubeadm", flags...).RunWithEcho(); err != nil {
 			return err
 		}
 	}