k8s: add setting for configuring cloudProvider

Adds a new setting `kubernetes.cloud-provider` for configuring whether the cloud provider is `aws` or `external`. Prior to this, the argument was hard-coded to `aws`.
bottlerocket-os · Apr 19, 2021 · 3a210f9 · 3a210f9
1 parent 04acb06
commit 3a210f9
Show file tree

Hide file tree

Showing 8 changed files with 56 additions and 7 deletions.
diff --git a/README.md b/README.md
@@ -308,6 +308,7 @@ The following settings can be optionally set to customize the node labels and ta
 The following settings are optional and allow you to further configure your cluster.
 * `settings.kubernetes.cluster-domain`: The DNS domain for this cluster, allowing all Kubernetes-run containers to search this domain before the host's search domains.  Defaults to `cluster.local`.
 * `settings.kubernetes.standalone-mode`: Whether to run the kubelet in standalone mode, without connecting to an API server.  Defaults to `false`.
+* `settings.kubernetes.cloud-provider`: The cloud provider for this cluster. Defaults to `aws` for AWS variants, and `external` for other variants.
 * `settings.kubernetes.authentication-mode`: Which authentication method the kubelet should use to connect to the API server, and for incoming requests.  Defaults to `aws` for AWS variants, and `tls` for other variants.
 * `settings.kubernetes.server-tls-bootstrap`: Enables or disables server certificate bootstrap.  When enabled, the kubelet will request a certificate from the certificates.k8s.io API.  This requires an approver to approve the certificate signing requests (CSR).  Defaults to `true`.
 * `settings.kubernetes.bootstrap-token`: The token to use for [TLS bootstrapping](https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet-tls-bootstrapping/).  This is only used with the `tls` authentication mode, and is otherwise ignored.

diff --git a/packages/kubernetes-1.16/kubelet-exec-start-conf b/packages/kubernetes-1.16/kubelet-exec-start-conf
@@ -2,7 +2,7 @@
 ExecStart=
 ExecStart=/usr/bin/kubelet \
 {{~#unless settings.kubernetes.standalone-mode}}
-    --cloud-provider aws \
+    --cloud-provider {{default "external" settings.kubernetes.cloud-provider}} \
     --kubeconfig /etc/kubernetes/kubelet/kubeconfig \
 {{~#if (eq settings.kubernetes.authentication-mode "tls")}}
     --bootstrap-kubeconfig /etc/kubernetes/kubelet/bootstrap-kubeconfig \

diff --git a/packages/kubernetes-1.17/kubelet-exec-start-conf b/packages/kubernetes-1.17/kubelet-exec-start-conf
@@ -2,7 +2,7 @@
 ExecStart=
 ExecStart=/usr/bin/kubelet \
 {{~#unless settings.kubernetes.standalone-mode}}
-    --cloud-provider aws \
+    --cloud-provider {{default "external" settings.kubernetes.cloud-provider}} \
     --kubeconfig /etc/kubernetes/kubelet/kubeconfig \
 {{~#if (eq settings.kubernetes.authentication-mode "tls")}}
     --bootstrap-kubeconfig /etc/kubernetes/kubelet/bootstrap-kubeconfig \

diff --git a/packages/kubernetes-1.18/kubelet-exec-start-conf b/packages/kubernetes-1.18/kubelet-exec-start-conf
@@ -2,7 +2,7 @@
 ExecStart=
 ExecStart=/usr/bin/kubelet \
 {{~#unless settings.kubernetes.standalone-mode}}
-    --cloud-provider aws \
+    --cloud-provider {{default "external" settings.kubernetes.cloud-provider}} \
     --kubeconfig /etc/kubernetes/kubelet/kubeconfig \
 {{~#if (eq settings.kubernetes.authentication-mode "tls")}}
     --bootstrap-kubeconfig /etc/kubernetes/kubelet/bootstrap-kubeconfig \

diff --git a/packages/kubernetes-1.19/kubelet-exec-start-conf b/packages/kubernetes-1.19/kubelet-exec-start-conf
@@ -2,7 +2,7 @@
 ExecStart=
 ExecStart=/usr/bin/kubelet \
 {{~#unless settings.kubernetes.standalone-mode}}
-    --cloud-provider aws \
+    --cloud-provider {{default "external" settings.kubernetes.cloud-provider}} \
     --kubeconfig /etc/kubernetes/kubelet/kubeconfig \
 {{~#if (eq settings.kubernetes.authentication-mode "tls")}}
     --bootstrap-kubeconfig /etc/kubernetes/kubelet/bootstrap-kubeconfig \

diff --git a/sources/models/src/aws-k8s-1.19/defaults.d/50-aws-k8s.toml b/sources/models/src/aws-k8s-1.19/defaults.d/50-aws-k8s.toml
@@ -59,6 +59,7 @@ cluster-domain = "cluster.local"
 standalone-mode = false
 authentication-mode = "aws"
 server-tls-bootstrap = true
+cloud-provider = "aws"
 
 # Metrics
 [settings.metrics]

diff --git a/sources/models/src/lib.rs b/sources/models/src/lib.rs
@@ -95,9 +95,10 @@ use std::net::Ipv4Addr;
 use crate::modeled_types::{
     BootstrapContainerMode, DNSDomain, ECSAgentLogLevel, ECSAttributeKey, ECSAttributeValue,
     FriendlyVersion, Identifier, KubernetesAuthenticationMode, KubernetesBootstrapToken,
-    KubernetesClusterName, KubernetesEvictionHardKey, KubernetesLabelKey, KubernetesLabelValue,
-    KubernetesQuantityValue, KubernetesReservedResourceKey, KubernetesTaintValue,
-    KubernetesThresholdValue, Lockdown, SingleLineString, SysctlKey, Url, ValidBase64,
+    KubernetesCloudProvider, KubernetesClusterName, KubernetesEvictionHardKey, KubernetesLabelKey,
+    KubernetesLabelValue, KubernetesQuantityValue, KubernetesReservedResourceKey,
+    KubernetesTaintValue, KubernetesThresholdValue, Lockdown, SingleLineString, SysctlKey, Url,
+    ValidBase64,
 };
 
 // Kubernetes static pod manifest settings
@@ -127,6 +128,7 @@ struct KubernetesSettings {
     kube_reserved: HashMap<KubernetesReservedResourceKey, KubernetesQuantityValue>,
     allowed_unsafe_sysctls: Vec<SingleLineString>,
     server_tls_bootstrap: bool,
+    cloud_provider: KubernetesCloudProvider,
 
     // Settings where we generate a value based on the runtime environment.  The user can specify a
     // value to override the generated one, but typically would not.

diff --git a/sources/models/src/modeled_types/kubernetes.rs b/sources/models/src/modeled_types/kubernetes.rs
@@ -723,3 +723,48 @@ mod test_kubernetes_quantity_value {
         }
     }
 }
+
+// =^..^=   =^..^=   =^..^=   =^..^=   =^..^=   =^..^=   =^..^=   =^..^=   =^..^=
+
+/// KubernetesCloudProvider represents a string that is a valid cloud provider for the
+/// kubelet.  It stores the original string and makes it accessible through standard traits.
+#[derive(Debug, Clone, Eq, PartialEq, Hash)]
+pub struct KubernetesCloudProvider {
+    inner: String,
+}
+
+impl TryFrom<&str> for KubernetesCloudProvider {
+    type Error = error::Error;
+
+    fn try_from(input: &str) -> Result<Self, error::Error> {
+        ensure!(
+            matches!(input, "aws" | "external"),
+            error::InvalidAuthenticationMode { input }
+        );
+        Ok(KubernetesCloudProvider {
+            inner: input.to_string(),
+        })
+    }
+}
+
+string_impls_for!(KubernetesCloudProvider, "KubernetesCloudProvider");
+
+#[cfg(test)]
+mod test_kubernetes_cloud_provider {
+    use super::KubernetesCloudProvider;
+    use std::convert::TryFrom;
+
+    #[test]
+    fn good_modes() {
+        for ok in &["aws", "external"] {
+            KubernetesCloudProvider::try_from(*ok).unwrap();
+        }
+    }
+
+    #[test]
+    fn bad_modes() {
+        for err in &["", "internal"] {
+            KubernetesCloudProvider::try_from(*err).unwrap_err();
+        }
+    }
+}