Add external cloud provider support for VPC (#1015)

cloud/scope/machine.go

@@ -29,15 +29,18 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/klog/v2/klogr"
+	"k8s.io/utils/pointer"
 
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	capiv1beta1 "sigs.k8s.io/cluster-api/api/v1beta1"
 	"sigs.k8s.io/cluster-api/util/patch"
 
 	infrav1beta2 "sigs.k8s.io/cluster-api-provider-ibmcloud/api/v1beta2"
+	"sigs.k8s.io/cluster-api-provider-ibmcloud/pkg/cloud/services/utils"
 	"sigs.k8s.io/cluster-api-provider-ibmcloud/pkg/cloud/services/vpc"
 	"sigs.k8s.io/cluster-api-provider-ibmcloud/pkg/endpoints"
+	"sigs.k8s.io/cluster-api-provider-ibmcloud/pkg/options"
 	"sigs.k8s.io/cluster-api-provider-ibmcloud/pkg/record"
 )
 
@@ -426,3 +429,19 @@ func fetchImageID(image *infrav1beta2.IBMVPCResourceReference, m *MachineScope)
 
 	return nil, fmt.Errorf("image does not exist - failed to find an image ID")
 }
+
+// SetProviderID will set the provider id for the machine.
+func (m *MachineScope) SetProviderID(id *string) error {
+	// Based on the ProviderIDFormat version the providerID format will be decided.
+	if options.ProviderIDFormatType(options.ProviderIDFormat) == options.ProviderIDFormatV2 {
+		accountID, err := utils.GetAccountID()
+		if err != nil {
+			m.Logger.Error(err, "failed to get cloud account id", err.Error())
+			return err
+		}
+		m.IBMVPCMachine.Spec.ProviderID = pointer.String(fmt.Sprintf("ibm://%s///%s/%s", accountID, m.Machine.Spec.ClusterName, *id))
+	} else {
+		m.IBMVPCMachine.Spec.ProviderID = pointer.String(fmt.Sprintf("ibmvpc://%s/%s", m.Machine.Spec.ClusterName, m.IBMVPCMachine.Name))
+	}
+	return nil
+}

cloud/scope/powervs_machine.go

@@ -573,11 +573,11 @@ func (m *PowerVSMachineScope) GetZone() string {
 // SetProviderID will set the provider id for the machine.
 func (m *PowerVSMachineScope) SetProviderID(id *string) {
 	// Based on the ProviderIDFormat version the providerID format will be decided.
-	if options.PowerVSProviderIDFormatType(options.PowerVSProviderIDFormat) == options.PowerVSProviderIDFormatV2 {
+	if options.ProviderIDFormatType(options.PowerVSProviderIDFormat) == options.PowerVSProviderIDFormatV2 {
 		if id != nil {
-			m.IBMPowerVSMachine.Spec.ProviderID = pointer.StringPtr(fmt.Sprintf("ibmpowervs://%s/%s/%s/%s", m.GetRegion(), m.GetZone(), m.IBMPowerVSMachine.Spec.ServiceInstanceID, *id))
+			m.IBMPowerVSMachine.Spec.ProviderID = pointer.String(fmt.Sprintf("ibmpowervs://%s/%s/%s/%s", m.GetRegion(), m.GetZone(), m.IBMPowerVSMachine.Spec.ServiceInstanceID, *id))
 		}
 	} else {
-		m.IBMPowerVSMachine.Spec.ProviderID = pointer.StringPtr(fmt.Sprintf("ibmpowervs://%s/%s", m.Machine.Spec.ClusterName, m.IBMPowerVSMachine.Name))
+		m.IBMPowerVSMachine.Spec.ProviderID = pointer.String(fmt.Sprintf("ibmpowervs://%s/%s", m.Machine.Spec.ClusterName, m.IBMPowerVSMachine.Name))
 	}
 }

config/manager/manager.yaml

@@ -29,6 +29,7 @@ spec:
         - "--leader-elect"
         - "--metrics-bind-addr=127.0.0.1:8080"
         - "--powervs-provider-id-fmt=${POWERVS_PROVIDER_ID_FORMAT:=v1}"
+        - "--provider-id-fmt=${PROVIDER_ID_FORMAT:=v1}"
         - "--service-endpoint=${SERVICE_ENDPOINT:=none}"
         - "--v=${LOGLEVEL:=0}"
         image: controller:latest

controllers/ibmvpcmachine_controller.go

@@ -30,7 +30,6 @@ import (
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/tools/record"
-	"k8s.io/utils/pointer"
 
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -168,7 +167,9 @@ func (r *IBMVPCMachineReconciler) reconcileNormal(machineScope *scope.MachineSco
 			},
 		}
 		_, ok := machineScope.IBMVPCMachine.Labels[capiv1beta1.MachineControlPlaneLabelName]
-		machineScope.IBMVPCMachine.Spec.ProviderID = pointer.StringPtr(fmt.Sprintf("ibmvpc://%s/%s", machineScope.Machine.Spec.ClusterName, machineScope.IBMVPCMachine.Name))
+		if err = machineScope.SetProviderID(instance.ID); err != nil {
+			return ctrl.Result{}, errors.Wrapf(err, "failed to set provider id IBMVPCMachine %s/%s", machineScope.IBMVPCMachine.Namespace, machineScope.IBMVPCMachine.Name)
+		}
 		if ok {
 			if machineScope.IBMVPCCluster.Spec.ControlPlaneLoadBalancer == nil {
 				options := &vpcv1.AddInstanceNetworkInterfaceFloatingIPOptions{}

docs/book/src/SUMMARY.md

@@ -10,7 +10,7 @@
     - [Prerequisites](./topics/vpc/prerequisites.md)
     - [Uploading an image](topics/vpc/uploading-an-image.md)
     - [Creating a cluster](./topics/vpc/creating-a-cluster.md)
-    - [Creating a cluster with Load Balancer](./topics/vpc/load-balancer.md)
+    - [Creating a cluster with Load Balancer and External Cloud Provider](./topics/vpc/load-balancer.md)
   - [Power VS Cluster](./topics/powervs/index.md)
     - [Prerequisites](./topics/powervs/prerequisites.md)
     - [Creating a cluster](./topics/powervs/creating-a-cluster.md)

docs/book/src/developer/tilt.md

@@ -10,7 +10,7 @@ This document describes how to use [kind](https://kind.sigs.k8s.io) and [Tilt](h
 2. [kind](https://kind.sigs.k8s.io) v0.9 or newer (other clusters can be
    used if `preload_images_for_kind` is set to false)
 3. [kustomize](https://kubectl.docs.kubernetes.io/installation/kustomize/)
-4. [Tilt](https://docs.tilt.dev/install.html) v0.22.2 or newer
+4. [Tilt](https://docs.tilt.dev/install.html) v0.30.8 or newer
 5. [envsubst](https://github.com/drone/envsubst) or similar to handle
    clusterctl var replacement
 6. Clone the [Cluster API](https://github.com/kubernetes-sigs/cluster-api) repository
@@ -55,9 +55,9 @@ extra_args:
     - '-v=5'
 ```
 
-### 1.  Configuration to deploy workload cluster with external cloud controller manager
+### 1.  Configuration to deploy Power VS workload cluster with external cloud controller manager
 
-To deploy workload cluster with [Power VS cloud controller manager](/topics/powervs/external-cloud-provider.html) which is currently in experimental stage, Set `POWERVS_PROVIDER_ID_FORMAT` to `v2` and enable cluster resourceset feature gateunder kustomize_substitutions.
+To deploy workload cluster with [Power VS cloud controller manager](/topics/powervs/external-cloud-provider.html) which is currently in experimental stage, Set `POWERVS_PROVIDER_ID_FORMAT` to `v2` and enable cluster resourceset feature gate under kustomize_substitutions.
 Currently, [ClusterResourceset](https://cluster-api.sigs.k8s.io/tasks/experimental-features/cluster-resource-set.html) is experimental feature so we need to enable the feature gate by setting `EXP_CLUSTER_RESOURCE_SET` variable under kustomize_substitutions.
 
 ```yaml
@@ -74,7 +74,26 @@ kustomize_substitutions:
   EXP_CLUSTER_RESOURCE_SET: "true"
 ```
 
-### 2.  Configuration to deploy workload cluster from ClusterClass template
+### 2.  Configuration to deploy VPC workload cluster with external cloud controller manager
+
+To deploy workload cluster with [cloud controller manager](/topics/vpc/load-balancer.html) which is currently in experimental stage, Set `PROVIDER_ID_FORMAT` to `v2` and enable cluster resourceset feature gate under kustomize_substitutions.
+Currently, [ClusterResourceset](https://cluster-api.sigs.k8s.io/tasks/experimental-features/cluster-resource-set.html) is experimental feature so we need to enable the feature gate by setting `EXP_CLUSTER_RESOURCE_SET` variable under kustomize_substitutions.
+
+```yaml
+default_registry: "gcr.io/you-project-name-here"
+provider_repos:
+- ../cluster-api-provider-ibmcloud
+enable_providers:
+- ibmcloud
+- kubeadm-bootstrap
+- kubeadm-control-plane
+kustomize_substitutions:
+  IBMCLOUD_API_KEY: "XXXXXXXXXXXXXXXXXX"
+  PROVIDER_ID_FORMAT: "v2"
+  EXP_CLUSTER_RESOURCE_SET: "true"
+```
+
+### 3.  Configuration to deploy workload cluster from ClusterClass template
 
 To deploy workload cluster with [clusterclass-template](/topics/powervs/clusterclass-cluster.html) under kustomize_substitutions set `POWERVS_PROVIDER_ID_FORMAT` to `v2`.
 Currently, both [ClusterClass](https://cluster-api.sigs.k8s.io/tasks/experimental-features/cluster-class/index.html) and [ClusterResourceset](https://cluster-api.sigs.k8s.io/tasks/experimental-features/cluster-resource-set.html) are experimental feature so we need to enable the feature gate by setting `EXP_CLUSTER_RESOURCE_SET`, `CLUSTER_TOPOLOGY` variable under kustomize_substitutions.
@@ -94,7 +113,7 @@ kustomize_substitutions:
   CLUSTER_TOPOLOGY: "true"
 ```
 
-### 3.  Configuration to deploy workload cluster with Custom Service Endpoint
+### 4.  Configuration to deploy workload cluster with Custom Service Endpoint
 
 To deploy workload cluster with Custom Service Endpoint, Set `SERVICE_ENDPOINT` environmental variable in semi-colon separated format: `${ServiceRegion}:${ServiceID1}=${URL1},${ServiceID2}=${URL2...}`
 ```yaml

docs/book/src/getting-started.md

@@ -48,28 +48,35 @@ it into a management cluster using `clusterctl`.
 
     > Note: Refer [Regions-Zones Mapping](/reference/regions-zones-mapping.html) for more information.
 
-   > Note: To deploy workload cluster with [Power VS cloud controller manager](/topics/powervs/external-cloud-provider.html) which is currently in experimental stage. Set the `POWERVS_PROVIDER_ID_FORMAT` environmental variable.
-   Currently, [ClusterResourceset](https://cluster-api.sigs.k8s.io/tasks/experimental-features/cluster-resource-set.html) is experimental feature so we need to enable the feature gate by setting `EXP_CLUSTER_RESOURCE_SET` environmental variables.
-   ```console
-      export POWERVS_PROVIDER_ID_FORMAT=v2
-      export EXP_CLUSTER_RESOURCE_SET=true
-    ```
-   > Note: To deploy workload cluster with [Power VS clusterclass-template](/topics/powervs/clusterclass-cluster.html). Set the `POWERVS_PROVIDER_ID_FORMAT` environmental variable.
-     Currently, both [ClusterClass](https://cluster-api.sigs.k8s.io/tasks/experimental-features/cluster-class/index.html) and [ClusterResourceset](https://cluster-api.sigs.k8s.io/tasks/experimental-features/cluster-resource-set.html) are experimental feature so we need to enable the feature gate by setting `EXP_CLUSTER_RESOURCE_SET`, `CLUSTER_TOPOLOGY` environmental variables.
+    > Note: To deploy VPC workload cluster with [IBM cloud controller manager](/topics/vpc/load-balancer.html) which is currently in experimental stage. Set the `PROVIDER_ID_FORMAT` environmental variable.
+    Currently, [ClusterResourceset](https://cluster-api.sigs.k8s.io/tasks/experimental-features/cluster-resource-set.html) is experimental feature, so we need to enable the feature gate by setting `EXP_CLUSTER_RESOURCE_SET` environmental variables.
     ```console
-      export POWERVS_PROVIDER_ID_FORMAT=v2
-      export EXP_CLUSTER_RESOURCE_SET=true
-      export CLUSTER_TOPOLOGY=true
-    ```
+       export PROVIDER_ID_FORMAT=v2
+       export EXP_CLUSTER_RESOURCE_SET=true
+     ```
 
-   > Note: To deploy workload cluster with Custom Service Endpoint, Set `SERVICE_ENDPOINT` environmental variable in semi-colon separated format:
+    > Note: To deploy workload cluster with [Power VS cloud controller manager](/topics/powervs/external-cloud-provider.html) which is currently in experimental stage. Set the `POWERVS_PROVIDER_ID_FORMAT` environmental variable.
+    Currently, [ClusterResourceset](https://cluster-api.sigs.k8s.io/tasks/experimental-features/cluster-resource-set.html) is experimental feature so we need to enable the feature gate by setting `EXP_CLUSTER_RESOURCE_SET` environmental variables.
     ```console
-   `${ServiceRegion1}:${ServiceID1}=${URL1},${ServiceID2}=${URL2};${ServiceRegion2}:${ServiceID1}=${URL1...}`.
-    ```
-   Supported ServiceIDs include - `vpc, powervs, rc`
-    ```console
-      export SERVICE_ENDPOINT=us-south:vpc=https://us-south-stage01.iaasdev.cloud.ibm.com,powervs=https://dal.power-iaas.test.cloud.ibm.com,rc=https://resource-controller.test.cloud.ibm.com
-    ```
+       export POWERVS_PROVIDER_ID_FORMAT=v2
+       export EXP_CLUSTER_RESOURCE_SET=true
+     ```
+    > Note: To deploy workload cluster with [Power VS clusterclass-template](/topics/powervs/clusterclass-cluster.html). Set the `POWERVS_PROVIDER_ID_FORMAT` environmental variable.
+      Currently, both [ClusterClass](https://cluster-api.sigs.k8s.io/tasks/experimental-features/cluster-class/index.html) and [ClusterResourceset](https://cluster-api.sigs.k8s.io/tasks/experimental-features/cluster-resource-set.html) are experimental feature so we need to enable the feature gate by setting `EXP_CLUSTER_RESOURCE_SET`, `CLUSTER_TOPOLOGY` environmental variables.
+     ```console
+        export POWERVS_PROVIDER_ID_FORMAT=v2
+        export EXP_CLUSTER_RESOURCE_SET=true
+        export CLUSTER_TOPOLOGY=true
+     ```
+
+    > Note: To deploy workload cluster with Custom Service Endpoint, Set `SERVICE_ENDPOINT` environmental variable in semi-colon separated format:
+     ```console
+        `${ServiceRegion1}:${ServiceID1}=${URL1},${ServiceID2}=${URL2};${ServiceRegion2}:${ServiceID1}=${URL1...}`.
+     ```
+    Supported ServiceIDs include - `vpc, powervs, rc`
+     ```console
+        export SERVICE_ENDPOINT=us-south:vpc=https://us-south-stage01.iaasdev.cloud.ibm.com,powervs=https://dal.power-iaas.test.cloud.ibm.com,rc=https://resource-controller.test.cloud.ibm.com
+     ```
 
 2. Initialize local bootstrap cluster as a management cluster
     

docs/book/src/topics/powervs/external-cloud-provider.md

@@ -1,5 +1,5 @@
 # IBM Power VS External Cloud Provider
-## This feature currently in experimental stage
+> ⚠️ **WARNING**: This feature is currently in experimental stage
 
 ## Steps
 

docs/book/src/topics/vpc/load-balancer.md

@@ -1,22 +1,32 @@
-## Provision workload Cluster with Load Balancer in IBM Cloud VPC
+## Provision workload Cluster with Load Balancer and external cloud provider in IBM Cloud VPC
 
 > ⚠️ **WARNING**: This feature is currently in experimental stage
 
-Using clusterctl, render the yaml through templates and deploy the cluster
+## Steps
+
+- To deploy a VPC workload cluster with Load Balancer IBM external [cloud provider](https://kubernetes.io/docs/concepts/architecture/cloud-controller/), create a cluster configuration with the [external cloud provider template](https://github.com/kubernetes-sigs/cluster-api-provider-ibmcloud/blob/main/templates/cluster-template-load-balancer.yaml)
+- The [external cloud provider template](https://github.com/kubernetes-sigs/cluster-api-provider-ibmcloud/blob/main/templates/cluster-template-load-balancer.yaml) will use [clusterresourceset](https://cluster-api.sigs.k8s.io/tasks/experimental-features/cluster-resource-set.html) and will create the necessary config map, secret and roles to run the cloud controller manager
+- As a prerequisite set the `provider-id-fmt` [flag](https://github.com/kubernetes-sigs/cluster-api-provider-ibmcloud/blob/ee70591709ac5ddaeed23222ccbfa78335d984a1/main.go#L183) with value v2
+
+### Deploy VPC cluster with Load Balancer and IBM cloud provider
 
 ```console
-IBMVPC_REGION=us-south \
-IBMVPC_ZONE=us-south-1 \
-IBMVPC_RESOURCEGROUP=4f15679623607b855b1a27a67f20e1c7 \
-IBMVPC_NAME=ibm-vpc-0 \
-IBMVPC_IMAGE_NAME=capibm-vpc-ubuntu-2004-kube-v1-25-2 \
-IBMVPC_PROFILE=bx2-4x16 \
-IBMVPC_SSHKEY_NAME=capi-vpc-key \
-clusterctl generate cluster ibm-vpc-0 --kubernetes-version v1.22.0 \
---target-namespace default \
---control-plane-machine-count=3 \
---worker-machine-count=1 \
---flavor=load-balancer | kubectl apply -f -
+  IBMCLOUD_API_KEY: "XXXXXXXXXXXXXXXXXX" \
+  IBMVPC_REGION=us-south \
+  IBMVPC_ZONE=us-south-1 \
+  IBMVPC_RESOURCEGROUP_NAME="ibm-hypershift-dev" \
+  IBMVPC_RESOURCEGROUP=4f15679623607b855b1a27a67f20e1c7 \
+  IBMVPC_NAME=ibm-vpc-0 \
+  IBMVPC_IMAGE_ID=r134-ea84bbec-7986-4ff5-8489-d9ec34611dd4 \
+  IBMVPC_PROFILE=bx2-4x16 \
+  IBMVPC_SSHKEY_ID=r134-2a82b725-e570-43d3-8b23-9539e8641944 \
+  IBMACCOUNT_ID="ibm-accountid" \
+  BASE64_API_KEY=$(echo -n $IBMCLOUD_API_KEY | base64) \
+  clusterctl generate cluster ibm-vpc-0 --kubernetes-version v1.25.2 \
+  --target-namespace default \
+  --control-plane-machine-count=1 \
+  --worker-machine-count=2 \
+  --flavor=load-balancer | kubectl apply -f -
 ```
 
 **Change disk size for the boot volume**

main.go

@@ -165,13 +165,21 @@ func initFlags(fs *pflag.FlagSet) {
 		"The minimum interval at which watched resources are reconciled.",
 	)
 
+	// TODO: Deprecate it to use provider-id-fmt for both vpc and power vs
 	fs.StringVar(
 		&options.PowerVSProviderIDFormat,
 		"powervs-provider-id-fmt",
 		string(options.PowerVSProviderIDFormatV1),
 		"ProviderID format is used set the Provider ID format for Machine",
 	)
 
+	fs.StringVar(
+		&options.ProviderIDFormat,
+		"provider-id-fmt",
+		string(options.ProviderIDFormatV1),
+		"ProviderID format is used set the Provider ID format for Machine (Currently for VPC machines only)",
+	)
+
 	fs.StringVar(
 		&endpoints.ServiceEndpointFormat,
 		"service-endpoint",
@@ -181,14 +189,21 @@ func initFlags(fs *pflag.FlagSet) {
 }
 
 func validateFlags() error {
-	switch options.PowerVSProviderIDFormatType(options.PowerVSProviderIDFormat) {
+	switch options.ProviderIDFormatType(options.PowerVSProviderIDFormat) {
 	case options.PowerVSProviderIDFormatV1:
-		setupLog.Info("Using v1 version of ProviderID format")
+		setupLog.Info("Using v1 version of Power VS ProviderID format")
 	case options.PowerVSProviderIDFormatV2:
-		setupLog.Info("Using v2 version of ProviderID format")
+		setupLog.Info("Using v2 version of Power VS ProviderID format")
+	default:
+		return fmt.Errorf("invalid value for flag powervs-provider-id-fmt: %s, Supported values: v1, v2 ", options.PowerVSProviderIDFormat)
+	}
+	switch options.ProviderIDFormatType(options.ProviderIDFormat) {
+	case options.ProviderIDFormatV1:
+		setupLog.Info("Using v1 version of VPC ProviderID format")
+	case options.ProviderIDFormatV2:
+		setupLog.Info("Using v2 version of VPC ProviderID format")
 	default:
-		errStr := fmt.Errorf("invalid value for flag powervs-provider-id-fmt: %s, Supported values: v1, v2 ", options.PowerVSProviderIDFormat)
-		return errStr
+		return fmt.Errorf("invalid value for flag provider-id-fmt: %s, Supported values: %s, %s ", options.ProviderIDFormat, options.ProviderIDFormatV1, options.ProviderIDFormatV2)
 	}
 	return nil
 }

pkg/cloud/services/utils/accounts.go

@@ -24,6 +24,8 @@ import (
 	"github.com/golang-jwt/jwt"
 
 	"github.com/IBM/go-sdk-core/v5/core"
+
+	"sigs.k8s.io/cluster-api-provider-ibmcloud/pkg/cloud/services/authenticator"
 )
 
 // GetAccount is function parses the account number from the token and returns it.
@@ -51,3 +53,12 @@ func GetAccount(auth core.Authenticator) (string, error) {
 
 	return token.Claims.(jwt.MapClaims)["account"].(map[string]interface{})["bss"].(string), nil
 }
+
+// GetAccountID will parse and returns user cloud account ID.
+func GetAccountID() (string, error) {
+	auth, err := authenticator.GetAuthenticator()
+	if err != nil {
+		return "", err
+	}
+	return GetAccount(auth)
+}

pkg/options/options.go

@@ -16,16 +16,28 @@ limitations under the License.
 
 package options
 
-// PowerVSProviderIDFormatType enum attribute to identify Power VS ProviderID format.
-type PowerVSProviderIDFormatType string
+// ProviderIDFormatType enum attribute to identify Power VS or VPC ProviderID format.
+type ProviderIDFormatType string
 
 const (
 	// PowerVSProviderIDFormatV1 will set provider id to machine as ibmpowervs://<cluster_name>/<vm_hostname>
-	PowerVSProviderIDFormatV1 PowerVSProviderIDFormatType = "v1"
+	PowerVSProviderIDFormatV1 ProviderIDFormatType = "v1"
 
 	// PowerVSProviderIDFormatV2 will set provider id to machine as ibmpowervs://<region>/<zone>/<service_instance_id>/<powervs_machine_id>
-	PowerVSProviderIDFormatV2 PowerVSProviderIDFormatType = "v2"
+	PowerVSProviderIDFormatV2 ProviderIDFormatType = "v2"
+
+	// ProviderIDFormatV1 will set provider id to machine as follows
+	// For VPC machines: ibmvpc://<cluster_name>/<vm_hostname>
+	ProviderIDFormatV1 ProviderIDFormatType = "v1"
+
+	// ProviderIDFormatV2 will set provider id to machine as follows
+	// For VPC machines: ibm://<account_id>///<cluster_id>/<vpc_machine_id>
+	ProviderIDFormatV2 ProviderIDFormatType = "v2"
 )
 
-// PowerVSProviderIDFormat is used to identify the Provider ID format for Machine.
-var PowerVSProviderIDFormat string
+var (
+	// PowerVSProviderIDFormat is used to identify the Provider ID format for Power VS Machine.
+	PowerVSProviderIDFormat string
+	// ProviderIDFormat is used to identify the Provider ID format for Machine.
+	ProviderIDFormat string
+)