using zfs for vanilla cluster test

Signed-off-by: danfengl <danfengl@vmware.com>
vmware-tanzu · Feb 20, 2024 · d197bb6 · d197bb6
1 parent b2f1588
commit d197bb6
Show file tree

Hide file tree

Showing 12 changed files with 82 additions and 44 deletions.
diff --git a/test/e2e/Makefile b/test/e2e/Makefile
@@ -82,7 +82,7 @@ STANDBY_CLUSTER_OBJECT_STORE_PROVIDER ?=
 OBJECT_STORE_PROVIDER ?=
 INSTALL_VELERO ?= true
 REGISTRY_CREDENTIAL_FILE ?=
-KIBISHII_DIRECTORY ?= github.com/vmware-tanzu-experiments/distributed-data-generator/kubernetes/yaml/
+KIBISHII_DIRECTORY ?= github.com/danfengliu/distributed-data-generator/kubernetes/yaml/
 
 
 # Flags to create an additional BSL for multiple credentials tests

diff --git a/test/e2e/backup/backup.go b/test/e2e/backup/backup.go
@@ -74,7 +74,7 @@ func BackupRestoreTest(backupRestoreTestConfig BackupRestoreTestConfig) {
 		veleroCfg.KibishiiDirectory = veleroCfg.KibishiiDirectory + backupRestoreTestConfig.kibishiiPatchSubDir
 		veleroCfg.UseVolumeSnapshots = useVolumeSnapshots
 		veleroCfg.UseNodeAgent = !useVolumeSnapshots
-		if veleroCfg.CloudProvider == "kind" {
+		if veleroCfg.CloudProvider == "KIND" {
 			Skip("Volume snapshots plugin and File System Backups are not supported on kind")
 			// on kind cluster snapshots are not supported since there is no velero snapshot plugin for kind volumes.
 			// and PodVolumeBackups are not supported because PVB creation gets skipped for hostpath volumes, which are the only

diff --git a/test/e2e/backups/deletion.go b/test/e2e/backups/deletion.go
@@ -54,8 +54,8 @@ func backup_deletion_test(useVolumeSnapshots bool) {
 	veleroCfg.UseNodeAgent = !useVolumeSnapshots
 
 	BeforeEach(func() {
-		if useVolumeSnapshots && veleroCfg.CloudProvider == "kind" {
-			Skip("Volume snapshots not supported on kind")
+		if useVolumeSnapshots && veleroCfg.CloudProvider == KIND {
+			Skip(fmt.Sprintf("Volume snapshots not supported on %s", KIND))
 		}
 		var err error
 		flag.Parse()

diff --git a/test/e2e/bsl-mgmt/deletion.go b/test/e2e/bsl-mgmt/deletion.go
@@ -61,8 +61,8 @@ func BslDeletionTest(useVolumeSnapshots bool) {
 	less := func(a, b string) bool { return a < b }
 
 	BeforeEach(func() {
-		if useVolumeSnapshots && veleroCfg.CloudProvider == "kind" {
-			Skip("Volume snapshots not supported on kind")
+		if useVolumeSnapshots && veleroCfg.CloudProvider == KIND {
+			Skip(fmt.Sprintf("Volume snapshots not supported on %s", KIND))
 		}
 		var err error
 		flag.Parse()
@@ -218,32 +218,33 @@ func BslDeletionTest(useVolumeSnapshots bool) {
 							backupName_2, 1)).To(Succeed())
 					})
 				}
-
-				var snapshotCheckPoint SnapshotCheckPoint
-				snapshotCheckPoint.NamespaceBackedUp = bslDeletionTestNs
-				By(fmt.Sprintf("Snapshot of bsl %s should be created in cloud object store", backupLocation_1), func() {
-					snapshotCheckPoint, err = GetSnapshotCheckPoint(*veleroCfg.ClientToInstallVelero, veleroCfg, 1, bslDeletionTestNs, backupName_1, []string{podName_1})
-					Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint")
-					Expect(SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider,
-						veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket,
-						veleroCfg.BSLConfig, backupName_1, snapshotCheckPoint)).To(Succeed())
-				})
-				By(fmt.Sprintf("Snapshot of bsl %s should be created in cloud object store", backupLocation_2), func() {
-					snapshotCheckPoint, err = GetSnapshotCheckPoint(*veleroCfg.ClientToInstallVelero, veleroCfg, 1, bslDeletionTestNs, backupName_2, []string{podName_2})
-					Expect(err).NotTo(HaveOccurred(), "Fail to get snapshot checkpoint")
-					var BSLCredentials, BSLConfig string
-					if veleroCfg.CloudProvider == "vsphere" {
-						BSLCredentials = veleroCfg.AdditionalBSLCredentials
-						BSLConfig = veleroCfg.AdditionalBSLConfig
-					} else { // Snapshotting with non-vSphere provider has nothing to do with BSL
-						BSLCredentials = veleroCfg.CloudCredentialsFile
-						BSLConfig = veleroCfg.BSLConfig
-					}
-
-					Expect(SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider,
-						BSLCredentials, veleroCfg.AdditionalBSLBucket,
-						BSLConfig, backupName_2, snapshotCheckPoint)).To(Succeed())
-				})
+				if veleroCfg.CloudProvider != VanillaZFS {
+					var snapshotCheckPoint SnapshotCheckPoint
+					snapshotCheckPoint.NamespaceBackedUp = bslDeletionTestNs
+					By(fmt.Sprintf("Snapshot of bsl %s should be created in cloud object store", backupLocation_1), func() {
+						snapshotCheckPoint, err = GetSnapshotCheckPoint(*veleroCfg.ClientToInstallVelero, veleroCfg, 1, bslDeletionTestNs, backupName_1, []string{podName_1})
+						Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint")
+						Expect(SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider,
+							veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket,
+							veleroCfg.BSLConfig, backupName_1, snapshotCheckPoint)).To(Succeed())
+					})
+					By(fmt.Sprintf("Snapshot of bsl %s should be created in cloud object store", backupLocation_2), func() {
+						snapshotCheckPoint, err = GetSnapshotCheckPoint(*veleroCfg.ClientToInstallVelero, veleroCfg, 1, bslDeletionTestNs, backupName_2, []string{podName_2})
+						Expect(err).NotTo(HaveOccurred(), "Fail to get snapshot checkpoint")
+						var BSLCredentials, BSLConfig string
+						if veleroCfg.CloudProvider == "vsphere" {
+							BSLCredentials = veleroCfg.AdditionalBSLCredentials
+							BSLConfig = veleroCfg.AdditionalBSLConfig
+						} else { // Snapshotting with non-vSphere provider has nothing to do with BSL
+							BSLCredentials = veleroCfg.CloudCredentialsFile
+							BSLConfig = veleroCfg.BSLConfig
+						}
+
+						Expect(SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider,
+							BSLCredentials, veleroCfg.AdditionalBSLBucket,
+							BSLConfig, backupName_2, snapshotCheckPoint)).To(Succeed())
+					})
+				}
 			} else { // For Restics
 				By(fmt.Sprintf("Resticrepositories for BSL %s should be created in Velero namespace", backupLocation_1), func() {
 					Expect(BackupRepositoriesCountShouldBe(context.Background(),

diff --git a/test/e2e/migration/migration.go b/test/e2e/migration/migration.go
@@ -60,8 +60,8 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version)
 		veleroCfg = VeleroCfg
 		UUIDgen, err = uuid.NewRandom()
 		migrationNamespace = "migration-" + UUIDgen.String()
-		if useVolumeSnapshots && veleroCfg.CloudProvider == "kind" {
-			Skip("Volume snapshots not supported on kind")
+		if useVolumeSnapshots && veleroCfg.CloudProvider == KIND {
+			Skip(fmt.Sprintf("Volume snapshots not supported on %s", KIND))
 		}
 
 		if veleroCfg.DefaultClusterContext == "" && veleroCfg.StandbyClusterContext == "" {

diff --git a/test/e2e/upgrade/upgrade.go b/test/e2e/upgrade/upgrade.go
@@ -68,8 +68,8 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC
 		if (len(veleroCfg.UpgradeFromVeleroVersion)) == 0 {
 			Skip("An original velero version is required to run upgrade test, please run test with upgrade-from-velero-version=<version>")
 		}
-		if useVolumeSnapshots && veleroCfg.CloudProvider == "kind" {
-			Skip("Volume snapshots not supported on kind")
+		if useVolumeSnapshots && veleroCfg.CloudProvider == KIND {
+			Skip(fmt.Sprintf("Volume snapshots not supported on %s", KIND))
 		}
 		if veleroCfg.VeleroCLI == "" {
 			Skip("VeleroCLI should be provide")

diff --git a/test/testdata/storage-class/vanilla-zfs.yaml b/test/testdata/storage-class/vanilla-zfs.yaml
@@ -0,0 +1,11 @@
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: e2e-storage-class
+parameters:
+  recordsize: "128k"
+  compression: "off"
+  dedup: "off"
+  fstype: "zfs"
+  poolname: "zfspv-pool"
+provisioner: zfs.csi.openebs.io
diff --git a/test/types.go b/test/types.go
@@ -28,6 +28,12 @@ import (
 const StorageClassName = "e2e-storage-class"
 const StorageClassName2 = "e2e-storage-class-2"
 const FeatureCSI = "EnableCSI"
+const VanillaZFS = "vanilla-zfs"
+const KIND = "kind"
+
+var PublicCloudProviders = []string{"aws", "azure", "gcp", "vsphere", VanillaZFS}
+var LocalCloudProviders = []string{"aws", "azure", "gcp", "kind", VanillaZFS}
+var CloudProviders = append(PublicCloudProviders, LocalCloudProviders...)
 
 var InstallVelero bool
 var UUIDgen uuid.UUID

diff --git a/test/util/kibishii/kibishii_utils.go b/test/util/kibishii/kibishii_utils.go
@@ -149,7 +149,7 @@ func RunKibishiiTests(veleroCfg VeleroConfig, backupName, restoreName, backupLoc
 			// 	return errors.Wrapf(err, "Error get vSphere snapshot uploads")
 			// }
 		} else {
-			// wait for a period to confirm no snapshots exist for the backup
+			// wait for a period to confirm no snapshots content exist for the backup
 			time.Sleep(5 * time.Minute)
 			if strings.EqualFold(veleroFeatures, FeatureCSI) {
 				_, err = GetSnapshotCheckPoint(*veleroCfg.ClientToInstallVelero, veleroCfg, 0,

diff --git a/test/util/providers/common.go b/test/util/providers/common.go
@@ -105,6 +105,7 @@ func IsObjectsInBucket(objectStoreProvider, cloudCredentialsFile, bslBucket, bsl
 func DeleteObjectsInBucket(objectStoreProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix string) error {
 	bslPrefix = getFullPrefix(bslPrefix, subPrefix)
 	fmt.Printf("|| VERIFICATION || - Delete backup %s in storage %s\n", backupName, bslPrefix)
+
 	if cloudCredentialsFile == "" {
 		return errors.New(fmt.Sprintf("|| ERROR || - Please provide credential file of cloud %s \n", objectStoreProvider))
 	}
@@ -121,9 +122,15 @@ func DeleteObjectsInBucket(objectStoreProvider, cloudCredentialsFile, bslBucket,
 
 func SnapshotsShouldNotExistInCloud(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName string, snapshotCheckPoint SnapshotCheckPoint) error {
 	fmt.Printf("|| VERIFICATION || - Snapshots should not exist in cloud, backup %s\n", backupName)
+
+	if cloudProvider == "vanilla-zfs" {
+		fmt.Printf("Skip snapshot check for cloud provider %s", cloudProvider)
+		return nil
+	}
 	if cloudCredentialsFile == "" {
 		return errors.New(fmt.Sprintf("|| ERROR || - Please provide credential file of cloud %s \n", cloudProvider))
 	}
+
 	snapshotCheckPoint.ExpectCount = 0
 	err := IsSnapshotExisted(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName, snapshotCheckPoint)
 	if err != nil {
@@ -135,19 +142,24 @@ func SnapshotsShouldNotExistInCloud(cloudProvider, cloudCredentialsFile, bslBuck
 
 func SnapshotsShouldBeCreatedInCloud(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName string, snapshotCheckPoint SnapshotCheckPoint) error {
 	fmt.Printf("|| VERIFICATION || - Snapshots should exist in cloud, backup %s\n", backupName)
+
+	if cloudProvider == "vanilla-zfs" {
+		fmt.Printf("Skip snapshot check for cloud provider %s", cloudProvider)
+		return nil
+	}
 	if cloudCredentialsFile == "" {
 		return errors.New(fmt.Sprintf("|| ERROR || - Please provide credential file of cloud %s \n", cloudProvider))
 	}
+
 	err := IsSnapshotExisted(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName, snapshotCheckPoint)
 	if err != nil {
-		return errors.Wrapf(err, fmt.Sprintf("|| UNEXPECTED ||Snapshots %s do not exist in cloud after backup as expected", backupName))
+		return errors.Wrapf(err, fmt.Sprintf("|| UNEXPECTED || - Snapshots %s do not exist in cloud after backup as expected", backupName))
 	}
-	fmt.Printf("|| EXPECTED || - Snapshots exist in cloud, backup %s\n", backupName)
+	fmt.Printf("|| EXPECTED || - Snapshots of backup %s exist in cloud %s\n", backupName, cloudProvider)
 	return nil
 }
 
 func IsSnapshotExisted(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName string, snapshotCheck SnapshotCheckPoint) error {
-
 	s, err := getProvider(cloudProvider)
 	if err != nil {
 		return errors.Wrapf(err, fmt.Sprintf("Cloud provider %s is not valid", cloudProvider))

diff --git a/test/util/velero/install.go b/test/util/velero/install.go
@@ -27,6 +27,7 @@ import (
 	"time"
 
 	"github.com/pkg/errors"
+	"golang.org/x/exp/slices"
 	apps "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
@@ -76,8 +77,8 @@ func VeleroInstall(ctx context.Context, veleroCfg *VeleroConfig, isStandbyCluste
 		veleroCfg.CloudProvider = veleroCfg.StandbyClusterCloudProvider
 	}
 
-	if veleroCfg.CloudProvider != "kind" {
-		fmt.Println("For cloud platforms, object store plugin provider will be set as cloud provider")
+	if slices.Contains(PublicCloudProviders, veleroCfg.CloudProvider) {
+		fmt.Println("For public cloud platforms, object store plugin provider will be set as cloud provider")
 		// If ObjectStoreProvider is not provided, then using the value same as CloudProvider
 		if veleroCfg.ObjectStoreProvider == "" {
 			veleroCfg.ObjectStoreProvider = veleroCfg.CloudProvider

diff --git a/test/util/velero/velero_utils.go b/test/util/velero/velero_utils.go
@@ -128,12 +128,19 @@ func getPluginsByVersion(version, cloudProvider, objectStoreProvider, feature st
 	}
 	var pluginsForFeature []string
 
-	if cloudProvider == "kind" {
+	if slices.Contains(LocalCloudProviders, cloudProvider) {
+		var pluginsCSI []string
 		plugins, ok := cloudMap["aws"]
 		if !ok {
 			return nil, errors.Errorf("fail to get plugins by version: %s and provider %s", version, cloudProvider)
 		}
-		return plugins, nil
+		if cloudProvider == VanillaZFS {
+			pluginsCSI, ok = cloudMap["csi"]
+			if !ok {
+				return nil, errors.Errorf("fail to get plugins by version: %s and provider %s", version, cloudProvider)
+			}
+		}
+		return append(plugins, pluginsCSI...), nil
 	}
 
 	plugins, ok := cloudMap[cloudProvider]