using zfs for vanilla cluster test

Signed-off-by: danfengl <danfengl@vmware.com>

test/e2e/Makefile

@@ -56,15 +56,15 @@ VELERO_IMAGE ?= velero/velero:main
 PLUGINS ?=
 RESTORE_HELPER_IMAGE ?=
 #Released version only
-UPGRADE_FROM_VELERO_VERSION ?= v1.10.2,v1.11.0
+UPGRADE_FROM_VELERO_VERSION ?= v1.12.3,v1.13.0
 # UPGRADE_FROM_VELERO_CLI can has the same format(a list divided by comma) with UPGRADE_FROM_VELERO_VERSION
 # Upgrade tests will be executed sequently according to the list by UPGRADE_FROM_VELERO_VERSION
 # So although length of UPGRADE_FROM_VELERO_CLI list is not equal with UPGRADE_FROM_VELERO_VERSION
 # Script will still read UPGRADE_FROM_VELERO_CLI list to match UPGRADE_FROM_VELERO_VERSION list from beginning
 # to the end, nil string will be set if UPGRADE_FROM_VELERO_CLI is shorter than UPGRADE_FROM_VELERO_VERSION
 UPGRADE_FROM_VELERO_CLI ?=
 
-MIGRATE_FROM_VELERO_VERSION ?= v1.11.0,self
+MIGRATE_FROM_VELERO_VERSION ?= v1.13.0,self
 MIGRATE_FROM_VELERO_CLI ?=
 
 VELERO_NAMESPACE ?= velero

test/e2e/backup/backup.go

@@ -74,18 +74,19 @@ func BackupRestoreTest(backupRestoreTestConfig BackupRestoreTestConfig) {
 		veleroCfg.KibishiiDirectory = veleroCfg.KibishiiDirectory + backupRestoreTestConfig.kibishiiPatchSubDir
 		veleroCfg.UseVolumeSnapshots = useVolumeSnapshots
 		veleroCfg.UseNodeAgent = !useVolumeSnapshots
-		if veleroCfg.CloudProvider == "kind" {
+		if veleroCfg.CloudProvider == KIND {
 			Skip("Volume snapshots plugin and File System Backups are not supported on kind")
 			// on kind cluster snapshots are not supported since there is no velero snapshot plugin for kind volumes.
 			// and PodVolumeBackups are not supported because PVB creation gets skipped for hostpath volumes, which are the only
 			// volumes created on kind clusters using the default storage class and provisioner (provisioner: rancher.io/local-path)
 			// This test suite checks for volume snapshots and PVBs generated from FileSystemBackups, so skip it on kind clusters
 		}
+
 		// [SKIP]: Static provisioning for vSphere CSI driver works differently from other drivers.
 		//         For vSphere CSI, after you create a PV specifying an existing volume handle, CSI
 		//         syncer will need to register it with CNS. For other CSI drivers, static provisioning
 		//         usually does not go through storage system at all.  That's probably why it took longer
-		if backupRestoreTestConfig.isRetainPVTest && veleroCfg.CloudProvider == "vsphere" {
+		if backupRestoreTestConfig.isRetainPVTest && veleroCfg.CloudProvider == VSPHERE {
 			Skip("Skip due to vSphere CSI driver long time issue of Static provisioning")
 		}
 		var err error

test/e2e/backups/deletion.go

@@ -54,8 +54,8 @@ func backup_deletion_test(useVolumeSnapshots bool) {
 	veleroCfg.UseNodeAgent = !useVolumeSnapshots
 
 	BeforeEach(func() {
-		if useVolumeSnapshots && veleroCfg.CloudProvider == "kind" {
-			Skip("Volume snapshots not supported on kind")
+		if useVolumeSnapshots && veleroCfg.CloudProvider == KIND {
+			Skip(fmt.Sprintf("Volume snapshots not supported on %s", KIND))
 		}
 		var err error
 		flag.Parse()
@@ -134,7 +134,7 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupNam
 		})
 	})
 
-	if providerName == "vsphere" && useVolumeSnapshots {
+	if providerName == VSPHERE && useVolumeSnapshots {
 		// Wait for uploads started by the Velero Plugin for vSphere to complete
 		// TODO - remove after upload progress monitoring is implemented
 		fmt.Println("Waiting for vSphere uploads to complete")

test/e2e/backups/ttl.go

@@ -120,7 +120,7 @@ func TTLTest() {
 
 		var snapshotCheckPoint SnapshotCheckPoint
 		if useVolumeSnapshots {
-			if veleroCfg.CloudProvider == "vsphere" {
+			if veleroCfg.CloudProvider == VSPHERE {
 				// TODO - remove after upload progress monitoring is implemented
 				By("Waiting for vSphere uploads to complete", func() {
 					Expect(WaitForVSphereUploadCompletion(ctx, time.Hour,
@@ -140,7 +140,7 @@ func TTLTest() {
 				fmt.Sprintf("Failed to delete namespace %s", BackupCfg.BackupName))
 		})
 
-		if veleroCfg.CloudProvider == "aws" && useVolumeSnapshots {
+		if veleroCfg.CloudProvider == AWS && useVolumeSnapshots {
 			fmt.Println("Waiting 7 minutes to make sure the snapshots are ready...")
 			time.Sleep(7 * time.Minute)
 		}

test/e2e/basic/backup-volume-info/base.go

@@ -26,9 +26,11 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 	"github.com/pkg/errors"
+	v1 "k8s.io/api/core/v1"
 
 	. "github.com/vmware-tanzu/velero/test"
 	. "github.com/vmware-tanzu/velero/test/e2e/test"
+	. "github.com/vmware-tanzu/velero/test/util/common"
 	. "github.com/vmware-tanzu/velero/test/util/k8s"
 	. "github.com/vmware-tanzu/velero/test/util/velero"
 )
@@ -45,12 +47,12 @@ func (v *BackupVolumeInfo) Init() error {
 	v.TestCase.Init()
 
 	BeforeEach(func() {
-		if v.VeleroCfg.CloudProvider == "vsphere" && (!strings.Contains(v.CaseBaseName, "fs-upload") && !strings.Contains(v.CaseBaseName, "skipped")) {
+		if v.VeleroCfg.CloudProvider == VSPHERE && (!strings.Contains(v.CaseBaseName, "fs-upload") && !strings.Contains(v.CaseBaseName, "skipped")) {
 			fmt.Printf("Skip snapshot case %s for vsphere environment.\n", v.CaseBaseName)
 			Skip("Skip snapshot case due to vsphere environment doesn't cover the CSI test, and it doesn't have a Velero native snapshot plugin.")
 		}
 
-		if strings.Contains(v.VeleroCfg.Features, "EnableCSI") {
+		if strings.Contains(v.VeleroCfg.Features, FEATURE_CSI) {
 			if strings.Contains(v.CaseBaseName, "native-snapshot") {
 				fmt.Printf("Skip native snapshot case %s when the CSI feature is enabled.\n", v.CaseBaseName)
 				Skip("Skip native snapshot case due to CSI feature is enabled.")
@@ -111,21 +113,42 @@ func (v *BackupVolumeInfo) CreateResources() error {
 		// Install StorageClass
 		Expect(InstallTestStorageClasses(fmt.Sprintf("../testdata/storage-class/%s-csi.yaml", v.VeleroCfg.CloudProvider))).To(Succeed(), "Failed to install StorageClass")
 
-		pvc, err := CreatePVC(v.Client, createNSName, "volume-info", CSIStorageClassName, nil)
-		Expect(err).To(Succeed())
-		vols := CreateVolumes(pvc.Name, []string{"volume-info"})
-
-		//Create deployment
+		// Create deployment
 		fmt.Printf("Creating deployment in namespaces ...%s\n", createNSName)
+		// Make sure PVC count is great than 3 to allow both empty volumes and file populated volumes exist per pod
+		pvcCount := 4
+		Expect(pvcCount > 3).To(Equal(true))
+
+		var vols []*v1.Volume
+		for i := 0; i <= pvcCount-1; i++ {
+			pvcName := fmt.Sprintf("volume-info-pvc-%d", i)
+			pvc, err := CreatePVC(v.Client, createNSName, pvcName, CSIStorageClassName, nil)
+			Expect(err).To(Succeed())
+			volumeName := fmt.Sprintf("volume-info-pv-%d", i)
+			vols = append(vols, CreateVolumes(pvc.Name, []string{volumeName})...)
+		}
 		deployment := NewDeployment(v.CaseBaseName, createNSName, 1, labels, nil).WithVolume(vols).Result()
-		deployment, err = CreateDeployment(v.Client.ClientGo, createNSName, deployment)
+		deployment, err := CreateDeployment(v.Client.ClientGo, createNSName, deployment)
 		if err != nil {
 			return errors.Wrap(err, fmt.Sprintf("failed to delete the namespace %q", createNSName))
 		}
 		err = WaitForReadyDeployment(v.Client.ClientGo, createNSName, deployment.Name)
 		if err != nil {
 			return errors.Wrap(err, fmt.Sprintf("failed to ensure job completion in namespace: %q", createNSName))
 		}
+		podList, err := ListPods(v.Ctx, v.Client, createNSName)
+		Expect(err).To(Succeed(), fmt.Sprintf("failed to list pods in namespace: %q with error %v", createNSName, err))
+
+		for _, pod := range podList.Items {
+			for i := 0; i <= pvcCount-1; i++ {
+				// Hitting issue https://github.com/vmware-tanzu/velero/issues/7388
+				// So populate data only to some of pods, leave other pods empty to verify empty PV datamover
+				if i%2 == 0 {
+					Expect(CreateFileToPod(v.Ctx, createNSName, pod.Name, DefaultContainerName, vols[i].Name,
+						fmt.Sprintf("file-%s", pod.Name), CreateFileContent(createNSName, pod.Name, vols[i].Name))).To(Succeed())
+				}
+			}
+		}
 	}
 	return nil
 }

test/e2e/bsl-mgmt/deletion.go

@@ -61,8 +61,8 @@ func BslDeletionTest(useVolumeSnapshots bool) {
 	less := func(a, b string) bool { return a < b }
 
 	BeforeEach(func() {
-		if useVolumeSnapshots && veleroCfg.CloudProvider == "kind" {
-			Skip("Volume snapshots not supported on kind")
+		if useVolumeSnapshots && veleroCfg.CloudProvider == KIND {
+			Skip(fmt.Sprintf("Volume snapshots not supported on %s", KIND))
 		}
 		var err error
 		flag.Parse()
@@ -203,7 +203,7 @@ func BslDeletionTest(useVolumeSnapshots bool) {
 			})
 
 			if useVolumeSnapshots {
-				if veleroCfg.CloudProvider == "vsphere" {
+				if veleroCfg.CloudProvider == VSPHERE {
 					// TODO - remove after upload progress monitoring is implemented
 					By("Waiting for vSphere uploads to complete", func() {
 						Expect(WaitForVSphereUploadCompletion(oneHourTimeout, time.Hour,
@@ -218,32 +218,33 @@ func BslDeletionTest(useVolumeSnapshots bool) {
 							backupName_2, 1)).To(Succeed())
 					})
 				}
-
-				var snapshotCheckPoint SnapshotCheckPoint
-				snapshotCheckPoint.NamespaceBackedUp = bslDeletionTestNs
-				By(fmt.Sprintf("Snapshot of bsl %s should be created in cloud object store", backupLocation_1), func() {
-					snapshotCheckPoint, err = GetSnapshotCheckPoint(*veleroCfg.ClientToInstallVelero, veleroCfg, 1, bslDeletionTestNs, backupName_1, []string{podName_1})
-					Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint")
-					Expect(SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider,
-						veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket,
-						veleroCfg.BSLConfig, backupName_1, snapshotCheckPoint)).To(Succeed())
-				})
-				By(fmt.Sprintf("Snapshot of bsl %s should be created in cloud object store", backupLocation_2), func() {
-					snapshotCheckPoint, err = GetSnapshotCheckPoint(*veleroCfg.ClientToInstallVelero, veleroCfg, 1, bslDeletionTestNs, backupName_2, []string{podName_2})
-					Expect(err).NotTo(HaveOccurred(), "Fail to get snapshot checkpoint")
-					var BSLCredentials, BSLConfig string
-					if veleroCfg.CloudProvider == "vsphere" {
-						BSLCredentials = veleroCfg.AdditionalBSLCredentials
-						BSLConfig = veleroCfg.AdditionalBSLConfig
-					} else { // Snapshotting with non-vSphere provider has nothing to do with BSL
-						BSLCredentials = veleroCfg.CloudCredentialsFile
-						BSLConfig = veleroCfg.BSLConfig
-					}
-
-					Expect(SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider,
-						BSLCredentials, veleroCfg.AdditionalBSLBucket,
-						BSLConfig, backupName_2, snapshotCheckPoint)).To(Succeed())
-				})
+				if veleroCfg.CloudProvider != VANILLA_ZFS {
+					var snapshotCheckPoint SnapshotCheckPoint
+					snapshotCheckPoint.NamespaceBackedUp = bslDeletionTestNs
+					By(fmt.Sprintf("Snapshot of bsl %s should be created in cloud object store", backupLocation_1), func() {
+						snapshotCheckPoint, err = GetSnapshotCheckPoint(*veleroCfg.ClientToInstallVelero, veleroCfg, 1, bslDeletionTestNs, backupName_1, []string{podName_1})
+						Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint")
+						Expect(SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider,
+							veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket,
+							veleroCfg.BSLConfig, backupName_1, snapshotCheckPoint)).To(Succeed())
+					})
+					By(fmt.Sprintf("Snapshot of bsl %s should be created in cloud object store", backupLocation_2), func() {
+						snapshotCheckPoint, err = GetSnapshotCheckPoint(*veleroCfg.ClientToInstallVelero, veleroCfg, 1, bslDeletionTestNs, backupName_2, []string{podName_2})
+						Expect(err).NotTo(HaveOccurred(), "Fail to get snapshot checkpoint")
+						var BSLCredentials, BSLConfig string
+						if veleroCfg.CloudProvider == VSPHERE {
+							BSLCredentials = veleroCfg.AdditionalBSLCredentials
+							BSLConfig = veleroCfg.AdditionalBSLConfig
+						} else { // Snapshotting with non-vSphere provider has nothing to do with BSL
+							BSLCredentials = veleroCfg.CloudCredentialsFile
+							BSLConfig = veleroCfg.BSLConfig
+						}
+
+						Expect(SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider,
+							BSLCredentials, veleroCfg.AdditionalBSLBucket,
+							BSLConfig, backupName_2, snapshotCheckPoint)).To(Succeed())
+					})
+				}
 			} else { // For Restics
 				By(fmt.Sprintf("Resticrepositories for BSL %s should be created in Velero namespace", backupLocation_1), func() {
 					Expect(BackupRepositoriesCountShouldBe(context.Background(),
@@ -309,7 +310,7 @@ func BslDeletionTest(useVolumeSnapshots bool) {
 			// })
 
 			if useVolumeSnapshots {
-				if veleroCfg.CloudProvider == "vsphere" {
+				if veleroCfg.CloudProvider == VSPHERE {
 					By(fmt.Sprintf("Snapshot in backup %s should still exist, because snapshot CR will be deleted 24 hours later if the status is a success", backupName_2), func() {
 						Expect(SnapshotCRsCountShouldBe(context.Background(), bslDeletionTestNs,
 							backupName_1, 1)).To(Succeed())
@@ -329,7 +330,7 @@ func BslDeletionTest(useVolumeSnapshots bool) {
 				})
 				By(fmt.Sprintf("Snapshot should not be deleted in cloud object store after deleting bsl %s", backupLocation_2), func() {
 					var BSLCredentials, BSLConfig string
-					if veleroCfg.CloudProvider == "vsphere" {
+					if veleroCfg.CloudProvider == VSPHERE {
 						BSLCredentials = veleroCfg.AdditionalBSLCredentials
 						BSLConfig = veleroCfg.AdditionalBSLConfig
 					} else {

test/e2e/e2e_suite_test.go

@@ -104,13 +104,20 @@ func init() {
 
 }
 
+// Add label [SkipVanillaZfs]:
+//   We found issue - https://github.com/openebs/zfs-localpv/issues/123 when using OpenEBS ZFS CSI Driver
+//   When PVC using storage class with reclaim policy as 'Delete', once PVC is deleted, snapshot associated will be deleted
+//   along with PVC deletion, after restoring workload, restored PVC is in pending status, due to failure of provision PV
+//   caused by no expected snapshot found. If we use retain as reclaim policy, then this label can be ignored, all test
+//   cases can be executed as expected successful result.
+
 var _ = Describe("[APIGroup][APIVersion][SKIP_KIND] Velero tests with various CRD API group versions", APIGropuVersionsTest)
 var _ = Describe("[APIGroup][APIExtensions][SKIP_KIND] CRD of apiextentions v1beta1 should be B/R successfully from cluster(k8s version < 1.22) to cluster(k8s version >= 1.22)", APIExtensionsVersionsTest)
 
 // Test backup and restore of Kibishi using restic
 var _ = Describe("[Basic][Restic] Velero tests on cluster using the plugin provider for object storage and Restic for volume backups", BackupRestoreWithRestic)
 
-var _ = Describe("[Basic][Snapshot] Velero tests on cluster using the plugin provider for object storage and snapshots for volume backups", BackupRestoreWithSnapshots)
+var _ = Describe("[Basic][Snapshot][SkipVanillaZfs] Velero tests on cluster using the plugin provider for object storage and snapshots for volume backups", BackupRestoreWithSnapshots)
 
 var _ = Describe("[Basic][Snapshot][RetainPV] Velero tests on cluster using the plugin provider for object storage and snapshots for volume backups", BackupRestoreRetainedPVWithSnapshots)
 
@@ -122,7 +129,7 @@ var _ = Describe("[Scale][LongTime] Backup/restore of 2500 namespaces", MultiNSB
 
 // Upgrade test by Kibishi using restic
 var _ = Describe("[Upgrade][Restic] Velero upgrade tests on cluster using the plugin provider for object storage and Restic for volume backups", BackupUpgradeRestoreWithRestic)
-var _ = Describe("[Upgrade][Snapshot] Velero upgrade tests on cluster using the plugin provider for object storage and snapshots for volume backups", BackupUpgradeRestoreWithSnapshots)
+var _ = Describe("[Upgrade][Snapshot][SkipVanillaZfs] Velero upgrade tests on cluster using the plugin provider for object storage and snapshots for volume backups", BackupUpgradeRestoreWithSnapshots)
 
 // test filter objects by namespace, type, or labels when backup or restore.
 var _ = Describe("[ResourceFiltering][ExcludeFromBackup] Resources with the label velero.io/exclude-from-backup=true are not included in backup", ExcludeFromBackupTest)
@@ -147,8 +154,8 @@ var _ = Describe("[BackupVolumeInfo][NativeSnapshot]", NativeSnapshotVolumeInfoT
 var _ = Describe("[ResourceModifier][Restore] Velero test on resource modifiers from the cluster restore", ResourceModifiersTest)
 
 var _ = Describe("[Backups][Deletion][Restic] Velero tests of Restic backup deletion", BackupDeletionWithRestic)
-var _ = Describe("[Backups][Deletion][Snapshot] Velero tests of snapshot backup deletion", BackupDeletionWithSnapshots)
-var _ = Describe("[Backups][TTL][LongTime][Snapshot] Local backups and restic repos will be deleted once the corresponding backup storage location is deleted", TTLTest)
+var _ = Describe("[Backups][Deletion][Snapshot][SkipVanillaZfs] Velero tests of snapshot backup deletion", BackupDeletionWithSnapshots)
+var _ = Describe("[Backups][TTL][LongTime][Snapshot][SkipVanillaZfs] Local backups and restic repos will be deleted once the corresponding backup storage location is deleted", TTLTest)
 var _ = Describe("[Backups][BackupsSync] Backups in object storage are synced to a new Velero and deleted backups in object storage are synced to be deleted in Velero", BackupsSyncTest)
 
 var _ = Describe("[Schedule][BR][Pause][LongTime] Backup will be created periodly by schedule defined by a Cron expression", ScheduleBackupTest)
@@ -157,16 +164,16 @@ var _ = Describe("[Schedule][BackupCreation][SKIP_KIND] Schedule controller woul
 
 var _ = Describe("[PrivilegesMgmt][SSR] Velero test on ssr object when controller namespace mix-ups", SSRTest)
 
-var _ = Describe("[BSL][Deletion][Snapshot] Local backups will be deleted once the corresponding backup storage location is deleted", BslDeletionWithSnapshots)
+var _ = Describe("[BSL][Deletion][Snapshot][SkipVanillaZfs] Local backups will be deleted once the corresponding backup storage location is deleted", BslDeletionWithSnapshots)
 var _ = Describe("[BSL][Deletion][Restic] Local backups and restic repos will be deleted once the corresponding backup storage location is deleted", BslDeletionWithRestic)
 
 var _ = Describe("[Migration][Restic] Migrate resources between clusters by Restic", MigrationWithRestic)
-var _ = Describe("[Migration][Snapshot] Migrate resources between clusters by snapshot", MigrationWithSnapshots)
+var _ = Describe("[Migration][Snapshot][SkipVanillaZfs] Migrate resources between clusters by snapshot", MigrationWithSnapshots)
 
 var _ = Describe("[NamespaceMapping][Single][Restic] Backup resources should follow the specific order in schedule", OneNamespaceMappingResticTest)
 var _ = Describe("[NamespaceMapping][Multiple][Restic] Backup resources should follow the specific order in schedule", MultiNamespacesMappingResticTest)
-var _ = Describe("[NamespaceMapping][Single][Snapshot] Backup resources should follow the specific order in schedule", OneNamespaceMappingSnapshotTest)
-var _ = Describe("[NamespaceMapping][Multiple][Snapshot] Backup resources should follow the specific order in schedule", MultiNamespacesMappingSnapshotTest)
+var _ = Describe("[NamespaceMapping][Single][Snapshot][SkipVanillaZfs] Backup resources should follow the specific order in schedule", OneNamespaceMappingSnapshotTest)
+var _ = Describe("[NamespaceMapping][Multiple][Snapshot]SkipVanillaZfs] Backup resources should follow the specific order in schedule", MultiNamespacesMappingSnapshotTest)
 
 var _ = Describe("[pv-backup][Opt-In] Backup resources should follow the specific order in schedule", OptInPVBackupTest)
 var _ = Describe("[pv-backup][Opt-Out] Backup resources should follow the specific order in schedule", OptOutPVBackupTest)