From 7c50c3cb8c676c0bfea5b6f761ce226f853f5d3f Mon Sep 17 00:00:00 2001 From: danfengl Date: Thu, 1 Feb 2024 09:19:08 +0000 Subject: [PATCH] using zfs for vanilla cluster test Signed-off-by: danfengl --- test/e2e/Makefile | 4 +- test/e2e/backup/backup.go | 5 +- test/e2e/backups/deletion.go | 6 +- test/e2e/backups/ttl.go | 4 +- test/e2e/basic/backup-volume-info/base.go | 39 +++++++++--- test/e2e/bsl-mgmt/deletion.go | 63 ++++++++++--------- test/e2e/e2e_suite_test.go | 23 ++++--- test/e2e/migration/migration.go | 10 +-- test/e2e/pv-backup/pv-backup-filter.go | 8 +-- test/e2e/test/test.go | 4 +- test/e2e/upgrade/upgrade.go | 10 +-- .../storage-class/vanilla-zfs-csi.yaml | 13 ++++ test/testdata/storage-class/vanilla-zfs.yaml | 12 ++++ .../volume-snapshot-class/vanilla-zfs.yaml | 10 +++ test/types.go | 12 ++++ test/util/common/common.go | 3 + test/util/k8s/common.go | 17 +++++ test/util/k8s/deployment.go | 7 ++- test/util/k8s/namespace.go | 29 ++++++--- test/util/k8s/node.go | 11 +++- test/util/k8s/pvc.go | 3 +- test/util/kibishii/kibishii_utils.go | 22 +++---- test/util/providers/common.go | 28 ++++++--- test/util/velero/install.go | 17 ++--- test/util/velero/velero_utils.go | 17 +++-- 25 files changed, 255 insertions(+), 122 deletions(-) create mode 100644 test/testdata/storage-class/vanilla-zfs-csi.yaml create mode 100644 test/testdata/storage-class/vanilla-zfs.yaml create mode 100644 test/testdata/volume-snapshot-class/vanilla-zfs.yaml diff --git a/test/e2e/Makefile b/test/e2e/Makefile index 13df7733eb..97a7ffb49a 100644 --- a/test/e2e/Makefile +++ b/test/e2e/Makefile @@ -56,7 +56,7 @@ VELERO_IMAGE ?= velero/velero:main PLUGINS ?= RESTORE_HELPER_IMAGE ?= #Released version only -UPGRADE_FROM_VELERO_VERSION ?= v1.10.2,v1.11.0 +UPGRADE_FROM_VELERO_VERSION ?= v1.12.3,v1.13.0 # UPGRADE_FROM_VELERO_CLI can has the same format(a list divided by comma) with UPGRADE_FROM_VELERO_VERSION # Upgrade tests will be executed sequently according to the list by UPGRADE_FROM_VELERO_VERSION # So although length of UPGRADE_FROM_VELERO_CLI list is not equal with UPGRADE_FROM_VELERO_VERSION @@ -64,7 +64,7 @@ UPGRADE_FROM_VELERO_VERSION ?= v1.10.2,v1.11.0 # to the end, nil string will be set if UPGRADE_FROM_VELERO_CLI is shorter than UPGRADE_FROM_VELERO_VERSION UPGRADE_FROM_VELERO_CLI ?= -MIGRATE_FROM_VELERO_VERSION ?= v1.11.0,self +MIGRATE_FROM_VELERO_VERSION ?= v1.13.0,self MIGRATE_FROM_VELERO_CLI ?= VELERO_NAMESPACE ?= velero diff --git a/test/e2e/backup/backup.go b/test/e2e/backup/backup.go index a88745c769..38d3ed6878 100644 --- a/test/e2e/backup/backup.go +++ b/test/e2e/backup/backup.go @@ -74,18 +74,19 @@ func BackupRestoreTest(backupRestoreTestConfig BackupRestoreTestConfig) { veleroCfg.KibishiiDirectory = veleroCfg.KibishiiDirectory + backupRestoreTestConfig.kibishiiPatchSubDir veleroCfg.UseVolumeSnapshots = useVolumeSnapshots veleroCfg.UseNodeAgent = !useVolumeSnapshots - if veleroCfg.CloudProvider == "kind" { + if veleroCfg.CloudProvider == Kind { Skip("Volume snapshots plugin and File System Backups are not supported on kind") // on kind cluster snapshots are not supported since there is no velero snapshot plugin for kind volumes. // and PodVolumeBackups are not supported because PVB creation gets skipped for hostpath volumes, which are the only // volumes created on kind clusters using the default storage class and provisioner (provisioner: rancher.io/local-path) // This test suite checks for volume snapshots and PVBs generated from FileSystemBackups, so skip it on kind clusters } + // [SKIP]: Static provisioning for vSphere CSI driver works differently from other drivers. // For vSphere CSI, after you create a PV specifying an existing volume handle, CSI // syncer will need to register it with CNS. For other CSI drivers, static provisioning // usually does not go through storage system at all. That's probably why it took longer - if backupRestoreTestConfig.isRetainPVTest && veleroCfg.CloudProvider == "vsphere" { + if backupRestoreTestConfig.isRetainPVTest && veleroCfg.CloudProvider == Vsphere { Skip("Skip due to vSphere CSI driver long time issue of Static provisioning") } var err error diff --git a/test/e2e/backups/deletion.go b/test/e2e/backups/deletion.go index a59ddfdfb4..30fe4440f3 100644 --- a/test/e2e/backups/deletion.go +++ b/test/e2e/backups/deletion.go @@ -54,8 +54,8 @@ func backup_deletion_test(useVolumeSnapshots bool) { veleroCfg.UseNodeAgent = !useVolumeSnapshots BeforeEach(func() { - if useVolumeSnapshots && veleroCfg.CloudProvider == "kind" { - Skip("Volume snapshots not supported on kind") + if useVolumeSnapshots && veleroCfg.CloudProvider == Kind { + Skip(fmt.Sprintf("Volume snapshots not supported on %s", Kind)) } var err error flag.Parse() @@ -134,7 +134,7 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupNam }) }) - if providerName == "vsphere" && useVolumeSnapshots { + if providerName == Vsphere && useVolumeSnapshots { // Wait for uploads started by the Velero Plugin for vSphere to complete // TODO - remove after upload progress monitoring is implemented fmt.Println("Waiting for vSphere uploads to complete") diff --git a/test/e2e/backups/ttl.go b/test/e2e/backups/ttl.go index 329e95ea3d..49045266e7 100644 --- a/test/e2e/backups/ttl.go +++ b/test/e2e/backups/ttl.go @@ -120,7 +120,7 @@ func TTLTest() { var snapshotCheckPoint SnapshotCheckPoint if useVolumeSnapshots { - if veleroCfg.CloudProvider == "vsphere" { + if veleroCfg.CloudProvider == Vsphere { // TODO - remove after upload progress monitoring is implemented By("Waiting for vSphere uploads to complete", func() { Expect(WaitForVSphereUploadCompletion(ctx, time.Hour, @@ -140,7 +140,7 @@ func TTLTest() { fmt.Sprintf("Failed to delete namespace %s", BackupCfg.BackupName)) }) - if veleroCfg.CloudProvider == "aws" && useVolumeSnapshots { + if veleroCfg.CloudProvider == Aws && useVolumeSnapshots { fmt.Println("Waiting 7 minutes to make sure the snapshots are ready...") time.Sleep(7 * time.Minute) } diff --git a/test/e2e/basic/backup-volume-info/base.go b/test/e2e/basic/backup-volume-info/base.go index a10e75a052..26155a1201 100644 --- a/test/e2e/basic/backup-volume-info/base.go +++ b/test/e2e/basic/backup-volume-info/base.go @@ -26,9 +26,11 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "github.com/pkg/errors" + v1 "k8s.io/api/core/v1" . "github.com/vmware-tanzu/velero/test" . "github.com/vmware-tanzu/velero/test/e2e/test" + . "github.com/vmware-tanzu/velero/test/util/common" . "github.com/vmware-tanzu/velero/test/util/k8s" . "github.com/vmware-tanzu/velero/test/util/velero" ) @@ -45,12 +47,12 @@ func (v *BackupVolumeInfo) Init() error { v.TestCase.Init() BeforeEach(func() { - if v.VeleroCfg.CloudProvider == "vsphere" && (!strings.Contains(v.CaseBaseName, "fs-upload") && !strings.Contains(v.CaseBaseName, "skipped")) { + if v.VeleroCfg.CloudProvider == Vsphere && (!strings.Contains(v.CaseBaseName, "fs-upload") && !strings.Contains(v.CaseBaseName, "skipped")) { fmt.Printf("Skip snapshot case %s for vsphere environment.\n", v.CaseBaseName) Skip("Skip snapshot case due to vsphere environment doesn't cover the CSI test, and it doesn't have a Velero native snapshot plugin.") } - if strings.Contains(v.VeleroCfg.Features, "EnableCSI") { + if strings.Contains(v.VeleroCfg.Features, FeatureCSI) { if strings.Contains(v.CaseBaseName, "native-snapshot") { fmt.Printf("Skip native snapshot case %s when the CSI feature is enabled.\n", v.CaseBaseName) Skip("Skip native snapshot case due to CSI feature is enabled.") @@ -111,14 +113,22 @@ func (v *BackupVolumeInfo) CreateResources() error { // Install StorageClass Expect(InstallTestStorageClasses(fmt.Sprintf("../testdata/storage-class/%s-csi.yaml", v.VeleroCfg.CloudProvider))).To(Succeed(), "Failed to install StorageClass") - pvc, err := CreatePVC(v.Client, createNSName, "volume-info", CSIStorageClassName, nil) - Expect(err).To(Succeed()) - vols := CreateVolumes(pvc.Name, []string{"volume-info"}) - - //Create deployment + // Create deployment fmt.Printf("Creating deployment in namespaces ...%s\n", createNSName) + // Make sure PVC count is great than 3 to allow both empty volumes and file populated volumes exist per pod + pvcCount := 4 + Expect(pvcCount > 3).To(Equal(true)) + + var vols []*v1.Volume + for i := 0; i <= pvcCount-1; i++ { + pvcName := fmt.Sprintf("volume-info-pvc-%d", i) + pvc, err := CreatePVC(v.Client, createNSName, pvcName, CSIStorageClassName, nil) + Expect(err).To(Succeed()) + volumeName := fmt.Sprintf("volume-info-pv-%d", i) + vols = append(vols, CreateVolumes(pvc.Name, []string{volumeName})...) + } deployment := NewDeployment(v.CaseBaseName, createNSName, 1, labels, nil).WithVolume(vols).Result() - deployment, err = CreateDeployment(v.Client.ClientGo, createNSName, deployment) + deployment, err := CreateDeployment(v.Client.ClientGo, createNSName, deployment) if err != nil { return errors.Wrap(err, fmt.Sprintf("failed to delete the namespace %q", createNSName)) } @@ -126,6 +136,19 @@ func (v *BackupVolumeInfo) CreateResources() error { if err != nil { return errors.Wrap(err, fmt.Sprintf("failed to ensure job completion in namespace: %q", createNSName)) } + podList, err := ListPods(v.Ctx, v.Client, createNSName) + Expect(err).To(Succeed(), fmt.Sprintf("failed to list pods in namespace: %q with error %v", createNSName, err)) + + for _, pod := range podList.Items { + for i := 0; i <= pvcCount-1; i++ { + // Hitting issue https://github.com/vmware-tanzu/velero/issues/7388 + // So populate data only to some of pods, leave other pods empty to verify empty PV datamover + if i%2 == 0 { + Expect(CreateFileToPod(v.Ctx, createNSName, pod.Name, DefaultContainerName, vols[i].Name, + fmt.Sprintf("file-%s", pod.Name), CreateFileContent(createNSName, pod.Name, vols[i].Name))).To(Succeed()) + } + } + } } return nil } diff --git a/test/e2e/bsl-mgmt/deletion.go b/test/e2e/bsl-mgmt/deletion.go index 0ff5a9fa98..9dc6f8e95a 100644 --- a/test/e2e/bsl-mgmt/deletion.go +++ b/test/e2e/bsl-mgmt/deletion.go @@ -61,8 +61,8 @@ func BslDeletionTest(useVolumeSnapshots bool) { less := func(a, b string) bool { return a < b } BeforeEach(func() { - if useVolumeSnapshots && veleroCfg.CloudProvider == "kind" { - Skip("Volume snapshots not supported on kind") + if useVolumeSnapshots && veleroCfg.CloudProvider == Kind { + Skip(fmt.Sprintf("Volume snapshots not supported on %s", Kind)) } var err error flag.Parse() @@ -203,7 +203,7 @@ func BslDeletionTest(useVolumeSnapshots bool) { }) if useVolumeSnapshots { - if veleroCfg.CloudProvider == "vsphere" { + if veleroCfg.CloudProvider == Vsphere { // TODO - remove after upload progress monitoring is implemented By("Waiting for vSphere uploads to complete", func() { Expect(WaitForVSphereUploadCompletion(oneHourTimeout, time.Hour, @@ -218,32 +218,33 @@ func BslDeletionTest(useVolumeSnapshots bool) { backupName_2, 1)).To(Succeed()) }) } - - var snapshotCheckPoint SnapshotCheckPoint - snapshotCheckPoint.NamespaceBackedUp = bslDeletionTestNs - By(fmt.Sprintf("Snapshot of bsl %s should be created in cloud object store", backupLocation_1), func() { - snapshotCheckPoint, err = GetSnapshotCheckPoint(*veleroCfg.ClientToInstallVelero, veleroCfg, 1, bslDeletionTestNs, backupName_1, []string{podName_1}) - Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint") - Expect(SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider, - veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, - veleroCfg.BSLConfig, backupName_1, snapshotCheckPoint)).To(Succeed()) - }) - By(fmt.Sprintf("Snapshot of bsl %s should be created in cloud object store", backupLocation_2), func() { - snapshotCheckPoint, err = GetSnapshotCheckPoint(*veleroCfg.ClientToInstallVelero, veleroCfg, 1, bslDeletionTestNs, backupName_2, []string{podName_2}) - Expect(err).NotTo(HaveOccurred(), "Fail to get snapshot checkpoint") - var BSLCredentials, BSLConfig string - if veleroCfg.CloudProvider == "vsphere" { - BSLCredentials = veleroCfg.AdditionalBSLCredentials - BSLConfig = veleroCfg.AdditionalBSLConfig - } else { // Snapshotting with non-vSphere provider has nothing to do with BSL - BSLCredentials = veleroCfg.CloudCredentialsFile - BSLConfig = veleroCfg.BSLConfig - } - - Expect(SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider, - BSLCredentials, veleroCfg.AdditionalBSLBucket, - BSLConfig, backupName_2, snapshotCheckPoint)).To(Succeed()) - }) + if veleroCfg.CloudProvider != VanillaZFS { + var snapshotCheckPoint SnapshotCheckPoint + snapshotCheckPoint.NamespaceBackedUp = bslDeletionTestNs + By(fmt.Sprintf("Snapshot of bsl %s should be created in cloud object store", backupLocation_1), func() { + snapshotCheckPoint, err = GetSnapshotCheckPoint(*veleroCfg.ClientToInstallVelero, veleroCfg, 1, bslDeletionTestNs, backupName_1, []string{podName_1}) + Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint") + Expect(SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider, + veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, + veleroCfg.BSLConfig, backupName_1, snapshotCheckPoint)).To(Succeed()) + }) + By(fmt.Sprintf("Snapshot of bsl %s should be created in cloud object store", backupLocation_2), func() { + snapshotCheckPoint, err = GetSnapshotCheckPoint(*veleroCfg.ClientToInstallVelero, veleroCfg, 1, bslDeletionTestNs, backupName_2, []string{podName_2}) + Expect(err).NotTo(HaveOccurred(), "Fail to get snapshot checkpoint") + var BSLCredentials, BSLConfig string + if veleroCfg.CloudProvider == Vsphere { + BSLCredentials = veleroCfg.AdditionalBSLCredentials + BSLConfig = veleroCfg.AdditionalBSLConfig + } else { // Snapshotting with non-vSphere provider has nothing to do with BSL + BSLCredentials = veleroCfg.CloudCredentialsFile + BSLConfig = veleroCfg.BSLConfig + } + + Expect(SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider, + BSLCredentials, veleroCfg.AdditionalBSLBucket, + BSLConfig, backupName_2, snapshotCheckPoint)).To(Succeed()) + }) + } } else { // For Restics By(fmt.Sprintf("Resticrepositories for BSL %s should be created in Velero namespace", backupLocation_1), func() { Expect(BackupRepositoriesCountShouldBe(context.Background(), @@ -309,7 +310,7 @@ func BslDeletionTest(useVolumeSnapshots bool) { // }) if useVolumeSnapshots { - if veleroCfg.CloudProvider == "vsphere" { + if veleroCfg.CloudProvider == Vsphere { By(fmt.Sprintf("Snapshot in backup %s should still exist, because snapshot CR will be deleted 24 hours later if the status is a success", backupName_2), func() { Expect(SnapshotCRsCountShouldBe(context.Background(), bslDeletionTestNs, backupName_1, 1)).To(Succeed()) @@ -329,7 +330,7 @@ func BslDeletionTest(useVolumeSnapshots bool) { }) By(fmt.Sprintf("Snapshot should not be deleted in cloud object store after deleting bsl %s", backupLocation_2), func() { var BSLCredentials, BSLConfig string - if veleroCfg.CloudProvider == "vsphere" { + if veleroCfg.CloudProvider == Vsphere { BSLCredentials = veleroCfg.AdditionalBSLCredentials BSLConfig = veleroCfg.AdditionalBSLConfig } else { diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index aa205c39ca..906846039f 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -104,13 +104,20 @@ func init() { } +// Add label [SkipVanillaZfs]: +// We found issue - https://github.com/openebs/zfs-localpv/issues/123 when using OpenEBS ZFS CSI Driver +// When PVC using storage class with reclaim policy as 'Delete', once PVC is deleted, snapshot associated will be deleted +// along with PVC deletion, after restoring workload, restored PVC is in pending status, due to failure of provision PV +// caused by no expected snapshot found. If we use retain as reclaim policy, then this label can be ignored, all test +// cases can be executed as expected successful result. + var _ = Describe("[APIGroup][APIVersion][SKIP_KIND] Velero tests with various CRD API group versions", APIGropuVersionsTest) var _ = Describe("[APIGroup][APIExtensions][SKIP_KIND] CRD of apiextentions v1beta1 should be B/R successfully from cluster(k8s version < 1.22) to cluster(k8s version >= 1.22)", APIExtensionsVersionsTest) // Test backup and restore of Kibishi using restic var _ = Describe("[Basic][Restic] Velero tests on cluster using the plugin provider for object storage and Restic for volume backups", BackupRestoreWithRestic) -var _ = Describe("[Basic][Snapshot] Velero tests on cluster using the plugin provider for object storage and snapshots for volume backups", BackupRestoreWithSnapshots) +var _ = Describe("[Basic][Snapshot][SkipVanillaZfs] Velero tests on cluster using the plugin provider for object storage and snapshots for volume backups", BackupRestoreWithSnapshots) var _ = Describe("[Basic][Snapshot][RetainPV] Velero tests on cluster using the plugin provider for object storage and snapshots for volume backups", BackupRestoreRetainedPVWithSnapshots) @@ -122,7 +129,7 @@ var _ = Describe("[Scale][LongTime] Backup/restore of 2500 namespaces", MultiNSB // Upgrade test by Kibishi using restic var _ = Describe("[Upgrade][Restic] Velero upgrade tests on cluster using the plugin provider for object storage and Restic for volume backups", BackupUpgradeRestoreWithRestic) -var _ = Describe("[Upgrade][Snapshot] Velero upgrade tests on cluster using the plugin provider for object storage and snapshots for volume backups", BackupUpgradeRestoreWithSnapshots) +var _ = Describe("[Upgrade][Snapshot][SkipVanillaZfs] Velero upgrade tests on cluster using the plugin provider for object storage and snapshots for volume backups", BackupUpgradeRestoreWithSnapshots) // test filter objects by namespace, type, or labels when backup or restore. var _ = Describe("[ResourceFiltering][ExcludeFromBackup] Resources with the label velero.io/exclude-from-backup=true are not included in backup", ExcludeFromBackupTest) @@ -147,8 +154,8 @@ var _ = Describe("[BackupVolumeInfo][NativeSnapshot]", NativeSnapshotVolumeInfoT var _ = Describe("[ResourceModifier][Restore] Velero test on resource modifiers from the cluster restore", ResourceModifiersTest) var _ = Describe("[Backups][Deletion][Restic] Velero tests of Restic backup deletion", BackupDeletionWithRestic) -var _ = Describe("[Backups][Deletion][Snapshot] Velero tests of snapshot backup deletion", BackupDeletionWithSnapshots) -var _ = Describe("[Backups][TTL][LongTime][Snapshot] Local backups and restic repos will be deleted once the corresponding backup storage location is deleted", TTLTest) +var _ = Describe("[Backups][Deletion][Snapshot][SkipVanillaZfs] Velero tests of snapshot backup deletion", BackupDeletionWithSnapshots) +var _ = Describe("[Backups][TTL][LongTime][Snapshot][SkipVanillaZfs] Local backups and restic repos will be deleted once the corresponding backup storage location is deleted", TTLTest) var _ = Describe("[Backups][BackupsSync] Backups in object storage are synced to a new Velero and deleted backups in object storage are synced to be deleted in Velero", BackupsSyncTest) var _ = Describe("[Schedule][BR][Pause][LongTime] Backup will be created periodly by schedule defined by a Cron expression", ScheduleBackupTest) @@ -157,16 +164,16 @@ var _ = Describe("[Schedule][BackupCreation][SKIP_KIND] Schedule controller woul var _ = Describe("[PrivilegesMgmt][SSR] Velero test on ssr object when controller namespace mix-ups", SSRTest) -var _ = Describe("[BSL][Deletion][Snapshot] Local backups will be deleted once the corresponding backup storage location is deleted", BslDeletionWithSnapshots) +var _ = Describe("[BSL][Deletion][Snapshot][SkipVanillaZfs] Local backups will be deleted once the corresponding backup storage location is deleted", BslDeletionWithSnapshots) var _ = Describe("[BSL][Deletion][Restic] Local backups and restic repos will be deleted once the corresponding backup storage location is deleted", BslDeletionWithRestic) var _ = Describe("[Migration][Restic] Migrate resources between clusters by Restic", MigrationWithRestic) -var _ = Describe("[Migration][Snapshot] Migrate resources between clusters by snapshot", MigrationWithSnapshots) +var _ = Describe("[Migration][Snapshot][SkipVanillaZfs] Migrate resources between clusters by snapshot", MigrationWithSnapshots) var _ = Describe("[NamespaceMapping][Single][Restic] Backup resources should follow the specific order in schedule", OneNamespaceMappingResticTest) var _ = Describe("[NamespaceMapping][Multiple][Restic] Backup resources should follow the specific order in schedule", MultiNamespacesMappingResticTest) -var _ = Describe("[NamespaceMapping][Single][Snapshot] Backup resources should follow the specific order in schedule", OneNamespaceMappingSnapshotTest) -var _ = Describe("[NamespaceMapping][Multiple][Snapshot] Backup resources should follow the specific order in schedule", MultiNamespacesMappingSnapshotTest) +var _ = Describe("[NamespaceMapping][Single][Snapshot][SkipVanillaZfs] Backup resources should follow the specific order in schedule", OneNamespaceMappingSnapshotTest) +var _ = Describe("[NamespaceMapping][Multiple][Snapshot]SkipVanillaZfs] Backup resources should follow the specific order in schedule", MultiNamespacesMappingSnapshotTest) var _ = Describe("[pv-backup][Opt-In] Backup resources should follow the specific order in schedule", OptInPVBackupTest) var _ = Describe("[pv-backup][Opt-Out] Backup resources should follow the specific order in schedule", OptOutPVBackupTest) diff --git a/test/e2e/migration/migration.go b/test/e2e/migration/migration.go index 0db2d983c1..b85c7b347c 100644 --- a/test/e2e/migration/migration.go +++ b/test/e2e/migration/migration.go @@ -60,8 +60,8 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) veleroCfg = VeleroCfg UUIDgen, err = uuid.NewRandom() migrationNamespace = "migration-" + UUIDgen.String() - if useVolumeSnapshots && veleroCfg.CloudProvider == "kind" { - Skip("Volume snapshots not supported on kind") + if useVolumeSnapshots && veleroCfg.CloudProvider == Kind { + Skip(fmt.Sprintf("Volume snapshots not supported on %s", Kind)) } if veleroCfg.DefaultClusterContext == "" && veleroCfg.StandbyClusterContext == "" { @@ -235,7 +235,7 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) }) if useVolumeSnapshots { - if veleroCfg.CloudProvider == "vsphere" { + if veleroCfg.CloudProvider == Vsphere { // TODO - remove after upload progress monitoring is implemented By("Waiting for vSphere uploads to complete", func() { Expect(WaitForVSphereUploadCompletion(context.Background(), time.Hour, @@ -260,7 +260,7 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) } } - if useVolumeSnapshots && veleroCfg.CloudProvider == "azure" && + if useVolumeSnapshots && veleroCfg.CloudProvider == Azure && strings.EqualFold(veleroCfg.Features, FeatureCSI) && !OriginVeleroCfg.SnapshotMoveData { By("Sleep 5 minutes to avoid snapshot recreated by unknown reason ", func() { @@ -270,7 +270,7 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) // the snapshots of AWS may be still in pending status when do the restore, wait for a while // to avoid this https://github.com/vmware-tanzu/velero/issues/1799 // TODO remove this after https://github.com/vmware-tanzu/velero/issues/3533 is fixed - if veleroCfg.CloudProvider == "aws" && useVolumeSnapshots && !OriginVeleroCfg.SnapshotMoveData { + if veleroCfg.CloudProvider == Aws && useVolumeSnapshots && !OriginVeleroCfg.SnapshotMoveData { fmt.Println("Waiting 5 minutes to make sure the snapshots are ready...") time.Sleep(5 * time.Minute) } diff --git a/test/e2e/pv-backup/pv-backup-filter.go b/test/e2e/pv-backup/pv-backup-filter.go index 07b14d0c16..f6fd7993d0 100644 --- a/test/e2e/pv-backup/pv-backup-filter.go +++ b/test/e2e/pv-backup/pv-backup-filter.go @@ -12,6 +12,7 @@ import ( . "github.com/vmware-tanzu/velero/test" . "github.com/vmware-tanzu/velero/test/e2e/test" + . "github.com/vmware-tanzu/velero/test/util/common" . "github.com/vmware-tanzu/velero/test/util/k8s" ) @@ -122,7 +123,7 @@ func (p *PVBackupFiltering) CreateResources() error { for i, pod := range p.podsList[index] { for j := range p.volumesList[i] { Expect(CreateFileToPod(p.Ctx, ns, pod, pod, p.volumesList[i][j], - FILE_NAME, fileContent(ns, pod, p.volumesList[i][j]))).To(Succeed()) + FILE_NAME, CreateFileContent(ns, pod, p.volumesList[i][j]))).To(Succeed()) } } }) @@ -173,9 +174,6 @@ func (p *PVBackupFiltering) Verify() error { return nil } -func fileContent(namespace, podName, volume string) string { - return fmt.Sprintf("ns-%s pod-%s volume-%s", namespace, podName, volume) -} func fileExist(ctx context.Context, namespace, podName, volume string) error { c, _, err := ReadFileFromPodVolume(ctx, namespace, podName, podName, volume, FILE_NAME) @@ -184,7 +182,7 @@ func fileExist(ctx context.Context, namespace, podName, volume string) error { FILE_NAME, volume, podName, namespace)) } c = strings.Replace(c, "\n", "", -1) - origin_content := strings.Replace(fileContent(namespace, podName, volume), "\n", "", -1) + origin_content := strings.Replace(CreateFileContent(namespace, podName, volume), "\n", "", -1) if c == origin_content { return nil } else { diff --git a/test/e2e/test/test.go b/test/e2e/test/test.go index 3eac6213f6..15bde3958c 100644 --- a/test/e2e/test/test.go +++ b/test/e2e/test/test.go @@ -86,7 +86,7 @@ func TestFunc(test VeleroBackupRestoreTest) func() { // TODO: Skip nodeport test until issue https://github.com/kubernetes/kubernetes/issues/114384 fixed // TODO: Although this issue is closed, but it's not fixed. // TODO: After bump up k8s version in AWS pipeline, this issue also apply for AWS pipeline. - if (veleroCfg.CloudProvider == "azure" || veleroCfg.CloudProvider == "aws") && strings.Contains(test.GetTestCase().CaseBaseName, "nodeport") { + if (veleroCfg.CloudProvider == Azure || veleroCfg.CloudProvider == Aws) && strings.Contains(test.GetTestCase().CaseBaseName, "nodeport") { Skip("Skip due to issue https://github.com/kubernetes/kubernetes/issues/114384 on AKS") } if InstallVelero { @@ -164,7 +164,7 @@ func (t *TestCase) Restore() error { // the snapshots of AWS may be still in pending status when do the restore, wait for a while // to avoid this https://github.com/vmware-tanzu/velero/issues/1799 // TODO remove this after https://github.com/vmware-tanzu/velero/issues/3533 is fixed - if t.UseVolumeSnapshots && veleroCfg.CloudProvider != "vsphere" { + if t.UseVolumeSnapshots && veleroCfg.CloudProvider != Vsphere { fmt.Println("Waiting 5 minutes to make sure the snapshots are ready...") time.Sleep(5 * time.Minute) } diff --git a/test/e2e/upgrade/upgrade.go b/test/e2e/upgrade/upgrade.go index 2ff0dcf1e5..e6e9b16736 100644 --- a/test/e2e/upgrade/upgrade.go +++ b/test/e2e/upgrade/upgrade.go @@ -68,8 +68,8 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC if (len(veleroCfg.UpgradeFromVeleroVersion)) == 0 { Skip("An original velero version is required to run upgrade test, please run test with upgrade-from-velero-version=") } - if useVolumeSnapshots && veleroCfg.CloudProvider == "kind" { - Skip("Volume snapshots not supported on kind") + if useVolumeSnapshots && veleroCfg.CloudProvider == Kind { + Skip(fmt.Sprintf("Volume snapshots not supported on %s", Kind)) } if veleroCfg.VeleroCLI == "" { Skip("VeleroCLI should be provide") @@ -190,7 +190,7 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC }) if useVolumeSnapshots { - if veleroCfg.CloudProvider == "vsphere" { + if veleroCfg.CloudProvider == Vsphere { // TODO - remove after upload progress monitoring is implemented By("Waiting for vSphere uploads to complete", func() { Expect(WaitForVSphereUploadCompletion(oneHourTimeout, time.Hour, @@ -214,7 +214,7 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC fmt.Sprintf("failed to delete namespace %s", upgradeNamespace)) }) - if useVolumeSnapshots && veleroCfg.CloudProvider == "azure" && strings.EqualFold(veleroCfg.Features, FeatureCSI) { + if useVolumeSnapshots && veleroCfg.CloudProvider == Azure && strings.EqualFold(veleroCfg.Features, FeatureCSI) { // Upgrade test is not running daily since no CSI plugin v1.0 released, because builds before // v1.0 have issues to fail upgrade case. By("Sleep 5 minutes to avoid snapshot recreated by unknown reason ", func() { @@ -224,7 +224,7 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC // the snapshots of AWS may be still in pending status when do the restore, wait for a while // to avoid this https://github.com/vmware-tanzu/velero/issues/1799 // TODO remove this after https://github.com/vmware-tanzu/velero/issues/3533 is fixed - if tmpCfg.CloudProvider == "aws" && useVolumeSnapshots { + if tmpCfg.CloudProvider == Aws && useVolumeSnapshots { fmt.Println("Waiting 5 minutes to make sure the snapshots are ready...") time.Sleep(5 * time.Minute) } diff --git a/test/testdata/storage-class/vanilla-zfs-csi.yaml b/test/testdata/storage-class/vanilla-zfs-csi.yaml new file mode 100644 index 0000000000..697b80a536 --- /dev/null +++ b/test/testdata/storage-class/vanilla-zfs-csi.yaml @@ -0,0 +1,13 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: e2e-csi-storage-class +parameters: + recordsize: "128k" + compression: "off" + dedup: "off" + fstype: "zfs" + poolname: "zfspv-pool" +provisioner: zfs.csi.openebs.io +volumeBindingMode: WaitForFirstConsumer +reclaimPolicy: Retain \ No newline at end of file diff --git a/test/testdata/storage-class/vanilla-zfs.yaml b/test/testdata/storage-class/vanilla-zfs.yaml new file mode 100644 index 0000000000..c7ada98b83 --- /dev/null +++ b/test/testdata/storage-class/vanilla-zfs.yaml @@ -0,0 +1,12 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: e2e-storage-class +parameters: + recordsize: "128k" + compression: "off" + dedup: "off" + fstype: "zfs" + poolname: "zfspv-pool" +provisioner: zfs.csi.openebs.io +volumeBindingMode: WaitForFirstConsumer \ No newline at end of file diff --git a/test/testdata/volume-snapshot-class/vanilla-zfs.yaml b/test/testdata/volume-snapshot-class/vanilla-zfs.yaml new file mode 100644 index 0000000000..3649215e84 --- /dev/null +++ b/test/testdata/volume-snapshot-class/vanilla-zfs.yaml @@ -0,0 +1,10 @@ +kind: VolumeSnapshotClass +apiVersion: snapshot.storage.k8s.io/v1 +metadata: + name: zfspv-snapclass + annotations: + snapshot.storage.kubernetes.io/is-default-class: "true" + labels: + velero.io/csi-volumesnapshot-class: "true" +driver: zfs.csi.openebs.io +deletionPolicy: Delete \ No newline at end of file diff --git a/test/types.go b/test/types.go index 2a4e06f291..e64c6d677a 100644 --- a/test/types.go +++ b/test/types.go @@ -29,6 +29,18 @@ const StorageClassName = "e2e-storage-class" const StorageClassName2 = "e2e-storage-class-2" const CSIStorageClassName = "e2e-csi-storage-class" const FeatureCSI = "EnableCSI" +const VanillaZFS = "vanilla-zfs" +const Kind = "kind" +const Azure = "azure" +const AzureCSI = "azure-csi" +const AwsCSI = "aws-csi" +const Aws = "aws" +const Gcp = "gcp" +const Vsphere = "vsphere" + +var PublicCloudProviders = []string{Aws, Azure, Gcp, Vsphere} +var LocalCloudProviders = []string{Kind, VanillaZFS} +var CloudProviders = append(PublicCloudProviders, LocalCloudProviders...) var InstallVelero bool var UUIDgen uuid.UUID diff --git a/test/util/common/common.go b/test/util/common/common.go index 420eec286c..d014d2a09f 100644 --- a/test/util/common/common.go +++ b/test/util/common/common.go @@ -110,3 +110,6 @@ func WriteToFile(content, fileName string) error { } return nil } +func CreateFileContent(namespace, podName, volume string) string { + return fmt.Sprintf("ns-%s pod-%s volume-%s", namespace, podName, volume) +} diff --git a/test/util/k8s/common.go b/test/util/k8s/common.go index 4f98f26340..728704ef60 100644 --- a/test/util/k8s/common.go +++ b/test/util/k8s/common.go @@ -176,6 +176,23 @@ func GetCRD(ctx context.Context, name string) ([]string, error) { return common.GetListByCmdPipes(ctx, cmds) } +func KubectlGetNS(ctx context.Context, name string) ([]string, error) { + cmds := []*common.OsCommandLine{} + cmd := &common.OsCommandLine{ + Cmd: "kubectl", + Args: []string{"get", "ns"}, + } + cmds = append(cmds, cmd) + + cmd = &common.OsCommandLine{ + Cmd: "grep", + Args: []string{name}, + } + cmds = append(cmds, cmd) + + return common.GetListByCmdPipes(ctx, cmds) +} + func AddLabelToPv(ctx context.Context, pv, label string) error { return exec.CommandContext(ctx, "kubectl", "label", "pv", pv, label).Run() } diff --git a/test/util/k8s/deployment.go b/test/util/k8s/deployment.go index 15a5d372f6..0fff022873 100644 --- a/test/util/k8s/deployment.go +++ b/test/util/k8s/deployment.go @@ -33,8 +33,9 @@ import ( const ( JobSelectorKey = "job" // Poll is how often to Poll pods, nodes and claims. - PollInterval = 2 * time.Second - PollTimeout = 15 * time.Minute + PollInterval = 2 * time.Second + PollTimeout = 15 * time.Minute + DefaultContainerName = "container-busybox" ) // DeploymentBuilder builds Deployment objects. @@ -51,7 +52,7 @@ func NewDeployment(name, ns string, replicas int32, labels map[string]string, co if containers == nil { containers = []v1.Container{ { - Name: "container-busybox", + Name: DefaultContainerName, Image: "gcr.io/velero-gcp/busybox:latest", Command: []string{"sleep", "1000000"}, // Make pod obeys the restricted pod security standards. diff --git a/test/util/k8s/namespace.go b/test/util/k8s/namespace.go index 479ac1b8a7..3790b8a966 100644 --- a/test/util/k8s/namespace.go +++ b/test/util/k8s/namespace.go @@ -85,21 +85,30 @@ func DeleteNamespace(ctx context.Context, client TestClient, namespace string, w var zero int64 = 0 policy := metav1.DeletePropagationForeground - if err := client.ClientGo.CoreV1().Namespaces().Delete(context.TODO(), namespace, metav1.DeleteOptions{ - GracePeriodSeconds: &zero, - PropagationPolicy: &policy, - }); err != nil { - return errors.Wrap(err, fmt.Sprintf("failed to delete the namespace %q", namespace)) - } - if !wait { - return nil - } + return waitutil.PollImmediateInfinite(5*time.Second, func() (bool, error) { - if _, err := client.ClientGo.CoreV1().Namespaces().Get(tenMinuteTimeout, namespace, metav1.GetOptions{}); err != nil { + // Retry namespace deletion, see issue: https://github.com/kubernetes/kubernetes/issues/60807 + if err := client.ClientGo.CoreV1().Namespaces().Delete(context.TODO(), namespace, metav1.DeleteOptions{ + GracePeriodSeconds: &zero, + PropagationPolicy: &policy, + }); err != nil { if apierrors.IsNotFound(err) { return true, nil + } else { + fmt.Printf("Delete namespace %s err: %v", namespace, err) + return false, errors.Wrap(err, fmt.Sprintf("failed to delete the namespace %q", namespace)) + } + } + if !wait { + return true, nil + } + ns, err := KubectlGetNS(tenMinuteTimeout, namespace) + if err != nil { + if len(ns) == 0 { + return true, nil } + fmt.Printf("Get namespace %s err: %v", namespace, err) return false, err } fmt.Printf("namespace %q is still being deleted...\n", namespace) diff --git a/test/util/k8s/node.go b/test/util/k8s/node.go index 9e402850c5..e1e44b17b1 100644 --- a/test/util/k8s/node.go +++ b/test/util/k8s/node.go @@ -27,17 +27,22 @@ func GetWorkerNodes(ctx context.Context) ([]string, error) { } var nodeNameList []string for nodeIndex, node := range nodes.Items { - // := v1.Node{} fmt.Println(nodeIndex) fmt.Println(node.GetName()) anns := node.GetAnnotations() + lbls := node.GetLabels() fmt.Println(anns) - fmt.Println(anns["cluster.x-k8s.io/owner-kind"]) - //"MachineSet" + fmt.Println(lbls) + // For Kubeadm vanilla cluster control-plane node selection + if _, ok := lbls["node-role.kubernetes.io/control-plane"]; ok { + continue + } + // For public cloud provider cluster control-plane node selection if anns["cluster.x-k8s.io/owner-kind"] == "KubeadmControlPlane" { continue } nodeNameList = append(nodeNameList, node.GetName()) } + fmt.Println(nodeNameList) return nodeNameList, nil } diff --git a/test/util/k8s/pvc.go b/test/util/k8s/pvc.go index 428064e53a..6eda352ea2 100644 --- a/test/util/k8s/pvc.go +++ b/test/util/k8s/pvc.go @@ -34,8 +34,7 @@ func (p *PVCBuilder) Result() *corev1.PersistentVolumeClaim { } func NewPVC(ns, name string) *PVCBuilder { - oMeta := metav1.ObjectMeta{} - oMeta = metav1.ObjectMeta{Name: name, Namespace: ns} + oMeta := metav1.ObjectMeta{Name: name, Namespace: ns} return &PVCBuilder{ &corev1.PersistentVolumeClaim{ ObjectMeta: oMeta, diff --git a/test/util/kibishii/kibishii_utils.go b/test/util/kibishii/kibishii_utils.go index 13280d6fa6..5046f4f21c 100644 --- a/test/util/kibishii/kibishii_utils.go +++ b/test/util/kibishii/kibishii_utils.go @@ -111,7 +111,7 @@ func RunKibishiiTests(veleroCfg VeleroConfig, backupName, restoreName, backupLoc // Checkpoint for a successful backup if useVolumeSnapshots { - if providerName == "vsphere" { + if providerName == Vsphere { // Wait for uploads started by the Velero Plugin for vSphere to complete // TODO - remove after upload progress monitoring is implemented fmt.Println("Waiting for vSphere uploads to complete") @@ -137,7 +137,7 @@ func RunKibishiiTests(veleroCfg VeleroConfig, backupName, restoreName, backupLoc if len(pvbs) != pvCount { return errors.New(fmt.Sprintf("PVB count %d should be %d in namespace %s", len(pvbs), pvCount, kibishiiNamespace)) } - if providerName == "vsphere" { + if providerName == Vsphere { // Wait for uploads started by the Velero Plugin for vSphere to complete // TODO - remove after upload progress monitoring is implemented @@ -149,8 +149,8 @@ func RunKibishiiTests(veleroCfg VeleroConfig, backupName, restoreName, backupLoc // return errors.Wrapf(err, "Error get vSphere snapshot uploads") // } } else { - // wait for a period to confirm no snapshots exist for the backup - time.Sleep(5 * time.Minute) + // wait for a period to confirm no snapshots content exist for the backup + time.Sleep(1 * time.Minute) if strings.EqualFold(veleroFeatures, FeatureCSI) { _, err = GetSnapshotCheckPoint(*veleroCfg.ClientToInstallVelero, veleroCfg, 0, kibishiiNamespace, backupName, KibishiiPVCNameList) @@ -176,9 +176,9 @@ func RunKibishiiTests(veleroCfg VeleroConfig, backupName, restoreName, backupLoc // To ensure Kibishii verification result is accurate ClearKibishiiData(oneHourTimeout, kibishiiNamespace, pod, "kibishii", "data") - fileContent := fileBaseContent + pod + CreateFileContent := fileBaseContent + pod err := CreateFileToPod(oneHourTimeout, kibishiiNamespace, pod, "kibishii", "data", - fileName, fileContent) + fileName, CreateFileContent) if err != nil { return errors.Wrapf(err, "failed to create file %s", fileName) } @@ -224,7 +224,7 @@ func RunKibishiiTests(veleroCfg VeleroConfig, backupName, restoreName, backupLoc RunDebug(context.Background(), veleroCLI, veleroNamespace, "", restoreName) return errors.Wrapf(err, "Restore %s failed from backup %s", restoreName, backupName) } - if !useVolumeSnapshots && providerName != "vsphere" { + if !useVolumeSnapshots && providerName != Vsphere { pvrs, err := GetPVR(oneHourTimeout, veleroCfg.VeleroNamespace, kibishiiNamespace) if err != nil { return errors.Wrapf(err, "failed to get PVR for namespace %s", kibishiiNamespace) @@ -244,13 +244,13 @@ func RunKibishiiTests(veleroCfg VeleroConfig, backupName, restoreName, backupLoc func installKibishii(ctx context.Context, namespace string, cloudPlatform, veleroFeatures, kibishiiDirectory string, useVolumeSnapshots bool, workerReplicas int) error { - if strings.EqualFold(cloudPlatform, "azure") && + if strings.EqualFold(cloudPlatform, Azure) && strings.EqualFold(veleroFeatures, FeatureCSI) { - cloudPlatform = "azure-csi" + cloudPlatform = AzureCSI } - if strings.EqualFold(cloudPlatform, "aws") && + if strings.EqualFold(cloudPlatform, Aws) && strings.EqualFold(veleroFeatures, FeatureCSI) { - cloudPlatform = "aws-csi" + cloudPlatform = AwsCSI } // We use kustomize to generate YAML for Kibishii from the checked-in yaml directories kibishiiInstallCmd := exec.CommandContext(ctx, "kubectl", "apply", "-n", namespace, "-k", diff --git a/test/util/providers/common.go b/test/util/providers/common.go index d8ca012517..dcb599f253 100644 --- a/test/util/providers/common.go +++ b/test/util/providers/common.go @@ -70,18 +70,18 @@ func ObjectsShouldNotBeInBucket(objectStoreProvider, cloudCredentialsFile, bslBu } // This function returns a storage interface based on the cloud provider for querying objects and snapshots -// When cloudProvider is kind, pass in object storage provider instead. For example, "aws". +// When cloudProvider is kind, pass in object storage provider instead. For example, AWS // Snapshots are not supported on kind. func getProvider(cloudProvider string) (ObjectsInStorage, error) { var s ObjectsInStorage switch cloudProvider { - case "aws", "vsphere": + case Aws, Vsphere: aws := AWSStorage("") s = &aws - case "gcp": + case Gcp: gcs := GCSStorage("") s = &gcs - case "azure": + case Azure: az := AzureStorage("") s = &az default: @@ -110,6 +110,7 @@ func IsObjectsInBucket(objectStoreProvider, cloudCredentialsFile, bslBucket, bsl func DeleteObjectsInBucket(objectStoreProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix string) error { bslPrefix = getFullPrefix(bslPrefix, subPrefix) fmt.Printf("|| VERIFICATION || - Delete backup %s in storage %s\n", backupName, bslPrefix) + if cloudCredentialsFile == "" { return errors.New(fmt.Sprintf("|| ERROR || - Please provide credential file of cloud %s \n", objectStoreProvider)) } @@ -126,9 +127,15 @@ func DeleteObjectsInBucket(objectStoreProvider, cloudCredentialsFile, bslBucket, func SnapshotsShouldNotExistInCloud(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName string, snapshotCheckPoint SnapshotCheckPoint) error { fmt.Printf("|| VERIFICATION || - Snapshots should not exist in cloud, backup %s\n", backupName) + + if cloudProvider == VanillaZFS { + fmt.Printf("Skip snapshot check for cloud provider %s", cloudProvider) + return nil + } if cloudCredentialsFile == "" { return errors.New(fmt.Sprintf("|| ERROR || - Please provide credential file of cloud %s \n", cloudProvider)) } + snapshotCheckPoint.ExpectCount = 0 err := IsSnapshotExisted(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName, snapshotCheckPoint) if err != nil { @@ -140,24 +147,29 @@ func SnapshotsShouldNotExistInCloud(cloudProvider, cloudCredentialsFile, bslBuck func SnapshotsShouldBeCreatedInCloud(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName string, snapshotCheckPoint SnapshotCheckPoint) error { fmt.Printf("|| VERIFICATION || - Snapshots should exist in cloud, backup %s\n", backupName) + + if cloudProvider == VanillaZFS { + fmt.Printf("Skip snapshot check for cloud provider %s", cloudProvider) + return nil + } if cloudCredentialsFile == "" { return errors.New(fmt.Sprintf("|| ERROR || - Please provide credential file of cloud %s \n", cloudProvider)) } + err := IsSnapshotExisted(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName, snapshotCheckPoint) if err != nil { - return errors.Wrapf(err, fmt.Sprintf("|| UNEXPECTED ||Snapshots %s do not exist in cloud after backup as expected", backupName)) + return errors.Wrapf(err, fmt.Sprintf("|| UNEXPECTED || - Snapshots %s do not exist in cloud after backup as expected", backupName)) } - fmt.Printf("|| EXPECTED || - Snapshots exist in cloud, backup %s\n", backupName) + fmt.Printf("|| EXPECTED || - Snapshots of backup %s exist in cloud %s\n", backupName, cloudProvider) return nil } func IsSnapshotExisted(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName string, snapshotCheck SnapshotCheckPoint) error { - s, err := getProvider(cloudProvider) if err != nil { return errors.Wrapf(err, fmt.Sprintf("Cloud provider %s is not valid", cloudProvider)) } - if cloudProvider == "vsphere" { + if cloudProvider == Vsphere { var retSnapshotIDs []string ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*2) defer ctxCancel() diff --git a/test/util/velero/install.go b/test/util/velero/install.go index 0efcf38491..ccbb6420c2 100644 --- a/test/util/velero/install.go +++ b/test/util/velero/install.go @@ -27,6 +27,7 @@ import ( "time" "github.com/pkg/errors" + "golang.org/x/exp/slices" apps "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -76,8 +77,8 @@ func VeleroInstall(ctx context.Context, veleroCfg *VeleroConfig, isStandbyCluste veleroCfg.CloudProvider = veleroCfg.StandbyClusterCloudProvider } - if veleroCfg.CloudProvider != "kind" { - fmt.Println("For cloud platforms, object store plugin provider will be set as cloud provider") + if slices.Contains(PublicCloudProviders, veleroCfg.CloudProvider) { + fmt.Println("For public cloud platforms, object store plugin provider will be set as cloud provider") // If ObjectStoreProvider is not provided, then using the value same as CloudProvider if veleroCfg.ObjectStoreProvider == "" { veleroCfg.ObjectStoreProvider = veleroCfg.CloudProvider @@ -98,12 +99,12 @@ func VeleroInstall(ctx context.Context, veleroCfg *VeleroConfig, isStandbyCluste } // TODO - handle this better - if veleroCfg.CloudProvider == "vsphere" { + if veleroCfg.CloudProvider == Vsphere { // We overrider the ObjectStoreProvider here for vSphere because we want to use the aws plugin for the // backup, but needed to pick up the provider plugins earlier. vSphere plugin no longer needs a Volume // Snapshot location specified if veleroCfg.ObjectStoreProvider == "" { - veleroCfg.ObjectStoreProvider = "aws" + veleroCfg.ObjectStoreProvider = Aws } if err := configvSpherePlugin(veleroCfg); err != nil { return errors.WithMessagef(err, "Failed to config vsphere plugin") @@ -117,7 +118,7 @@ func VeleroInstall(ctx context.Context, veleroCfg *VeleroConfig, isStandbyCluste // For AWS IRSA credential test, AWS IAM service account is required, so if ServiceAccountName and EKSPolicyARN // are both provided, we assume IRSA test is running, otherwise skip this IAM service account creation part. - if veleroCfg.CloudProvider == "aws" && veleroInstallOptions.ServiceAccountName != "" { + if veleroCfg.CloudProvider == Aws && veleroInstallOptions.ServiceAccountName != "" { if veleroCfg.EKSPolicyARN == "" { return errors.New("Please provide EKSPolicyARN for IRSA test.") } @@ -280,14 +281,16 @@ func installVeleroServer(ctx context.Context, cli, cloudProvider string, options if len(options.Features) > 0 { args = append(args, "--features", options.Features) - if !strings.EqualFold(cloudProvider, "vsphere") && strings.EqualFold(options.Features, FeatureCSI) && options.UseVolumeSnapshots { - fmt.Println("Start to install Azure VolumeSnapshotClass ...") + if !strings.EqualFold(cloudProvider, Vsphere) && strings.EqualFold(options.Features, FeatureCSI) && options.UseVolumeSnapshots { + // https://github.com/openebs/zfs-localpv/blob/develop/docs/snapshot.md + fmt.Printf("Start to install %s VolumeSnapshotClass ... \n", cloudProvider) if err := KubectlApplyByFile(ctx, fmt.Sprintf("../testdata/volume-snapshot-class/%s.yaml", cloudProvider)); err != nil { fmt.Println("Fail to install VolumeSnapshotClass when CSI feature is enabled: ", err) return err } } } + if options.GarbageCollectionFrequency > 0 { args = append(args, fmt.Sprintf("--garbage-collection-frequency=%v", options.GarbageCollectionFrequency)) } diff --git a/test/util/velero/velero_utils.go b/test/util/velero/velero_utils.go index 0fc850b417..81c734f2ba 100644 --- a/test/util/velero/velero_utils.go +++ b/test/util/velero/velero_utils.go @@ -107,7 +107,7 @@ var pluginsMatrix = map[string]map[string][]string{ "main": { "aws": {"velero/velero-plugin-for-aws:main"}, "azure": {"velero/velero-plugin-for-microsoft-azure:main"}, - "vsphere": {"vsphereveleroplugin/velero-plugin-for-vsphere:v1.5.1"}, + "vsphere": {"vsphereveleroplugin/velero-plugin-for-vsphere:v1.5.2"}, "gcp": {"velero/velero-plugin-for-gcp:main"}, "csi": {"velero/velero-plugin-for-csi:main"}, "datamover": {"velero/velero-plugin-for-aws:main"}, @@ -128,12 +128,19 @@ func getPluginsByVersion(version, cloudProvider, objectStoreProvider, feature st } var pluginsForFeature []string - if cloudProvider == "kind" { - plugins, ok := cloudMap["aws"] + if slices.Contains(LocalCloudProviders, cloudProvider) { + var pluginsCSI []string + plugins, ok := cloudMap[Aws] if !ok { return nil, errors.Errorf("fail to get plugins by version: %s and provider %s", version, cloudProvider) } - return plugins, nil + if cloudProvider == VanillaZFS { + pluginsCSI, ok = cloudMap["csi"] + if !ok { + return nil, errors.Errorf("fail to get plugins by version: %s and provider %s", version, cloudProvider) + } + } + return append(plugins, pluginsCSI...), nil } plugins, ok := cloudMap[cloudProvider] @@ -1244,7 +1251,7 @@ func GetSnapshotCheckPoint(client TestClient, veleroCfg VeleroConfig, expectCoun snapshotCheckPoint.ExpectCount = expectCount snapshotCheckPoint.NamespaceBackedUp = namespaceBackedUp snapshotCheckPoint.PodName = KibishiiPVCNameList - if (veleroCfg.CloudProvider == "azure" || veleroCfg.CloudProvider == "aws") && strings.EqualFold(veleroCfg.Features, FeatureCSI) { + if (veleroCfg.CloudProvider == Azure || veleroCfg.CloudProvider == Aws) && strings.EqualFold(veleroCfg.Features, FeatureCSI) { snapshotCheckPoint.EnableCSI = true resourceName := "snapshot.storage.k8s.io"