From 8a9c004de15667008da220bdd84d542d7e82fe4b Mon Sep 17 00:00:00 2001 From: Xun Jiang Date: Wed, 13 Apr 2022 00:08:23 +0800 Subject: [PATCH] Add --insecure-tls for Restic backup and restore command. 1. Add --insecure-tls in PodVolumeBackup and PodVolumeRestore controller. 2. Change --last flag in Restic command to --latest=1 due to Restic version update. Signed-off-by: Xun Jiang --- pkg/controller/pod_volume_backup_controller.go | 17 +++++++++++++++++ pkg/controller/pod_volume_restore_controller.go | 10 +++++++++- pkg/restic/command_factory.go | 2 +- pkg/restic/command_factory_test.go | 9 ++++++--- pkg/restic/exec_commands.go | 13 +++++++++---- pkg/restic/repository_manager.go | 8 ++++---- pkg/restic/repository_manager_test.go | 2 +- 7 files changed, 47 insertions(+), 14 deletions(-) diff --git a/pkg/controller/pod_volume_backup_controller.go b/pkg/controller/pod_volume_backup_controller.go index 7c92ca187a..ee7a629984 100644 --- a/pkg/controller/pod_volume_backup_controller.go +++ b/pkg/controller/pod_volume_backup_controller.go @@ -281,6 +281,14 @@ func (c *podVolumeBackupController) processBackup(req *velerov1api.PodVolumeBack } } + // #4820: restrieve insecureSkipTLSVerify from BSL configuration for + // AWS plugin. If nothing is return, that means insecureSkipTLSVerify + // is not enable for Restic command. + skipTLSRet := restic.GetInsecureSkipTLSVerifyFromBSL(backupLocation, log) + if len(skipTLSRet) > 0 { + resticCmd.ExtraFlags = append(resticCmd.ExtraFlags, skipTLSRet) + } + var stdout, stderr string var emptySnapshot bool @@ -300,6 +308,15 @@ func (c *podVolumeBackupController) processBackup(req *velerov1api.PodVolumeBack cmd.Env = env cmd.CACertFile = caCertFile + // TODO: + // #4820: restrieve insecureSkipTLSVerify from BSL configuration for + // AWS plugin. If nothing is return, that means insecureSkipTLSVerify + // is not enable for Restic command. + skipTLSRet := restic.GetInsecureSkipTLSVerifyFromBSL(backupLocation, log) + if len(skipTLSRet) > 0 { + cmd.ExtraFlags = append(cmd.ExtraFlags, skipTLSRet) + } + snapshotID, err = restic.GetSnapshotID(cmd) if err != nil { log.WithError(err).Error("Error getting SnapshotID") diff --git a/pkg/controller/pod_volume_restore_controller.go b/pkg/controller/pod_volume_restore_controller.go index e04de5c655..2837dc169a 100644 --- a/pkg/controller/pod_volume_restore_controller.go +++ b/pkg/controller/pod_volume_restore_controller.go @@ -372,9 +372,17 @@ func (c *podVolumeRestoreController) restorePodVolume(req *velerov1api.PodVolume } resticCmd.Env = env + // #4820: restrieve insecureSkipTLSVerify from BSL configuration for + // AWS plugin. If nothing is return, that means insecureSkipTLSVerify + // is not enable for Restic command. + skipTLSRet := restic.GetInsecureSkipTLSVerifyFromBSL(backupLocation, log) + if len(skipTLSRet) > 0 { + resticCmd.ExtraFlags = append(resticCmd.ExtraFlags, skipTLSRet) + } + var stdout, stderr string - if stdout, stderr, err = restic.RunRestore(resticCmd, log, c.updateRestoreProgressFunc(req, log)); err != nil { + if stdout, stderr, err = restic.RunRestore(resticCmd, log, c.updateRestoreProgressFunc(req, log), skipTLSRet); err != nil { return errors.Wrapf(err, "error running restic restore, cmd=%s, stdout=%s, stderr=%s", resticCmd.String(), stdout, stderr) } log.Debugf("Ran command=%s, stdout=%s, stderr=%s", resticCmd.String(), stdout, stderr) diff --git a/pkg/restic/command_factory.go b/pkg/restic/command_factory.go index 16625cd3f0..9e21742104 100644 --- a/pkg/restic/command_factory.go +++ b/pkg/restic/command_factory.go @@ -64,7 +64,7 @@ func GetSnapshotCommand(repoIdentifier, passwordFile string, tags map[string]str Command: "snapshots", RepoIdentifier: repoIdentifier, PasswordFile: passwordFile, - ExtraFlags: []string{"--json", "--last", getSnapshotTagFlag(tags)}, + ExtraFlags: []string{"--json", "--latest=1", getSnapshotTagFlag(tags)}, } } diff --git a/pkg/restic/command_factory_test.go b/pkg/restic/command_factory_test.go index e8145a28f6..4a38bc1901 100644 --- a/pkg/restic/command_factory_test.go +++ b/pkg/restic/command_factory_test.go @@ -58,7 +58,7 @@ func TestGetSnapshotCommand(t *testing.T) { assert.Equal(t, "password-file", c.PasswordFile) // set up expected flag names - expectedFlags := []string{"--json", "--last", "--tag"} + expectedFlags := []string{"--json", "--latest=1", "--tag"} // for tracking actual flag names actualFlags := []string{} // for tracking actual --tag values as a map @@ -68,10 +68,11 @@ func TestGetSnapshotCommand(t *testing.T) { for _, flag := range c.ExtraFlags { // split into 2 parts from the first = sign (if any) parts := strings.SplitN(flag, "=", 2) - // parts[0] is the flag name - actualFlags = append(actualFlags, parts[0]) + // convert --tag data to a map if parts[0] == "--tag" { + actualFlags = append(actualFlags, parts[0]) + // split based on , tags := strings.Split(parts[1], ",") // loop through each key-value tag pair @@ -81,6 +82,8 @@ func TestGetSnapshotCommand(t *testing.T) { // record actual key & value actualTags[kvs[0]] = kvs[1] } + } else { + actualFlags = append(actualFlags, flag) } } diff --git a/pkg/restic/exec_commands.go b/pkg/restic/exec_commands.go index 63e3e13616..8cc05a9fa3 100644 --- a/pkg/restic/exec_commands.go +++ b/pkg/restic/exec_commands.go @@ -183,8 +183,9 @@ func getSummaryLine(b []byte) ([]byte, error) { // RunRestore runs a `restic restore` command and monitors the volume size to // provide progress updates to the caller. -func RunRestore(restoreCmd *Command, log logrus.FieldLogger, updateFunc func(velerov1api.PodVolumeOperationProgress)) (string, string, error) { - snapshotSize, err := getSnapshotSize(restoreCmd.RepoIdentifier, restoreCmd.PasswordFile, restoreCmd.CACertFile, restoreCmd.Args[0], restoreCmd.Env) +func RunRestore(restoreCmd *Command, log logrus.FieldLogger, updateFunc func(velerov1api.PodVolumeOperationProgress), insecureTLS string) (string, string, error) { + + snapshotSize, err := getSnapshotSize(restoreCmd.RepoIdentifier, restoreCmd.PasswordFile, restoreCmd.CACertFile, restoreCmd.Args[0], restoreCmd.Env, insecureTLS) if err != nil { return "", "", errors.Wrap(err, "error getting snapshot size") } @@ -230,11 +231,15 @@ func RunRestore(restoreCmd *Command, log logrus.FieldLogger, updateFunc func(vel return stdout, stderr, err } -func getSnapshotSize(repoIdentifier, passwordFile, caCertFile, snapshotID string, env []string) (int64, error) { +func getSnapshotSize(repoIdentifier, passwordFile, caCertFile, snapshotID string, env []string, insecureTLS string) (int64, error) { cmd := StatsCommand(repoIdentifier, passwordFile, snapshotID) cmd.Env = env cmd.CACertFile = caCertFile + if len(insecureTLS) > 0 { + cmd.ExtraFlags = append(cmd.ExtraFlags, insecureTLS) + } + stdout, stderr, err := exec.RunCommand(cmd.Cmd()) if err != nil { return 0, errors.Wrapf(err, "error running command, stderr=%s", stderr) @@ -245,7 +250,7 @@ func getSnapshotSize(repoIdentifier, passwordFile, caCertFile, snapshotID string } if err := json.Unmarshal([]byte(stdout), &snapshotStats); err != nil { - return 0, errors.Wrap(err, "error unmarshalling restic stats result") + return 0, errors.Wrapf(err, "error unmarshalling restic stats result, stdout=%s", stdout) } return snapshotStats.TotalSize, nil diff --git a/pkg/restic/repository_manager.go b/pkg/restic/repository_manager.go index 634e1a6373..613898725b 100644 --- a/pkg/restic/repository_manager.go +++ b/pkg/restic/repository_manager.go @@ -191,10 +191,10 @@ func (rm *repositoryManager) ConnectToRepo(repo *velerov1api.ResticRepository) e defer rm.repoLocker.Unlock(repo.Name) snapshotsCmd := SnapshotsCommand(repo.Spec.ResticIdentifier) - // use the '--last' flag to minimize the amount of data fetched since + // use the '--latest=1' flag to minimize the amount of data fetched since // we're just validating that the repo exists and can be authenticated // to. - snapshotsCmd.ExtraFlags = append(snapshotsCmd.ExtraFlags, "--last") + snapshotsCmd.ExtraFlags = append(snapshotsCmd.ExtraFlags, "--latest=1") return rm.exec(snapshotsCmd, repo.Spec.BackupStorageLocation) } @@ -275,7 +275,7 @@ func (rm *repositoryManager) exec(cmd *Command, backupLocation string) error { // #4820: restrieve insecureSkipTLSVerify from BSL configuration for // AWS plugin. If nothing is return, that means insecureSkipTLSVerify // is not enable for Restic command. - skipTLSRet := getInsecureSkipTLSVerifyFromBSL(loc, rm.log) + skipTLSRet := GetInsecureSkipTLSVerifyFromBSL(loc, rm.log) if len(skipTLSRet) > 0 { cmd.ExtraFlags = append(cmd.ExtraFlags, skipTLSRet) } @@ -296,7 +296,7 @@ func (rm *repositoryManager) exec(cmd *Command, backupLocation string) error { // getInsecureSkipTLSVerifyFromBSL get insecureSkipTLSVerify flag from BSL configuration, // Then return --insecure-tls flag with boolean value as result. -func getInsecureSkipTLSVerifyFromBSL(backupLocation *velerov1api.BackupStorageLocation, logger logrus.FieldLogger) string { +func GetInsecureSkipTLSVerifyFromBSL(backupLocation *velerov1api.BackupStorageLocation, logger logrus.FieldLogger) string { result := "" if backupLocation == nil { diff --git a/pkg/restic/repository_manager_test.go b/pkg/restic/repository_manager_test.go index 15c4454ce8..97365cb2b0 100644 --- a/pkg/restic/repository_manager_test.go +++ b/pkg/restic/repository_manager_test.go @@ -113,7 +113,7 @@ func TestGetInsecureSkipTLSVerifyFromBSL(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - res := getInsecureSkipTLSVerifyFromBSL(test.backupLocation, test.logger) + res := GetInsecureSkipTLSVerifyFromBSL(test.backupLocation, test.logger) assert.Equal(t, test.expected, res) })