Update integrated Restic version and add insecureSkipTLSVerify for Re…

Makefile

@@ -82,7 +82,7 @@ see: https://velero.io/docs/main/build-from-source/#making-images-and-updating-v
 endef
 
 # The version of restic binary to be downloaded
-RESTIC_VERSION ?= 0.12.1
+RESTIC_VERSION ?= 0.13.1
 
 CLI_PLATFORMS ?= linux-amd64 linux-arm linux-arm64 darwin-amd64 darwin-arm64 windows-amd64 linux-ppc64le
 BUILDX_PLATFORMS ?= $(subst -,/,$(ARCH))

changelogs/unreleased/4821-jxun

@@ -0,0 +1 @@
+Update integrated Restic version and add insecureSkipTLSVerify for Restic CLI.

pkg/controller/pod_volume_backup_controller.go

@@ -281,6 +281,14 @@ func (c *podVolumeBackupController) processBackup(req *velerov1api.PodVolumeBack
 		}
 	}
 
+	// #4820: restrieve insecureSkipTLSVerify from BSL configuration for
+	// AWS plugin. If nothing is return, that means insecureSkipTLSVerify
+	// is not enable for Restic command.
+	skipTLSRet := restic.GetInsecureSkipTLSVerifyFromBSL(backupLocation, log)
+	if len(skipTLSRet) > 0 {
+		resticCmd.ExtraFlags = append(resticCmd.ExtraFlags, skipTLSRet)
+	}
+
 	var stdout, stderr string
 
 	var emptySnapshot bool
@@ -300,6 +308,11 @@ func (c *podVolumeBackupController) processBackup(req *velerov1api.PodVolumeBack
 		cmd.Env = env
 		cmd.CACertFile = caCertFile
 
+		// #4820: also apply the insecureTLS flag to Restic snapshots command
+		if len(skipTLSRet) > 0 {
+			cmd.ExtraFlags = append(cmd.ExtraFlags, skipTLSRet)
+		}
+
 		snapshotID, err = restic.GetSnapshotID(cmd)
 		if err != nil {
 			log.WithError(err).Error("Error getting SnapshotID")

pkg/controller/pod_volume_restore_controller.go

@@ -372,9 +372,17 @@ func (c *podVolumeRestoreController) restorePodVolume(req *velerov1api.PodVolume
 	}
 	resticCmd.Env = env
 
+	// #4820: restrieve insecureSkipTLSVerify from BSL configuration for
+	// AWS plugin. If nothing is return, that means insecureSkipTLSVerify
+	// is not enable for Restic command.
+	skipTLSRet := restic.GetInsecureSkipTLSVerifyFromBSL(backupLocation, log)
+	if len(skipTLSRet) > 0 {
+		resticCmd.ExtraFlags = append(resticCmd.ExtraFlags, skipTLSRet)
+	}
+
 	var stdout, stderr string
 
-	if stdout, stderr, err = restic.RunRestore(resticCmd, log, c.updateRestoreProgressFunc(req, log)); err != nil {
+	if stdout, stderr, err = restic.RunRestore(resticCmd, log, c.updateRestoreProgressFunc(req, log), skipTLSRet); err != nil {
 		return errors.Wrapf(err, "error running restic restore, cmd=%s, stdout=%s, stderr=%s", resticCmd.String(), stdout, stderr)
 	}
 	log.Debugf("Ran command=%s, stdout=%s, stderr=%s", resticCmd.String(), stdout, stderr)

pkg/restic/command_factory.go

@@ -64,7 +64,7 @@ func GetSnapshotCommand(repoIdentifier, passwordFile string, tags map[string]str
 		Command:        "snapshots",
 		RepoIdentifier: repoIdentifier,
 		PasswordFile:   passwordFile,
-		ExtraFlags:     []string{"--json", "--last", getSnapshotTagFlag(tags)},
+		ExtraFlags:     []string{"--json", "--latest=1", getSnapshotTagFlag(tags)},
 	}
 }
 

pkg/restic/command_factory_test.go

@@ -58,7 +58,7 @@ func TestGetSnapshotCommand(t *testing.T) {
 	assert.Equal(t, "password-file", c.PasswordFile)
 
 	// set up expected flag names
-	expectedFlags := []string{"--json", "--last", "--tag"}
+	expectedFlags := []string{"--json", "--latest=1", "--tag"}
 	// for tracking actual flag names
 	actualFlags := []string{}
 	// for tracking actual --tag values as a map
@@ -68,10 +68,11 @@ func TestGetSnapshotCommand(t *testing.T) {
 	for _, flag := range c.ExtraFlags {
 		// split into 2 parts from the first = sign (if any)
 		parts := strings.SplitN(flag, "=", 2)
-		// parts[0] is the flag name
-		actualFlags = append(actualFlags, parts[0])
+
 		// convert --tag data to a map
 		if parts[0] == "--tag" {
+			actualFlags = append(actualFlags, parts[0])
+
 			// split based on ,
 			tags := strings.Split(parts[1], ",")
 			// loop through each key-value tag pair
@@ -81,6 +82,8 @@ func TestGetSnapshotCommand(t *testing.T) {
 				// record actual key & value
 				actualTags[kvs[0]] = kvs[1]
 			}
+		} else {
+			actualFlags = append(actualFlags, flag)
 		}
 	}
 

pkg/restic/exec_commands.go

@@ -183,8 +183,8 @@ func getSummaryLine(b []byte) ([]byte, error) {
 
 // RunRestore runs a `restic restore` command and monitors the volume size to
 // provide progress updates to the caller.
-func RunRestore(restoreCmd *Command, log logrus.FieldLogger, updateFunc func(velerov1api.PodVolumeOperationProgress)) (string, string, error) {
-	snapshotSize, err := getSnapshotSize(restoreCmd.RepoIdentifier, restoreCmd.PasswordFile, restoreCmd.CACertFile, restoreCmd.Args[0], restoreCmd.Env)
+func RunRestore(restoreCmd *Command, log logrus.FieldLogger, updateFunc func(velerov1api.PodVolumeOperationProgress), insecureTLS string) (string, string, error) {
+	snapshotSize, err := getSnapshotSize(restoreCmd.RepoIdentifier, restoreCmd.PasswordFile, restoreCmd.CACertFile, restoreCmd.Args[0], restoreCmd.Env, insecureTLS)
 	if err != nil {
 		return "", "", errors.Wrap(err, "error getting snapshot size")
 	}
@@ -230,11 +230,15 @@ func RunRestore(restoreCmd *Command, log logrus.FieldLogger, updateFunc func(vel
 	return stdout, stderr, err
 }
 
-func getSnapshotSize(repoIdentifier, passwordFile, caCertFile, snapshotID string, env []string) (int64, error) {
+func getSnapshotSize(repoIdentifier, passwordFile, caCertFile, snapshotID string, env []string, insecureTLS string) (int64, error) {
 	cmd := StatsCommand(repoIdentifier, passwordFile, snapshotID)
 	cmd.Env = env
 	cmd.CACertFile = caCertFile
 
+	if len(insecureTLS) > 0 {
+		cmd.ExtraFlags = append(cmd.ExtraFlags, insecureTLS)
+	}
+
 	stdout, stderr, err := exec.RunCommand(cmd.Cmd())
 	if err != nil {
 		return 0, errors.Wrapf(err, "error running command, stderr=%s", stderr)
@@ -245,7 +249,7 @@ func getSnapshotSize(repoIdentifier, passwordFile, caCertFile, snapshotID string
 	}
 
 	if err := json.Unmarshal([]byte(stdout), &snapshotStats); err != nil {
-		return 0, errors.Wrap(err, "error unmarshalling restic stats result")
+		return 0, errors.Wrapf(err, "error unmarshalling restic stats result, stdout=%s", stdout)
 	}
 
 	return snapshotStats.TotalSize, nil

pkg/restic/repository_manager.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"strconv"
 
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
@@ -95,6 +96,12 @@ type repositoryManager struct {
 	credentialsFileStore credentials.FileStore
 }
 
+const (
+	// insecureSkipTLSVerifyKey is the flag in BackupStorageLocation's config
+	// to indicate whether to skip TLS verify to setup insecure HTTPS connection.
+	insecureSkipTLSVerifyKey = "insecureSkipTLSVerify"
+)
+
 // NewRepositoryManager constructs a RepositoryManager.
 func NewRepositoryManager(
 	ctx context.Context,
@@ -184,10 +191,10 @@ func (rm *repositoryManager) ConnectToRepo(repo *velerov1api.ResticRepository) e
 	defer rm.repoLocker.Unlock(repo.Name)
 
 	snapshotsCmd := SnapshotsCommand(repo.Spec.ResticIdentifier)
-	// use the '--last' flag to minimize the amount of data fetched since
+	// use the '--latest=1' flag to minimize the amount of data fetched since
 	// we're just validating that the repo exists and can be authenticated
 	// to.
-	snapshotsCmd.ExtraFlags = append(snapshotsCmd.ExtraFlags, "--last")
+	snapshotsCmd.ExtraFlags = append(snapshotsCmd.ExtraFlags, "--latest=1")
 
 	return rm.exec(snapshotsCmd, repo.Spec.BackupStorageLocation)
 }
@@ -265,6 +272,14 @@ func (rm *repositoryManager) exec(cmd *Command, backupLocation string) error {
 	}
 	cmd.Env = env
 
+	// #4820: restrieve insecureSkipTLSVerify from BSL configuration for
+	// AWS plugin. If nothing is return, that means insecureSkipTLSVerify
+	// is not enable for Restic command.
+	skipTLSRet := GetInsecureSkipTLSVerifyFromBSL(loc, rm.log)
+	if len(skipTLSRet) > 0 {
+		cmd.ExtraFlags = append(cmd.ExtraFlags, skipTLSRet)
+	}
+
 	stdout, stderr, err := veleroexec.RunCommand(cmd.Cmd())
 	rm.log.WithFields(logrus.Fields{
 		"repository": cmd.RepoName(),
@@ -278,3 +293,33 @@ func (rm *repositoryManager) exec(cmd *Command, backupLocation string) error {
 
 	return nil
 }
+
+// getInsecureSkipTLSVerifyFromBSL get insecureSkipTLSVerify flag from BSL configuration,
+// Then return --insecure-tls flag with boolean value as result.
+func GetInsecureSkipTLSVerifyFromBSL(backupLocation *velerov1api.BackupStorageLocation, logger logrus.FieldLogger) string {
+	result := ""
+
+	if backupLocation == nil {
+		logger.Info("bsl is nil. return empty.")
+		return result
+	}
+
+	backendType := getBackendType(backupLocation.Spec.Provider)
+
+	// Only check insecureSkipTLSVerifyKey for AWS compatible backend.
+	// Due to this is only possible for on-premise environment. On-premise
+	// environment use velero AWS plugin as object store plugin.
+	if backendType == AWSBackend {
+		if strRet, ok := backupLocation.Spec.Config[insecureSkipTLSVerifyKey]; ok {
+			_, err := strconv.ParseBool(strRet)
+			if err == nil {
+				result = "--insecure-tls" + "=" + strRet
+				return result
+			} else {
+				logger.Infof("Fail to convert string to bool for insecureSkipTLSVerifyKey flag: %s.", err.Error())
+			}
+		}
+	}
+
+	return result
+}

pkg/restic/repository_manager_test.go

@@ -0,0 +1,121 @@
+/*
+Copyright the Velero contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package restic
+
+import (
+	"testing"
+
+	"github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/assert"
+
+	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
+)
+
+func TestGetInsecureSkipTLSVerifyFromBSL(t *testing.T) {
+	log := logrus.StandardLogger()
+	tests := []struct {
+		name           string
+		backupLocation *velerov1api.BackupStorageLocation
+		logger         logrus.FieldLogger
+		expected       string
+	}{
+		{
+			"Test with nil BSL. Should return empty string.",
+			nil,
+			log,
+			"",
+		},
+		{
+			"Test with none AWS BSL. Should return empty string.",
+			&velerov1api.BackupStorageLocation{
+				Spec: velerov1api.BackupStorageLocationSpec{
+					Provider: "azure",
+				},
+			},
+			log,
+			"",
+		},
+		{
+			"Test with AWS BSL's insecureSkipTLSVerify set to false.",
+			&velerov1api.BackupStorageLocation{
+				Spec: velerov1api.BackupStorageLocationSpec{
+					Provider: "aws",
+					Config: map[string]string{
+						"insecureSkipTLSVerify": "false",
+					},
+				},
+			},
+			log,
+			"--insecure-tls=false",
+		},
+		{
+			"Test with AWS BSL's insecureSkipTLSVerify set to true.",
+			&velerov1api.BackupStorageLocation{
+				Spec: velerov1api.BackupStorageLocationSpec{
+					Provider: "aws",
+					Config: map[string]string{
+						"insecureSkipTLSVerify": "true",
+					},
+				},
+			},
+			log,
+			"--insecure-tls=true",
+		},
+		{
+			"Test with AWS BSL's insecureSkipTLSVerify set to invalid.",
+			&velerov1api.BackupStorageLocation{
+				Spec: velerov1api.BackupStorageLocationSpec{
+					Provider: "aws",
+					Config: map[string]string{
+						"insecureSkipTLSVerify": "invalid",
+					},
+				},
+			},
+			log,
+			"",
+		},
+		{
+			"Test with AWS without insecureSkipTLSVerify.",
+			&velerov1api.BackupStorageLocation{
+				Spec: velerov1api.BackupStorageLocationSpec{
+					Provider: "aws",
+					Config:   map[string]string{},
+				},
+			},
+			log,
+			"",
+		},
+		{
+			"Test with AWS without config.",
+			&velerov1api.BackupStorageLocation{
+				Spec: velerov1api.BackupStorageLocationSpec{
+					Provider: "aws",
+				},
+			},
+			log,
+			"",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			res := GetInsecureSkipTLSVerifyFromBSL(test.backupLocation, test.logger)
+
+			assert.Equal(t, test.expected, res)
+		})
+	}
+}