Store PodVolumeBackups in obj storage & use as source of truth

changelogs/unreleased/1577-carlisia

@@ -0,0 +1 @@
+Store restic PodVolumeBackups in obj storage & use that as source of truth like regular backups.

pkg/backup/builder.go → pkg/backup/backup_builder.go

@@ -31,12 +31,12 @@ type Builder struct {
 
 // NewBuilder returns a Builder for a Backup with no namespace/name.
 func NewBuilder() *Builder {
-	return NewNamedBuilder("", "")
+	return NewNamedBackupBuilder("", "")
 }
 
-// NewNamedBuilder returns a Builder for a Backup with the specified namespace
+// NewNamedBackupBuilder returns a Builder for a Backup with the specified namespace
 // and name.
-func NewNamedBuilder(namespace, name string) *Builder {
+func NewNamedBackupBuilder(namespace, name string) *Builder {
 	return &Builder{
 		backup: velerov1api.Backup{
 			TypeMeta: metav1.TypeMeta{

pkg/backup/backup_new_test.go

@@ -2101,7 +2101,7 @@ func newSnapshotLocation(ns, name, provider string) *velerov1.VolumeSnapshotLoca
 }
 
 func defaultBackup() *Builder {
-	return NewNamedBuilder(velerov1.DefaultNamespace, "backup-1")
+	return NewNamedBackupBuilder(velerov1.DefaultNamespace, "backup-1")
 }
 
 func toUnstructuredOrFail(t *testing.T, obj interface{}) map[string]interface{} {

pkg/backup/item_backupper.go

@@ -33,6 +33,7 @@ import (
 	kubeerrs "k8s.io/apimachinery/pkg/util/errors"
 
 	api "github.com/heptio/velero/pkg/apis/velero/v1"
+	velerov1api "github.com/heptio/velero/pkg/apis/velero/v1"
 	"github.com/heptio/velero/pkg/client"
 	"github.com/heptio/velero/pkg/discovery"
 	"github.com/heptio/velero/pkg/kuberesource"
@@ -217,15 +218,10 @@ func (ib *defaultItemBackupper) backupItem(logger logrus.FieldLogger, obj runtim
 	}
 
 	if groupResource == kuberesource.Pods && pod != nil {
-		// this function will return partial results, so process volumeSnapshots
+		// this function will return partial results, so process podVolumeBackups
 		// even if there are errors.
-		volumeSnapshots, errs := ib.backupPodVolumes(log, pod, resticVolumesToBackup)
-
-		// annotate the pod with the successful volume snapshots
-		for volume, snapshot := range volumeSnapshots {
-			restic.SetPodSnapshotAnnotation(metadata, volume, snapshot)
-		}
-
+		podVolumeBackups, errs := ib.backupPodVolumes(log, pod, resticVolumesToBackup)
+		ib.backupRequest.PodVolumeBackups = podVolumeBackups
 		backupErrs = append(backupErrs, errs...)
 	}
 
@@ -269,9 +265,9 @@ func (ib *defaultItemBackupper) backupItem(logger logrus.FieldLogger, obj runtim
 	return nil
 }
 
-// backupPodVolumes triggers restic backups of the specified pod volumes, and returns a map of volume name -> snapshot ID
+// backupPodVolumes triggers restic backups of the specified pod volumes, and returns a list of PodVolumeBackups
 // for volumes that were successfully backed up, and a slice of any errors that were encountered.
-func (ib *defaultItemBackupper) backupPodVolumes(log logrus.FieldLogger, pod *corev1api.Pod, volumes []string) (map[string]string, []error) {
+func (ib *defaultItemBackupper) backupPodVolumes(log logrus.FieldLogger, pod *corev1api.Pod, volumes []string) ([]*velerov1api.PodVolumeBackup, []error) {
 	if len(volumes) == 0 {
 		return nil, nil
 	}

pkg/backup/item_backupper_test.go

@@ -28,14 +28,12 @@ import (
 	"github.com/stretchr/testify/require"
 	"k8s.io/apimachinery/pkg/api/meta"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
-	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/util/sets"
 
-	v1 "github.com/heptio/velero/pkg/apis/velero/v1"
+	velerov1api "github.com/heptio/velero/pkg/apis/velero/v1"
 	"github.com/heptio/velero/pkg/plugin/velero"
-	resticmocks "github.com/heptio/velero/pkg/restic/mocks"
 	"github.com/heptio/velero/pkg/util/collections"
 	velerotest "github.com/heptio/velero/pkg/util/test"
 )
@@ -102,10 +100,10 @@ func TestBackupItemNoSkips(t *testing.T) {
 				w             = &fakeTarWriter{}
 			)
 
-			backup.Backup = new(v1.Backup)
+			backup.Backup = new(velerov1api.Backup)
 			backup.NamespaceIncludesExcludes = collections.NewIncludesExcludes()
 			backup.ResourceIncludesExcludes = collections.NewIncludesExcludes()
-			backup.SnapshotLocations = []*v1.VolumeSnapshotLocation{
+			backup.SnapshotLocations = []*velerov1api.VolumeSnapshotLocation{
 				newSnapshotLocation("velero", "default", "default"),
 			}
 
@@ -245,7 +243,7 @@ func TestBackupItemNoSkips(t *testing.T) {
 
 type addAnnotationAction struct{}
 
-func (a *addAnnotationAction) Execute(item runtime.Unstructured, backup *v1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, error) {
+func (a *addAnnotationAction) Execute(item runtime.Unstructured, backup *velerov1api.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, error) {
 	// since item actions run out-of-proc, do a deep-copy here to simulate passing data
 	// across a process boundary.
 	copy := item.(*unstructured.Unstructured).DeepCopy()
@@ -269,74 +267,6 @@ func (a *addAnnotationAction) AppliesTo() (velero.ResourceSelector, error) {
 	panic("not implemented")
 }
 
-func TestResticAnnotationsPersist(t *testing.T) {
-	var (
-		w   = &fakeTarWriter{}
-		obj = &unstructured.Unstructured{
-			Object: map[string]interface{}{
-				"metadata": map[string]interface{}{
-					"namespace": "myns",
-					"name":      "bar",
-					"annotations": map[string]interface{}{
-						"backup.velero.io/backup-volumes": "volume-1,volume-2",
-					},
-				},
-			},
-		}
-		req = &Request{
-			NamespaceIncludesExcludes: collections.NewIncludesExcludes(),
-			ResourceIncludesExcludes:  collections.NewIncludesExcludes(),
-			ResolvedActions: []resolvedAction{
-				{
-					BackupItemAction:          &addAnnotationAction{},
-					namespaceIncludesExcludes: collections.NewIncludesExcludes(),
-					resourceIncludesExcludes:  collections.NewIncludesExcludes(),
-					selector:                  labels.Everything(),
-				},
-			},
-		}
-		resticBackupper = &resticmocks.Backupper{}
-		b               = (&defaultItemBackupperFactory{}).newItemBackupper(
-			req,
-			make(map[itemKey]struct{}),
-			nil,
-			w,
-			&velerotest.FakeDynamicFactory{},
-			velerotest.NewFakeDiscoveryHelper(true, nil),
-			resticBackupper,
-			newPVCSnapshotTracker(),
-			nil,
-		).(*defaultItemBackupper)
-	)
-
-	resticBackupper.
-		On("BackupPodVolumes", mock.Anything, mock.Anything, mock.Anything).
-		Return(map[string]string{"volume-1": "snapshot-1", "volume-2": "snapshot-2"}, nil)
-
-	// our expected backed-up object is the passed-in object, plus the annotation
-	// that the backup item action adds, plus the annotations that the restic
-	// backupper adds
-	expected := obj.DeepCopy()
-	annotations := expected.GetAnnotations()
-	if annotations == nil {
-		annotations = make(map[string]string)
-	}
-	annotations["foo"] = "bar"
-	annotations["snapshot.velero.io/volume-1"] = "snapshot-1"
-	annotations["snapshot.velero.io/volume-2"] = "snapshot-2"
-	expected.SetAnnotations(annotations)
-
-	// method under test
-	require.NoError(t, b.backupItem(velerotest.NewLogger(), obj, schema.ParseGroupResource("pods")))
-
-	// get the actual backed-up item
-	require.Len(t, w.data, 1)
-	actual, err := velerotest.GetAsMap(string(w.data[0]))
-	require.NoError(t, err)
-
-	assert.EqualValues(t, expected.Object, actual)
-}
-
 type fakeTarWriter struct {
 	closeCalled      bool
 	headers          []*tar.Header

pkg/backup/pod_volume_backup_builder.go

@@ -0,0 +1,92 @@
+/*
+Copyright 2019 the Velero contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package backup
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	velerov1api "github.com/heptio/velero/pkg/apis/velero/v1"
+)
+
+// PodVolumeBackupBuilder is a helper for concisely constructing PodVolumeBackup API objects.
+type PodVolumeBackupBuilder struct {
+	podVolumeBackup velerov1api.PodVolumeBackup
+}
+
+// NewPodVolumeBackupBuilder returns a PodVolumeBackupBuilder for a PodVolumeBackup with no namespace/name.
+func NewPodVolumeBackupBuilder() *PodVolumeBackupBuilder {
+	return NewNamedPodVolumeBackupBuilder("", "")
+}
+
+// NewNamedPodVolumeBackupBuilder returns a PodVolumeBackupBuilder for a Backup with the specified namespace
+// and name.
+func NewNamedPodVolumeBackupBuilder(namespace, name string) *PodVolumeBackupBuilder {
+	return &PodVolumeBackupBuilder{
+		podVolumeBackup: velerov1api.PodVolumeBackup{
+			TypeMeta: metav1.TypeMeta{
+				APIVersion: velerov1api.SchemeGroupVersion.String(),
+				Kind:       "PodVolumeBackup",
+			},
+			ObjectMeta: metav1.ObjectMeta{
+				Namespace: namespace,
+				Name:      name,
+			},
+		},
+	}
+}
+
+// PodVolumeBackup returns the built PodVolumeBackup API object.
+func (p *PodVolumeBackupBuilder) PodVolumeBackup() *velerov1api.PodVolumeBackup {
+	return &p.podVolumeBackup
+}
+
+// Namespace sets the PodVolumeBackup's namespace.
+func (p *PodVolumeBackupBuilder) Namespace(namespace string) *PodVolumeBackupBuilder {
+	p.podVolumeBackup.Namespace = namespace
+	return p
+}
+
+// Name sets the PodVolumeBackup's name.
+func (p *PodVolumeBackupBuilder) Name(name string) *PodVolumeBackupBuilder {
+	p.podVolumeBackup.Name = name
+	return p
+}
+
+// Labels sets the PodVolumeBackup's labels.
+func (p *PodVolumeBackupBuilder) Labels(vals ...string) *PodVolumeBackupBuilder {
+	if p.podVolumeBackup.Labels == nil {
+		p.podVolumeBackup.Labels = map[string]string{}
+	}
+
+	// if we don't have an even number of values, e.g. a key and a value
+	// for each pair, add an empty-string value at the end to serve as
+	// the default value for the last key.
+	if len(vals)%2 != 0 {
+		vals = append(vals, "")
+	}
+
+	for i := 0; i < len(vals); i += 2 {
+		p.podVolumeBackup.Labels[vals[i]] = vals[i+1]
+	}
+	return p
+}
+
+// Phase sets the PodVolumeBackup's phase.
+func (p *PodVolumeBackupBuilder) Phase(phase velerov1api.PodVolumeBackupPhase) *PodVolumeBackupBuilder {
+	p.podVolumeBackup.Status.Phase = phase
+	return p
+}

pkg/backup/request.go

@@ -18,5 +18,6 @@ type Request struct {
 	ResourceHooks             []resourceHook
 	ResolvedActions           []resolvedAction
 
-	VolumeSnapshots []*volume.Snapshot
+	VolumeSnapshots  []*volume.Snapshot
+	PodVolumeBackups []*velerov1api.PodVolumeBackup
 }

pkg/cmd/server/server.go

@@ -557,10 +557,12 @@ func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string
 
 	backupSyncControllerRunInfo := func() controllerRunInfo {
 		backupSyncContoller := controller.NewBackupSyncController(
+			s.veleroClient.VeleroV1(),
 			s.veleroClient.VeleroV1(),
 			s.veleroClient.VeleroV1(),
 			s.sharedInformerFactory.Velero().V1().Backups(),
 			s.sharedInformerFactory.Velero().V1().BackupStorageLocations(),
+			s.sharedInformerFactory.Velero().V1().PodVolumeBackups(),
 			s.config.backupSyncPeriod,
 			s.namespace,
 			s.config.defaultBackupLocation,

pkg/controller/backup_controller.go

@@ -576,14 +576,33 @@ func persistBackup(backup *pkgbackup.Request, backupContents, backupLog *os.File
 		errs = append(errs, errors.Wrap(err, "error closing gzip writer"))
 	}
 
+	podVolumeBackups := new(bytes.Buffer)
+	gzw = gzip.NewWriter(podVolumeBackups)
+	defer gzw.Close()
+
+	if err := json.NewEncoder(gzw).Encode(backup.PodVolumeBackups); err != nil {
+		errs = append(errs, errors.Wrap(err, "error encoding pod volume backups"))
+	}
+	if err := gzw.Close(); err != nil {
+		errs = append(errs, errors.Wrap(err, "error closing gzip writer"))
+	}
+
 	if len(errs) > 0 {
 		// Don't upload the JSON files or backup tarball if encoding to json fails.
 		backupJSON = nil
 		backupContents = nil
 		volumeSnapshots = nil
 	}
 
-	if err := backupStore.PutBackup(backup.Name, backupJSON, backupContents, backupLog, volumeSnapshots); err != nil {
+	backupInfo := persistence.BackupInfo{
+		Name:             backup.Name,
+		Metadata:         backupJSON,
+		Contents:         backupContents,
+		Log:              backupLog,
+		PodVolumeBackups: podVolumeBackups,
+		VolumeSnapshots:  volumeSnapshots,
+	}
+	if err := backupStore.PutBackup(backupInfo); err != nil {
 		errs = append(errs, err)
 	}
 

pkg/controller/backup_controller_test.go

@@ -57,7 +57,7 @@ func (b *fakeBackupper) Backup(logger logrus.FieldLogger, backup *pkgbackup.Requ
 }
 
 func defaultBackup() *pkgbackup.Builder {
-	return pkgbackup.NewNamedBuilder(velerov1api.DefaultNamespace, "backup-1")
+	return pkgbackup.NewNamedBackupBuilder(velerov1api.DefaultNamespace, "backup-1")
 }
 
 func TestProcessBackupNonProcessedItems(t *testing.T) {
@@ -556,16 +556,18 @@ func TestProcessBackupCompletions(t *testing.T) {
 
 			pluginManager.On("GetBackupItemActions").Return(nil, nil)
 			pluginManager.On("CleanupClients").Return(nil)
-
 			backupper.On("Backup", mock.Anything, mock.Anything, mock.Anything, []velero.BackupItemAction(nil), pluginManager).Return(nil)
+			backupStore.On("BackupExists", test.backupLocation.Spec.StorageType.ObjectStorage.Bucket, test.backup.Name).Return(test.backupExists, test.existenceCheckError)
 
-			// Ensure we have a CompletionTimestamp when uploading.
+			// Ensure we have a CompletionTimestamp when uploading and that the backup name matches the backup in the object store.
 			// Failures will display the bytes in buf.
-			completionTimestampIsPresent := func(buf *bytes.Buffer) bool {
-				return strings.Contains(buf.String(), `"completionTimestamp": "2006-01-02T22:04:05Z"`)
+			hasNameAndCompletionTimestamp := func(info persistence.BackupInfo) bool {
+				buf := new(bytes.Buffer)
+				buf.ReadFrom(info.Metadata)
+				return info.Name == test.backup.Name &&
+					strings.Contains(buf.String(), `"completionTimestamp": "2006-01-02T22:04:05Z"`)
 			}
-			backupStore.On("BackupExists", test.backupLocation.Spec.StorageType.ObjectStorage.Bucket, test.backup.Name).Return(test.backupExists, test.existenceCheckError)
-			backupStore.On("PutBackup", test.backup.Name, mock.MatchedBy(completionTimestampIsPresent), mock.Anything, mock.Anything, mock.Anything).Return(nil)
+			backupStore.On("PutBackup", mock.MatchedBy(hasNameAndCompletionTimestamp)).Return(nil)
 
 			// add the test's backup to the informer/lister store
 			require.NotNil(t, test.backup)