From e46e89cb6179522803f126294e064fe05ae6b4d2 Mon Sep 17 00:00:00 2001 From: Steve Kriss Date: Wed, 19 Sep 2018 12:51:30 -0600 Subject: [PATCH] have restic share main Ark bucket Signed-off-by: Steve Kriss --- docs/restic.md | 26 +-------- docs/storage-layout-reorg-v0.10.md | 50 ++++++++++++++++- .../aws/05-ark-backupstoragelocation.yaml | 7 +-- .../azure/05-ark-backupstoragelocation.yaml | 6 -- .../gcp/05-ark-backupstoragelocation.yaml | 8 +-- .../ibm/05-ark-backupstoragelocation.yaml | 6 -- pkg/cmd/server/server.go | 36 ++++++------ .../pod_volume_restore_controller.go | 8 +-- pkg/restic/common.go | 8 +-- pkg/restic/config.go | 29 +++++----- pkg/restic/config_test.go | 55 ++++++++++++++----- 11 files changed, 131 insertions(+), 108 deletions(-) diff --git a/docs/restic.md b/docs/restic.md index d631dcb053..e17020a282 100644 --- a/docs/restic.md +++ b/docs/restic.md @@ -23,15 +23,9 @@ cross-volume-type data migrations. Stay tuned as this evolves! ### Prerequisites -- A working install of Ark version 0.9.0 or later. See [Set up Ark][2] +- A working install of Ark version 0.10.0 or later. See [Set up Ark][2] - A local clone of [the latest release tag of the Ark repository][3] -#### Additional steps if upgrading from version 0.9 alpha - -- Manually delete all of the repositories/data from your existing restic bucket -- Delete all Ark backups from your cluster using `ark backup delete` -- Delete all secrets named `ark-restic-credentials` across all namespaces in your cluster - ### Instructions 1. Download an updated Ark client from the [latest release][3], and move it to a location in your PATH. @@ -49,20 +43,6 @@ cross-volume-type data migrations. Stay tuned as this evolves! - GCP: `kubectl apply -f examples/gcp/20-restic-daemonset.yaml` - Minio: `kubectl apply -f examples/minio/30-restic-daemonset.yaml` -1. Create a new bucket for restic to store its data in, and give the `heptio-ark` IAM user access to it, similarly to -the main Ark bucket you've already set up. Note that this must be a different bucket than the main Ark bucket. -We plan to remove this limitation in a future release. - -1. Uncomment `resticLocation` in your Ark config and set the value appropriately, then apply: - - - AWS: `kubectl apply -f examples/aws/00-ark-config.yaml` - - Azure: `kubectl apply -f examples/azure/10-ark-config.yaml` - - GCP: `kubectl apply -f examples/gcp/00-ark-config.yaml` - - Minio: `kubectl apply -f examples/minio/10-ark-config.yaml` - - Note that `resticLocation` may either be just a bucket name, e.g. `my-restic-bucket`, or a bucket name plus a prefix under - which you'd like the restic data to be stored, e.g. `my-restic-bucket/ark-repos`. - You're now ready to use Ark with restic. ## Back up @@ -139,8 +119,6 @@ You're now ready to use Ark with restic. ## Limitations -- You cannot use the main Ark bucket for storing restic backups. We plan to address this issue -in a future release. - `hostPath` volumes are not supported. [Local persistent volumes][4] are supported. - Those of you familiar with [restic][1] may know that it encrypts all of its data. We've decided to use a static, common encryption key for all restic repositories created by Ark. **This means that anyone who has access to your @@ -264,4 +242,4 @@ on to running other init containers/the main containers. [2]: cloud-common.md [3]: https://github.com/heptio/ark/releases/ [4]: https://kubernetes.io/docs/concepts/storage/volumes/#local -[5]: http://restic.readthedocs.io/en/latest/100_references.html#terminology \ No newline at end of file +[5]: http://restic.readthedocs.io/en/latest/100_references.html#terminology diff --git a/docs/storage-layout-reorg-v0.10.md b/docs/storage-layout-reorg-v0.10.md index 4ac5223174..d4eb47fdea 100644 --- a/docs/storage-layout-reorg-v0.10.md +++ b/docs/storage-layout-reorg-v0.10.md @@ -26,6 +26,25 @@ Prior to v0.10, Ark stored data in an object storage bucket using the following ... ``` +Ark also stored restic data, if applicable, in a separate object storage bucket, structured as: + +``` +/[/] + namespace-1/ + data/ + index/ + keys/ + snapshots/ + config + namespace-2/ + data/ + index/ + keys/ + snapshots/ + config + ... +``` + As of v0.10, we've reorganized this layout to provide a cleaner and more extensible directory structure. The new layout looks like: ``` @@ -48,6 +67,20 @@ As of v0.10, we've reorganized this layout to provide a cleaner and more extensi restore-of-backup-2-logs.gz restore-of-backup-2-results.gz ... + restic/ + namespace-1/ + data/ + index/ + keys/ + snapshots/ + config + namespace-2/ + data/ + index/ + keys/ + snapshots/ + config + ... ... ``` @@ -104,7 +137,22 @@ rclone copy ${RCLONE_REMOTE_NAME}:${ARK_TEMP_MIGRATION_BUCKET} ${RCLONE_REMOTE_N # contains an exact copy of the temporary bucket's contents: rclone check ${RCLONE_REMOTE_NAME}:${ARK_BUCKET}/backups ${RCLONE_REMOTE_NAME}:${ARK_TEMP_MIGRATION_BUCKET} -# 9. Once you've confirmed that Ark v0.10 works with your revised Ark +# 9. OPTIONAL: If you have restic data to migrate: + +# a. Copy the contents of your Ark restic location into your +# Ark bucket, under the 'restic/' directory/prefix: + ARK_RESTIC_LOCATION= + rclone copy ${RCLONE_REMOTE_NAME}:${ARK_RESTIC_LOCATION} ${RCLONE_REMOTE_NAME}:${ARK_BUCKET}/restic + +# b. Check that the 'restic/' directory in your Ark bucket now +# contains an exact copy of your restic location: + rclone check ${RCLONE_REMOTE_NAME}:${ARK_BUCKET}/restic ${RCLONE_REMOTE_NAME}:${ARK_RESTIC_LOCATION} + +# c. Delete your ResticRepository custom resources to allow Ark +# to find them in the new location: + kubectl -n heptio-ark delete resticrepositories --all + +# 10. Once you've confirmed that Ark v0.10 works with your revised Ark # bucket, you can delete the temporary migration bucket. ``` diff --git a/examples/aws/05-ark-backupstoragelocation.yaml b/examples/aws/05-ark-backupstoragelocation.yaml index 04789c4781..da3abe6df7 100644 --- a/examples/aws/05-ark-backupstoragelocation.yaml +++ b/examples/aws/05-ark-backupstoragelocation.yaml @@ -24,9 +24,4 @@ spec: bucket: config: region: - # Uncomment the below line to enable restic integration. - # The format for resticLocation is [/], - # e.g. "my-restic-bucket" or "my-restic-bucket/repos". - # This MUST be a different bucket than the main Ark bucket - # specified just above. - # restic-location: + diff --git a/examples/azure/05-ark-backupstoragelocation.yaml b/examples/azure/05-ark-backupstoragelocation.yaml index 9a94d2a2e1..6e06c495af 100644 --- a/examples/azure/05-ark-backupstoragelocation.yaml +++ b/examples/azure/05-ark-backupstoragelocation.yaml @@ -25,9 +25,3 @@ spec: config: resourceGroup: storageAccount: - # Uncomment the below line to enable restic integration. - # The format for resticLocation is [/], - # e.g. "my-restic-bucket" or "my-restic-bucket/repos". - # This MUST be a different bucket than the main Ark bucket - # specified just above. - # restic-location: \ No newline at end of file diff --git a/examples/gcp/05-ark-backupstoragelocation.yaml b/examples/gcp/05-ark-backupstoragelocation.yaml index 346aff838a..34c83b98c9 100644 --- a/examples/gcp/05-ark-backupstoragelocation.yaml +++ b/examples/gcp/05-ark-backupstoragelocation.yaml @@ -21,10 +21,4 @@ metadata: spec: provider: gcp objectStorage: - bucket: - # Uncomment the below line to enable restic integration. - # The format for resticLocation is [/], - # e.g. "my-restic-bucket" or "my-restic-bucket/repos". - # This MUST be a different bucket than the main Ark bucket - # specified just above. - # restic-location: \ No newline at end of file + bucket: diff --git a/examples/ibm/05-ark-backupstoragelocation.yaml b/examples/ibm/05-ark-backupstoragelocation.yaml index 2e65b45ad3..21f89df2b5 100644 --- a/examples/ibm/05-ark-backupstoragelocation.yaml +++ b/examples/ibm/05-ark-backupstoragelocation.yaml @@ -26,9 +26,3 @@ spec: s3ForcePathStyle: "true" s3Url: region: - # Uncomment the below line to enable restic integration. - # The format for resticLocation is [/], - # e.g. "my-restic-bucket" or "my-restic-bucket/repos". - # This MUST be a different bucket than the main Ark bucket - # specified just above. - # restic-location: diff --git a/pkg/cmd/server/server.go b/pkg/cmd/server/server.go index 28cf9b8a71..ad2edf8381 100644 --- a/pkg/cmd/server/server.go +++ b/pkg/cmd/server/server.go @@ -289,10 +289,8 @@ func (s *server) run() error { s.blockStore = blockStore } - if backupStorageLocation.Spec.Config[restic.ResticLocationConfigKey] != "" { - if err := s.initRestic(backupStorageLocation); err != nil { - return err - } + if err := s.initRestic(backupStorageLocation); err != nil { + return err } if err := s.runControllers(config, backupStorageLocation); err != nil { @@ -749,22 +747,20 @@ func (s *server) runControllers(config *api.Config, defaultBackupLocation *api.B wg.Done() }() - if s.resticManager != nil { - resticRepoController := controller.NewResticRepositoryController( - s.logger, - s.sharedInformerFactory.Ark().V1().ResticRepositories(), - s.arkClient.ArkV1(), - defaultBackupLocation, - s.resticManager, - ) - wg.Add(1) - go func() { - // TODO only having a single worker may be an issue since maintenance - // can take a long time. - resticRepoController.Run(ctx, 1) - wg.Done() - }() - } + resticRepoController := controller.NewResticRepositoryController( + s.logger, + s.sharedInformerFactory.Ark().V1().ResticRepositories(), + s.arkClient.ArkV1(), + defaultBackupLocation, + s.resticManager, + ) + wg.Add(1) + go func() { + // TODO only having a single worker may be an issue since maintenance + // can take a long time. + resticRepoController.Run(ctx, 1) + wg.Done() + }() // SHARED INFORMERS HAVE TO BE STARTED AFTER ALL CONTROLLERS go s.sharedInformerFactory.Start(ctx.Done()) diff --git a/pkg/controller/pod_volume_restore_controller.go b/pkg/controller/pod_volume_restore_controller.go index 78b4672400..73ed524fe2 100644 --- a/pkg/controller/pod_volume_restore_controller.go +++ b/pkg/controller/pod_volume_restore_controller.go @@ -161,11 +161,9 @@ func (c *podVolumeRestoreController) podHandler(obj interface{}) { return } - selector, err := labels.Parse(fmt.Sprintf("%s=%s", arkv1api.PodUIDLabel, pod.UID)) - if err != nil { - log.WithError(err).Error("Unable to parse label selector %s", fmt.Sprintf("%s=%s", arkv1api.PodUIDLabel, pod.UID)) - return - } + selector := labels.Set(map[string]string{ + arkv1api.PodUIDLabel: string(pod.UID), + }).AsSelector() pvrs, err := c.podVolumeRestoreLister.List(selector) if err != nil { diff --git a/pkg/restic/common.go b/pkg/restic/common.go index 3b8797430d..d14c0db27a 100644 --- a/pkg/restic/common.go +++ b/pkg/restic/common.go @@ -36,7 +36,6 @@ const ( DaemonSet = "restic" InitContainer = "restic-wait" DefaultMaintenanceFrequency = 24 * time.Hour - ResticLocationConfigKey = "restic-location" podAnnotationPrefix = "snapshot.ark.heptio.com/" volumesToBackupAnnotation = "backup.ark.heptio.com/backup-volumes" @@ -117,10 +116,9 @@ type SnapshotIdentifier struct { // GetSnapshotsInBackup returns a list of all restic snapshot ids associated with // a given Ark backup. func GetSnapshotsInBackup(backup *arkv1api.Backup, podVolumeBackupLister arkv1listers.PodVolumeBackupLister) ([]SnapshotIdentifier, error) { - selector, err := labels.Parse(fmt.Sprintf("%s=%s", arkv1api.BackupNameLabel, backup.Name)) - if err != nil { - return nil, errors.WithStack(err) - } + selector := labels.Set(map[string]string{ + arkv1api.BackupNameLabel: backup.Name, + }).AsSelector() podVolumeBackups, err := podVolumeBackupLister.List(selector) if err != nil { diff --git a/pkg/restic/config.go b/pkg/restic/config.go index 1e70d7d919..7d3e417324 100644 --- a/pkg/restic/config.go +++ b/pkg/restic/config.go @@ -18,10 +18,12 @@ package restic import ( "fmt" + "path" "strings" arkv1api "github.com/heptio/ark/pkg/apis/ark/v1" "github.com/heptio/ark/pkg/cloudprovider/aws" + "github.com/heptio/ark/pkg/persistence" ) type BackendType string @@ -39,18 +41,15 @@ var getAWSBucketRegion = aws.GetBucketRegion // getRepoPrefix returns the prefix of the value of the --repo flag for // restic commands, i.e. everything except the "/". func getRepoPrefix(location *arkv1api.BackupStorageLocation) string { - var ( - resticLocation = location.Spec.Config[ResticLocationConfigKey] - parts = strings.SplitN(resticLocation, "/", 2) - bucket, path, prefix string - ) - - if len(parts) >= 1 { - bucket = parts[0] - } - if len(parts) >= 2 { - path = parts[1] + var provider, bucket, prefix, bucketAndPrefix string + + if location.Spec.ObjectStorage != nil { + layout := persistence.NewObjectStoreLayout(location.Spec.ObjectStorage.Prefix) + + bucket = location.Spec.ObjectStorage.Bucket + prefix = layout.GetResticDir() } + bucketAndPrefix = path.Join(bucket, prefix) switch BackendType(location.Spec.Provider) { case AWSBackend: @@ -69,14 +68,14 @@ func getRepoPrefix(location *arkv1api.BackupStorageLocation) string { url = fmt.Sprintf("s3-%s.amazonaws.com", region) } - return fmt.Sprintf("s3:%s/%s", url, resticLocation) + return fmt.Sprintf("s3:%s/%s", url, bucketAndPrefix) case AzureBackend: - prefix = "azure" + provider = "azure" case GCPBackend: - prefix = "gs" + provider = "gs" } - return fmt.Sprintf("%s:%s:/%s", prefix, bucket, path) + return fmt.Sprintf("%s:%s:/%s", provider, bucket, prefix) } // GetRepoIdentifier returns the string to be used as the value of the --repo flag in diff --git a/pkg/restic/config_test.go b/pkg/restic/config_test.go index 982c1dca1c..31ff68140c 100644 --- a/pkg/restic/config_test.go +++ b/pkg/restic/config_test.go @@ -34,10 +34,15 @@ func TestGetRepoIdentifier(t *testing.T) { backupLocation := &arkv1api.BackupStorageLocation{ Spec: arkv1api.BackupStorageLocationSpec{ Provider: "aws", - Config: map[string]string{ResticLocationConfigKey: "bucket/prefix"}, + StorageType: arkv1api.StorageType{ + ObjectStorage: &arkv1api.ObjectStorageLocation{ + Bucket: "bucket", + Prefix: "prefix", + }, + }, }, } - assert.Equal(t, "s3:s3.amazonaws.com/bucket/prefix/repo-1", GetRepoIdentifier(backupLocation, "repo-1")) + assert.Equal(t, "s3:s3.amazonaws.com/bucket/prefix/restic/repo-1", GetRepoIdentifier(backupLocation, "repo-1")) // stub implementation of getAWSBucketRegion getAWSBucketRegion = func(string) (string, error) { @@ -47,43 +52,67 @@ func TestGetRepoIdentifier(t *testing.T) { backupLocation = &arkv1api.BackupStorageLocation{ Spec: arkv1api.BackupStorageLocationSpec{ Provider: "aws", - Config: map[string]string{ResticLocationConfigKey: "bucket"}, + StorageType: arkv1api.StorageType{ + ObjectStorage: &arkv1api.ObjectStorageLocation{ + Bucket: "bucket", + }, + }, }, } - assert.Equal(t, "s3:s3-us-west-2.amazonaws.com/bucket/repo-1", GetRepoIdentifier(backupLocation, "repo-1")) + assert.Equal(t, "s3:s3-us-west-2.amazonaws.com/bucket/restic/repo-1", GetRepoIdentifier(backupLocation, "repo-1")) backupLocation = &arkv1api.BackupStorageLocation{ Spec: arkv1api.BackupStorageLocationSpec{ Provider: "aws", - Config: map[string]string{ResticLocationConfigKey: "bucket/prefix"}, + StorageType: arkv1api.StorageType{ + ObjectStorage: &arkv1api.ObjectStorageLocation{ + Bucket: "bucket", + Prefix: "prefix", + }, + }, }, } - assert.Equal(t, "s3:s3-us-west-2.amazonaws.com/bucket/prefix/repo-1", GetRepoIdentifier(backupLocation, "repo-1")) + assert.Equal(t, "s3:s3-us-west-2.amazonaws.com/bucket/prefix/restic/repo-1", GetRepoIdentifier(backupLocation, "repo-1")) backupLocation = &arkv1api.BackupStorageLocation{ Spec: arkv1api.BackupStorageLocationSpec{ Provider: "aws", Config: map[string]string{ - ResticLocationConfigKey: "bucket/prefix", - "s3Url": "alternate-url", + "s3Url": "alternate-url", + }, + StorageType: arkv1api.StorageType{ + ObjectStorage: &arkv1api.ObjectStorageLocation{ + Bucket: "bucket", + Prefix: "prefix", + }, }, }, } - assert.Equal(t, "s3:alternate-url/bucket/prefix/repo-1", GetRepoIdentifier(backupLocation, "repo-1")) + assert.Equal(t, "s3:alternate-url/bucket/prefix/restic/repo-1", GetRepoIdentifier(backupLocation, "repo-1")) backupLocation = &arkv1api.BackupStorageLocation{ Spec: arkv1api.BackupStorageLocationSpec{ Provider: "azure", - Config: map[string]string{ResticLocationConfigKey: "bucket/prefix"}, + StorageType: arkv1api.StorageType{ + ObjectStorage: &arkv1api.ObjectStorageLocation{ + Bucket: "bucket", + Prefix: "prefix", + }, + }, }, } - assert.Equal(t, "azure:bucket:/prefix/repo-1", GetRepoIdentifier(backupLocation, "repo-1")) + assert.Equal(t, "azure:bucket:/prefix/restic/repo-1", GetRepoIdentifier(backupLocation, "repo-1")) backupLocation = &arkv1api.BackupStorageLocation{ Spec: arkv1api.BackupStorageLocationSpec{ Provider: "gcp", - Config: map[string]string{ResticLocationConfigKey: "bucket-2/prefix-2"}, + StorageType: arkv1api.StorageType{ + ObjectStorage: &arkv1api.ObjectStorageLocation{ + Bucket: "bucket-2", + Prefix: "prefix-2", + }, + }, }, } - assert.Equal(t, "gs:bucket-2:/prefix-2/repo-2", GetRepoIdentifier(backupLocation, "repo-2")) + assert.Equal(t, "gs:bucket-2:/prefix-2/restic/repo-2", GetRepoIdentifier(backupLocation, "repo-2")) }