From 02d50dae39254bf409b74bb8d0f371c142109cfb Mon Sep 17 00:00:00 2001 From: Naadir Jeewa Date: Fri, 31 Jan 2020 17:24:46 +0000 Subject: [PATCH] Support use of AWS Secrets Manager for userdata privacy --- Makefile | 4 +- api/v1alpha2/awsmachine_conversion.go | 43 +++- api/v1alpha2/awsmachine_conversion_test.go | 84 ++++++++ api/v1alpha2/awsmachine_types.go | 21 ++ api/v1alpha2/zz_generated.conversion.go | 24 +++ api/v1alpha2/zz_generated.deepcopy.go | 20 ++ api/v1alpha3/awsmachine_types.go | 21 ++ api/v1alpha3/awsmachine_webhook.go | 28 ++- api/v1alpha3/awsmachine_webhook_test.go | 5 +- api/v1alpha3/awsmachinetemplate_webhook.go | 14 +- .../awsmachinetemplate_webhook_test.go | 71 ++++++ api/v1alpha3/types.go | 18 ++ api/v1alpha3/webhooks.go | 35 +++ api/v1alpha3/zz_generated.deepcopy.go | 16 ++ ...tructure.cluster.x-k8s.io_awsmachines.yaml | 33 +++ ....cluster.x-k8s.io_awsmachinetemplates.yaml | 36 ++++ controllers/awsmachine_controller.go | 142 +++++++++--- controllers/awsmachine_controller_test.go | 160 ++++++++++++-- docs/README.md | 2 + docs/userdata-privacy.md | 30 +++ go.mod | 7 +- go.sum | 20 +- pkg/cloud/converters/tags.go | 17 ++ pkg/cloud/scope/clients.go | 2 + pkg/cloud/scope/cluster.go | 7 + pkg/cloud/scope/machine.go | 61 +++++- pkg/cloud/scope/machine_test.go | 204 ++++++++++++++++++ .../services/cloudformation/bootstrap.go | 33 ++- pkg/cloud/services/ec2/instances.go | 37 +--- pkg/cloud/services/ec2/instances_test.go | 2 +- pkg/cloud/services/interfaces.go | 9 +- pkg/cloud/services/mock_services/doc.go | 2 + .../ec2_machine_interface_mock.go | 8 +- .../secretsmanager_machine_interface_mock.go | 79 +++++++ .../services/secretsmanager/cloudinit.go | 94 ++++++++ .../services/secretsmanager/cloudinit_test.go | 33 +++ pkg/cloud/services/secretsmanager/secret.go | 104 +++++++++ .../secretsmanager/secret_fetch_script.go | 156 ++++++++++++++ pkg/cloud/services/secretsmanager/service.go | 35 +++ pkg/cloud/services/userdata/utils.go | 19 ++ 40 files changed, 1632 insertions(+), 104 deletions(-) create mode 100644 api/v1alpha2/awsmachine_conversion_test.go create mode 100644 api/v1alpha3/awsmachinetemplate_webhook_test.go create mode 100644 api/v1alpha3/webhooks.go create mode 100644 docs/userdata-privacy.md create mode 100644 pkg/cloud/scope/machine_test.go create mode 100644 pkg/cloud/services/mock_services/secretsmanager_machine_interface_mock.go create mode 100644 pkg/cloud/services/secretsmanager/cloudinit.go create mode 100644 pkg/cloud/services/secretsmanager/cloudinit_test.go create mode 100644 pkg/cloud/services/secretsmanager/secret.go create mode 100644 pkg/cloud/services/secretsmanager/secret_fetch_script.go create mode 100644 pkg/cloud/services/secretsmanager/service.go diff --git a/Makefile b/Makefile index 77f8c54653..3fa579bc2e 100644 --- a/Makefile +++ b/Makefile @@ -153,8 +153,7 @@ generate: ## Generate code $(MAKE) generate-manifests .PHONY: generate-go -generate-go: $(CONTROLLER_GEN) $(MOCKGEN) $(CONVERSION_GEN) ## Runs Go related generate targets - go generate ./... +generate-go: $(CONTROLLER_GEN) $(CONVERSION_GEN) $(MOCKGEN) ## Runs Go related generate targets $(CONTROLLER_GEN) \ paths=./api/... \ object:headerFile=./hack/boilerplate/boilerplate.generatego.txt @@ -163,6 +162,7 @@ generate-go: $(CONTROLLER_GEN) $(MOCKGEN) $(CONVERSION_GEN) ## Runs Go related g --input-dirs=./api/v1alpha2 \ --output-file-base=zz_generated.conversion \ --go-header-file=./hack/boilerplate/boilerplate.generatego.txt + go generate ./... .PHONY: generate-manifests generate-manifests: $(CONTROLLER_GEN) ## Generate manifests e.g. CRD, RBAC etc. diff --git a/api/v1alpha2/awsmachine_conversion.go b/api/v1alpha2/awsmachine_conversion.go index 403e06a4b8..812c5462bd 100644 --- a/api/v1alpha2/awsmachine_conversion.go +++ b/api/v1alpha2/awsmachine_conversion.go @@ -19,23 +19,52 @@ package v1alpha2 import ( apiconversion "k8s.io/apimachinery/pkg/conversion" infrav1alpha3 "sigs.k8s.io/cluster-api-provider-aws/api/v1alpha3" + utilconversion "sigs.k8s.io/cluster-api/util/conversion" "sigs.k8s.io/controller-runtime/pkg/conversion" ) // ConvertTo converts this AWSMachine to the Hub version (v1alpha3). func (src *AWSMachine) ConvertTo(dstRaw conversion.Hub) error { // nolint dst := dstRaw.(*infrav1alpha3.AWSMachine) + if err := Convert_v1alpha2_AWSMachine_To_v1alpha3_AWSMachine(src, dst, nil); err != nil { return err } + // Manually restore data from annotations + restored := &infrav1alpha3.AWSMachine{} + if ok, err := utilconversion.UnmarshalData(src, restored); err != nil || !ok { + return err + } + + restoreAWSMachineSpec(&restored.Spec, &dst.Spec) + return nil } +func restoreAWSMachineSpec(restored *infrav1alpha3.AWSMachineSpec, dst *infrav1alpha3.AWSMachineSpec) { + dst.ImageLookupBaseOS = restored.ImageLookupBaseOS + // Conversion for route: v1alpha3 --> management cluster running v1alpha2 on <= v0.4.8 --> v1alpha3 + if !dst.CloudInit.InsecureSkipSecretsManager && dst.CloudInit.SecretARN == "" { + dst.CloudInit.InsecureSkipSecretsManager = restored.CloudInit.InsecureSkipSecretsManager + dst.CloudInit.SecretARN = restored.CloudInit.SecretARN + } +} + // ConvertFrom converts from the Hub version (v1alpha3) to this version. func (dst *AWSMachine) ConvertFrom(srcRaw conversion.Hub) error { // nolint src := srcRaw.(*infrav1alpha3.AWSMachine) - return Convert_v1alpha3_AWSMachine_To_v1alpha2_AWSMachine(src, dst, nil) + + if err := Convert_v1alpha3_AWSMachine_To_v1alpha2_AWSMachine(src, dst, nil); err != nil { + return err + } + + // Preserve Hub data on down-conversion. + if err := utilconversion.MarshalData(src, dst); err != nil { + return err + } + + return nil } // ConvertTo converts this AWSMachineList to the Hub version (v1alpha3). @@ -101,3 +130,15 @@ func Convert_v1alpha2_AWSMachineSpec_To_v1alpha3_AWSMachineSpec(in *AWSMachineSp return nil } + +func Convert_v1alpha2_CloudInit_To_v1alpha3_CloudInit(in *CloudInit, out *infrav1alpha3.CloudInit, s apiconversion.Scope) error { // nolint + out.SecretARN = in.SecretARN + out.InsecureSkipSecretsManager = !in.EnableSecureSecretsManager + return nil +} + +func Convert_v1alpha3_CloudInit_To_v1alpha2_CloudInit(in *infrav1alpha3.CloudInit, out *CloudInit, s apiconversion.Scope) error { // nolint + out.SecretARN = in.SecretARN + out.EnableSecureSecretsManager = !in.InsecureSkipSecretsManager + return nil +} diff --git a/api/v1alpha2/awsmachine_conversion_test.go b/api/v1alpha2/awsmachine_conversion_test.go new file mode 100644 index 0000000000..7db2fc3b34 --- /dev/null +++ b/api/v1alpha2/awsmachine_conversion_test.go @@ -0,0 +1,84 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha2 + +import ( + "testing" + + . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + infrav1alpha3 "sigs.k8s.io/cluster-api-provider-aws/api/v1alpha3" +) + +func TestConvertAWSMachine(t *testing.T) { + g := NewWithT(t) + + t.Run("from hub", func(t *testing.T) { + t.Run("should restore SecretARN, assuming old version of object without field", func(t *testing.T) { + src := &infrav1alpha3.AWSMachine{ + ObjectMeta: metav1.ObjectMeta{}, + Spec: infrav1alpha3.AWSMachineSpec{ + CloudInit: infrav1alpha3.CloudInit{ + InsecureSkipSecretsManager: true, + SecretARN: "something", + }, + }, + } + dst := &AWSMachine{} + g.Expect(dst.ConvertFrom(src)).To(Succeed()) + restored := &infrav1alpha3.AWSMachine{} + g.Expect(dst.ConvertTo(restored)).To(Succeed()) + g.Expect(restored.Spec.CloudInit.SecretARN).To(Equal(src.Spec.CloudInit.SecretARN)) + g.Expect(restored.Spec.CloudInit.InsecureSkipSecretsManager).To(Equal(src.Spec.CloudInit.InsecureSkipSecretsManager)) + }) + }) + t.Run("should prefer newer cloudinit data on the v1alpha2 obj", func(t *testing.T) { + src := &infrav1alpha3.AWSMachine{ + ObjectMeta: metav1.ObjectMeta{}, + Spec: infrav1alpha3.AWSMachineSpec{ + CloudInit: infrav1alpha3.CloudInit{ + SecretARN: "something", + }, + }, + } + dst := &AWSMachine{ + Spec: AWSMachineSpec{ + CloudInit: &CloudInit{ + EnableSecureSecretsManager: true, + SecretARN: "something-else", + }, + }, + } + g.Expect(dst.ConvertFrom(src)).To(Succeed()) + restored := &infrav1alpha3.AWSMachine{} + g.Expect(dst.ConvertTo(restored)).To(Succeed()) + g.Expect(restored.Spec.CloudInit.SecretARN).To(Equal(src.Spec.CloudInit.SecretARN)) + }) + t.Run("should restore ImageLookupBaseOS", func(t *testing.T) { + src := &infrav1alpha3.AWSMachine{ + ObjectMeta: metav1.ObjectMeta{}, + Spec: infrav1alpha3.AWSMachineSpec{ + ImageLookupBaseOS: "amazon-linux", + }, + } + dst := &AWSMachine{} + g.Expect(dst.ConvertFrom(src)).To(Succeed()) + restored := &infrav1alpha3.AWSMachine{} + g.Expect(dst.ConvertTo(restored)).To(Succeed()) + g.Expect(restored.Spec.ImageLookupBaseOS).To(Equal(src.Spec.ImageLookupBaseOS)) + }) +} diff --git a/api/v1alpha2/awsmachine_types.go b/api/v1alpha2/awsmachine_types.go index 43a8845a61..3ca1cc97ce 100644 --- a/api/v1alpha2/awsmachine_types.go +++ b/api/v1alpha2/awsmachine_types.go @@ -88,6 +88,27 @@ type AWSMachineSpec struct { // +optional // +kubebuilder:validation:MaxItems=2 NetworkInterfaces []string `json:"networkInterfaces,omitempty"` + + // CloudInit defines options related to the bootstrapping systems where + // CloudInit is used. + // +optional + CloudInit *CloudInit `json:"cloudInit,omitempty"` +} + +// CloudInit defines options related to the bootstrapping systems where +// CloudInit is used. +type CloudInit struct { + // enableSecureSecretsManager, when set to true will use AWS Secrets Manager to ensure + // userdata privacy. A cloud-init boothook shell script is prepended to download + // the userdata from Secrets Manager and additionally delete the secret. + // +optional + EnableSecureSecretsManager bool `json:"enableSecureSecretsManager,omitempty"` + + // SecretARN is the Amazon Resource Name of the secret. This is stored + // temporarily, and deleted when the machine registers as a node against + // the workload cluster. + // +optional + SecretARN string `json:"secretARN,omitempty"` } // AWSMachineStatus defines the observed state of AWSMachine diff --git a/api/v1alpha2/zz_generated.conversion.go b/api/v1alpha2/zz_generated.conversion.go index 92f521d4d2..7d80f13bd4 100644 --- a/api/v1alpha2/zz_generated.conversion.go +++ b/api/v1alpha2/zz_generated.conversion.go @@ -282,6 +282,11 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddConversionFunc((*CloudInit)(nil), (*v1alpha3.CloudInit)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha2_CloudInit_To_v1alpha3_CloudInit(a.(*CloudInit), b.(*v1alpha3.CloudInit), scope) + }); err != nil { + return err + } if err := s.AddConversionFunc((*v1alpha3.AWSClusterSpec)(nil), (*AWSClusterSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha3_AWSClusterSpec_To_v1alpha2_AWSClusterSpec(a.(*v1alpha3.AWSClusterSpec), b.(*AWSClusterSpec), scope) }); err != nil { @@ -317,6 +322,11 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddConversionFunc((*v1alpha3.CloudInit)(nil), (*CloudInit)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha3_CloudInit_To_v1alpha2_CloudInit(a.(*v1alpha3.CloudInit), b.(*CloudInit), scope) + }); err != nil { + return err + } return nil } @@ -563,6 +573,7 @@ func autoConvert_v1alpha2_AWSMachineSpec_To_v1alpha3_AWSMachineSpec(in *AWSMachi out.SSHKeyName = in.SSHKeyName out.RootDeviceSize = in.RootDeviceSize out.NetworkInterfaces = *(*[]string)(unsafe.Pointer(&in.NetworkInterfaces)) + // WARNING: in.CloudInit requires manual conversion: inconvertible types (*sigs.k8s.io/cluster-api-provider-aws/api/v1alpha2.CloudInit vs sigs.k8s.io/cluster-api-provider-aws/api/v1alpha3.CloudInit) return nil } @@ -583,6 +594,7 @@ func autoConvert_v1alpha3_AWSMachineSpec_To_v1alpha2_AWSMachineSpec(in *v1alpha3 out.SSHKeyName = in.SSHKeyName out.RootDeviceSize = in.RootDeviceSize out.NetworkInterfaces = *(*[]string)(unsafe.Pointer(&in.NetworkInterfaces)) + // WARNING: in.CloudInit requires manual conversion: inconvertible types (sigs.k8s.io/cluster-api-provider-aws/api/v1alpha3.CloudInit vs *sigs.k8s.io/cluster-api-provider-aws/api/v1alpha2.CloudInit) return nil } @@ -880,6 +892,18 @@ func Convert_v1alpha3_ClassicELBListener_To_v1alpha2_ClassicELBListener(in *v1al return autoConvert_v1alpha3_ClassicELBListener_To_v1alpha2_ClassicELBListener(in, out, s) } +func autoConvert_v1alpha2_CloudInit_To_v1alpha3_CloudInit(in *CloudInit, out *v1alpha3.CloudInit, s conversion.Scope) error { + // WARNING: in.EnableSecureSecretsManager requires manual conversion: does not exist in peer-type + out.SecretARN = in.SecretARN + return nil +} + +func autoConvert_v1alpha3_CloudInit_To_v1alpha2_CloudInit(in *v1alpha3.CloudInit, out *CloudInit, s conversion.Scope) error { + // WARNING: in.InsecureSkipSecretsManager requires manual conversion: does not exist in peer-type + out.SecretARN = in.SecretARN + return nil +} + func autoConvert_v1alpha2_Filter_To_v1alpha3_Filter(in *Filter, out *v1alpha3.Filter, s conversion.Scope) error { out.Name = in.Name out.Values = *(*[]string)(unsafe.Pointer(&in.Values)) diff --git a/api/v1alpha2/zz_generated.deepcopy.go b/api/v1alpha2/zz_generated.deepcopy.go index 982620abf0..7be7c3cd8e 100644 --- a/api/v1alpha2/zz_generated.deepcopy.go +++ b/api/v1alpha2/zz_generated.deepcopy.go @@ -272,6 +272,11 @@ func (in *AWSMachineSpec) DeepCopyInto(out *AWSMachineSpec) { *out = make([]string, len(*in)) copy(*out, *in) } + if in.CloudInit != nil { + in, out := &in.CloudInit, &out.CloudInit + *out = new(CloudInit) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AWSMachineSpec. @@ -567,6 +572,21 @@ func (in *ClassicELBListener) DeepCopy() *ClassicELBListener { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CloudInit) DeepCopyInto(out *CloudInit) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudInit. +func (in *CloudInit) DeepCopy() *CloudInit { + if in == nil { + return nil + } + out := new(CloudInit) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Filter) DeepCopyInto(out *Filter) { *out = *in diff --git a/api/v1alpha3/awsmachine_types.go b/api/v1alpha3/awsmachine_types.go index d7f7d16a95..5e0bfc976a 100644 --- a/api/v1alpha3/awsmachine_types.go +++ b/api/v1alpha3/awsmachine_types.go @@ -92,6 +92,27 @@ type AWSMachineSpec struct { // +optional // +kubebuilder:validation:MaxItems=2 NetworkInterfaces []string `json:"networkInterfaces,omitempty"` + + // CloudInit defines options related to the bootstrapping systems where + // CloudInit is used. + // +optional + CloudInit CloudInit `json:"cloudInit,omitempty"` +} + +// CloudInit defines options related to the bootstrapping systems where +// CloudInit is used. +type CloudInit struct { + // InsecureSkipSecretsManager, when set to true will not use AWS Secrets Manager + // to ensure privacy of userdata. + // By default, a cloud-init boothook shell script is prepended to download + // the userdata from Secrets Manager and additionally delete the secret. + InsecureSkipSecretsManager bool `json:"insecureSkipSecretsManager,omitempty"` + + // SecretARN is the Amazon Resource Name of the secret. This is stored + // temporarily, and deleted when the machine registers as a node against + // the workload cluster. + // +optional + SecretARN string `json:"secretARN,omitempty"` } // AWSMachineStatus defines the observed state of AWSMachine diff --git a/api/v1alpha3/awsmachine_webhook.go b/api/v1alpha3/awsmachine_webhook.go index cc5b712829..1225463586 100644 --- a/api/v1alpha3/awsmachine_webhook.go +++ b/api/v1alpha3/awsmachine_webhook.go @@ -43,7 +43,7 @@ var _ webhook.Validator = &AWSMachine{} // ValidateCreate implements webhook.Validator so a webhook will be registered for the type func (r *AWSMachine) ValidateCreate() error { - return nil + return aggregateObjErrors(r.GroupVersionKind().GroupKind(), r.Name, r.validateCloudInitSecret()) } // ValidateUpdate implements webhook.Validator so a webhook will be registered for the type @@ -63,6 +63,8 @@ func (r *AWSMachine) ValidateUpdate(old runtime.Object) error { var allErrs field.ErrorList + allErrs = append(allErrs, r.validateCloudInitSecret()...) + newAWSMachineSpec := newAWSMachine["spec"].(map[string]interface{}) oldAWSMachineSpec := oldAWSMachine["spec"].(map[string]interface{}) @@ -78,14 +80,30 @@ func (r *AWSMachine) ValidateUpdate(old runtime.Object) error { delete(oldAWSMachineSpec, "additionalSecurityGroups") delete(newAWSMachineSpec, "additionalSecurityGroups") + // allow changes to secretARN + if cloudInit, ok := oldAWSMachineSpec["cloudInit"].(map[string]interface{}); ok { + delete(cloudInit, "secretARN") + } + + if cloudInit, ok := newAWSMachineSpec["cloudInit"].(map[string]interface{}); ok { + delete(cloudInit, "secretARN") + } + if !reflect.DeepEqual(oldAWSMachineSpec, newAWSMachineSpec) { allErrs = append(allErrs, field.Forbidden(field.NewPath("spec"), "cannot be modified")) - return apierrors.NewInvalid( - GroupVersion.WithKind("AWSMachine").GroupKind(), - r.Name, allErrs) } - return nil + return aggregateObjErrors(r.GroupVersionKind().GroupKind(), r.Name, allErrs) +} + +func (r *AWSMachine) validateCloudInitSecret() field.ErrorList { + var allErrs field.ErrorList + + if r.Spec.CloudInit.SecretARN != "" && r.Spec.CloudInit.InsecureSkipSecretsManager { + allErrs = append(allErrs, field.Forbidden(field.NewPath("spec", "cloudInit", "secretARN"), "cannot be set if spec.cloudInit.insecureSkipSecretsManager is true")) + } + + return allErrs } // ValidateDelete implements webhook.Validator so a webhook will be registered for the type diff --git a/api/v1alpha3/awsmachine_webhook_test.go b/api/v1alpha3/awsmachine_webhook_test.go index c8b42e964e..7bebb968a4 100644 --- a/api/v1alpha3/awsmachine_webhook_test.go +++ b/api/v1alpha3/awsmachine_webhook_test.go @@ -30,7 +30,7 @@ func TestAWSMachine_ValidateUpdate(t *testing.T) { wantErr bool }{ { - name: "change in providerid, tags and securitygroups", + name: "change in providerid, cloudinit, tags and securitygroups", oldMachine: &AWSMachine{ Spec: AWSMachineSpec{ ProviderID: nil, @@ -49,6 +49,9 @@ func TestAWSMachine_ValidateUpdate(t *testing.T) { ID: pointer.StringPtr("ID"), }, }, + CloudInit: CloudInit{ + SecretARN: "test", + }, }, }, wantErr: false, diff --git a/api/v1alpha3/awsmachinetemplate_webhook.go b/api/v1alpha3/awsmachinetemplate_webhook.go index ac915862f2..70307d3038 100644 --- a/api/v1alpha3/awsmachinetemplate_webhook.go +++ b/api/v1alpha3/awsmachinetemplate_webhook.go @@ -21,6 +21,7 @@ import ( "reflect" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/validation/field" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/webhook" ) @@ -37,7 +38,18 @@ var _ webhook.Validator = &AWSMachineTemplate{} // ValidateCreate implements webhook.Validator so a webhook will be registered for the type func (r *AWSMachineTemplate) ValidateCreate() error { - return nil + var allErrs field.ErrorList + spec := r.Spec.Template.Spec + + if spec.CloudInit.SecretARN != "" { + allErrs = append(allErrs, field.Forbidden(field.NewPath("spec", "template", "spec", "cloudInit", "secretARN"), "cannot be set in templates")) + } + + if spec.ProviderID != nil { + allErrs = append(allErrs, field.Forbidden(field.NewPath("spec", "template", "spec", "providerID"), "cannot be set in templates")) + } + + return aggregateObjErrors(r.GroupVersionKind().GroupKind(), r.Name, allErrs) } // ValidateUpdate implements webhook.Validator so a webhook will be registered for the type diff --git a/api/v1alpha3/awsmachinetemplate_webhook_test.go b/api/v1alpha3/awsmachinetemplate_webhook_test.go new file mode 100644 index 0000000000..8cf9e81e0d --- /dev/null +++ b/api/v1alpha3/awsmachinetemplate_webhook_test.go @@ -0,0 +1,71 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha3 + +import ( + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/pointer" +) + +func TestAWSMachineTemplateInvalid(t *testing.T) { + tests := []struct { + name string + inputTemplate *AWSMachineTemplate + wantError bool + }{ + { + name: "don't allow providerID", + inputTemplate: &AWSMachineTemplate{ + ObjectMeta: metav1.ObjectMeta{}, + Spec: AWSMachineTemplateSpec{ + Template: AWSMachineTemplateResource{ + Spec: AWSMachineSpec{ + ProviderID: pointer.StringPtr("something"), + }, + }, + }, + }, + wantError: true, + }, + { + name: "don't allow secretARN", + inputTemplate: &AWSMachineTemplate{ + ObjectMeta: metav1.ObjectMeta{}, + Spec: AWSMachineTemplateSpec{ + Template: AWSMachineTemplateResource{ + Spec: AWSMachineSpec{ + CloudInit: CloudInit{ + SecretARN: "something", + }, + }, + }, + }, + }, + wantError: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.inputTemplate.ValidateCreate() + if (err != nil) != tt.wantError { + t.Errorf("unexpected result - wanted %+v, got %+v", tt.wantError, err) + } + }) + } +} diff --git a/api/v1alpha3/types.go b/api/v1alpha3/types.go index 86cdabd4da..b53594b896 100644 --- a/api/v1alpha3/types.go +++ b/api/v1alpha3/types.go @@ -22,6 +22,7 @@ import ( "time" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/sets" ) // AWSResourceReference is a reference to a specific AWS resource by ID, ARN, or filters. @@ -474,6 +475,23 @@ var ( // InstanceStateStopped is the string representing an instance // that has been stopped and can be restarted InstanceStateStopped = InstanceState("stopped") + + // InstanceOperationalStates defines the set of states in which an EC2 instance is + // or can return to running, and supports all EC2 operations + InstanceOperationalStates = sets.NewString( + string(InstanceStatePending), + string(InstanceStateRunning), + string(InstanceStateStopping), + string(InstanceStateStopped), + ) + + // InstanceKnownStates represents all known EC2 instance states + InstanceKnownStates = InstanceOperationalStates.Union( + sets.NewString( + string(InstanceStateShuttingDown), + string(InstanceStateTerminated), + ), + ) ) // Instance describes an AWS instance. diff --git a/api/v1alpha3/webhooks.go b/api/v1alpha3/webhooks.go new file mode 100644 index 0000000000..306dccb947 --- /dev/null +++ b/api/v1alpha3/webhooks.go @@ -0,0 +1,35 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha3 + +import ( + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/validation/field" +) + +func aggregateObjErrors(gk schema.GroupKind, name string, allErrs field.ErrorList) error { + if len(allErrs) == 0 { + return nil + } + + return apierrors.NewInvalid( + gk, + name, + allErrs, + ) +} diff --git a/api/v1alpha3/zz_generated.deepcopy.go b/api/v1alpha3/zz_generated.deepcopy.go index 3f513992c4..ba1e1c9c39 100644 --- a/api/v1alpha3/zz_generated.deepcopy.go +++ b/api/v1alpha3/zz_generated.deepcopy.go @@ -266,6 +266,7 @@ func (in *AWSMachineSpec) DeepCopyInto(out *AWSMachineSpec) { *out = make([]string, len(*in)) copy(*out, *in) } + out.CloudInit = in.CloudInit } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AWSMachineSpec. @@ -581,6 +582,21 @@ func (in *ClassicELBListener) DeepCopy() *ClassicELBListener { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CloudInit) DeepCopyInto(out *CloudInit) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudInit. +func (in *CloudInit) DeepCopy() *CloudInit { + if in == nil { + return nil + } + out := new(CloudInit) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Filter) DeepCopyInto(out *Filter) { *out = *in diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_awsmachines.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_awsmachines.yaml index ba89863c14..42b2183a15 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_awsmachines.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_awsmachines.yaml @@ -127,6 +127,22 @@ spec: to use for this instance. If multiple subnets are matched for the availability zone, the first one return is picked. type: string + cloudInit: + description: CloudInit defines options related to the bootstrapping + systems where CloudInit is used. + properties: + enableSecureSecretsManager: + description: enableSecureSecretsManager, when set to true will + use AWS Secrets Manager to ensure userdata privacy. A cloud-init + boothook shell script is prepended to download the userdata + from Secrets Manager and additionally delete the secret. + type: boolean + secretARN: + description: SecretARN is the Amazon Resource Name of the secret. + This is stored temporarily, and deleted when the machine registers + as a node against the workload cluster. + type: string + type: object iamInstanceProfile: description: IAMInstanceProfile is a name of an IAM instance profile to assign to the instance @@ -378,6 +394,23 @@ spec: description: ID of resource type: string type: object + cloudInit: + description: CloudInit defines options related to the bootstrapping + systems where CloudInit is used. + properties: + insecureSkipSecretsManager: + description: InsecureSkipSecretsManager, when set to true will + not use AWS Secrets Manager to ensure privacy of userdata. By + default, a cloud-init boothook shell script is prepended to + download the userdata from Secrets Manager and additionally + delete the secret. + type: boolean + secretARN: + description: SecretARN is the Amazon Resource Name of the secret. + This is stored temporarily, and deleted when the machine registers + as a node against the workload cluster. + type: string + type: object failureDomain: description: FailureDomain is the failure domain unique identifier this Machine should be attached to, as defined in Cluster API. For diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_awsmachinetemplates.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_awsmachinetemplates.yaml index d31f59b9b1..413a8f8f7d 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_awsmachinetemplates.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_awsmachinetemplates.yaml @@ -139,6 +139,24 @@ spec: zone to use for this instance. If multiple subnets are matched for the availability zone, the first one return is picked. type: string + cloudInit: + description: CloudInit defines options related to the bootstrapping + systems where CloudInit is used. + properties: + enableSecureSecretsManager: + description: enableSecureSecretsManager, when set to true + will use AWS Secrets Manager to ensure userdata privacy. + A cloud-init boothook shell script is prepended to download + the userdata from Secrets Manager and additionally delete + the secret. + type: boolean + secretARN: + description: SecretARN is the Amazon Resource Name of + the secret. This is stored temporarily, and deleted + when the machine registers as a node against the workload + cluster. + type: string + type: object iamInstanceProfile: description: IAMInstanceProfile is a name of an IAM instance profile to assign to the instance @@ -336,6 +354,24 @@ spec: description: ID of resource type: string type: object + cloudInit: + description: CloudInit defines options related to the bootstrapping + systems where CloudInit is used. + properties: + insecureSkipSecretsManager: + description: InsecureSkipSecretsManager, when set to true + will not use AWS Secrets Manager to ensure privacy of + userdata. By default, a cloud-init boothook shell script + is prepended to download the userdata from Secrets Manager + and additionally delete the secret. + type: boolean + secretARN: + description: SecretARN is the Amazon Resource Name of + the secret. This is stored temporarily, and deleted + when the machine registers as a node against the workload + cluster. + type: string + type: object failureDomain: description: FailureDomain is the failure domain unique identifier this Machine should be attached to, as defined in Cluster diff --git a/controllers/awsmachine_controller.go b/controllers/awsmachine_controller.go index 30825133e9..03aeab412a 100644 --- a/controllers/awsmachine_controller.go +++ b/controllers/awsmachine_controller.go @@ -31,6 +31,8 @@ import ( "sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/services" "sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/services/ec2" "sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/services/elb" + "sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/services/secretsmanager" + "sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/services/userdata" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" "sigs.k8s.io/cluster-api/controllers/noderefutil" capierrors "sigs.k8s.io/cluster-api/errors" @@ -46,19 +48,28 @@ import ( // AWSMachineReconciler reconciles a AwsMachine object type AWSMachineReconciler struct { client.Client - Log logr.Logger - Recorder record.EventRecorder - serviceFactory func(*scope.ClusterScope) services.EC2MachineInterface + Log logr.Logger + Recorder record.EventRecorder + ec2ServiceFactory func(*scope.ClusterScope) services.EC2MachineInterface + secretsManagerServiceFactory func(*scope.ClusterScope) services.SecretsManagerInterface } func (r *AWSMachineReconciler) getEC2Service(scope *scope.ClusterScope) services.EC2MachineInterface { - if r.serviceFactory != nil { - return r.serviceFactory(scope) + if r.ec2ServiceFactory != nil { + return r.ec2ServiceFactory(scope) } return ec2.NewService(scope) } +func (r *AWSMachineReconciler) getSecretsManagerService(scope *scope.ClusterScope) services.SecretsManagerInterface { + if r.secretsManagerServiceFactory != nil { + return r.secretsManagerServiceFactory(scope) + } + + return secretsmanager.NewService(scope) +} + // +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=awsmachines,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=awsmachines/status,verbs=get;update;patch // +kubebuilder:rbac:groups=cluster.x-k8s.io,resources=machines;machines/status,verbs=get;list;watch @@ -174,6 +185,11 @@ func (r *AWSMachineReconciler) reconcileDelete(machineScope *scope.MachineScope, machineScope.Info("Handling deleted AWSMachine") ec2Service := r.getEC2Service(clusterScope) + secretSvc := r.getSecretsManagerService(clusterScope) + + if err := r.deleteEncryptedBootstrapDataSecret(machineScope, secretSvc); err != nil { + return reconcile.Result{}, err + } instance, err := r.findInstance(machineScope, ec2Service) if err != nil { @@ -268,10 +284,16 @@ func (r *AWSMachineReconciler) findInstance(scope *scope.MachineScope, ec2svc se func (r *AWSMachineReconciler) reconcileNormal(ctx context.Context, machineScope *scope.MachineScope, clusterScope *scope.ClusterScope) (reconcile.Result, error) { machineScope.Info("Reconciling AWSMachine") + + secretSvc := r.getSecretsManagerService(clusterScope) + // If the AWSMachine is in an error state, return early. - if machineScope.AWSMachine.Status.FailureReason != nil || machineScope.AWSMachine.Status.FailureMessage != nil { + if machineScope.HasFailed() { + // If we are in a failed state, delete the secret regardless of instance state + if err := r.deleteEncryptedBootstrapDataSecret(machineScope, secretSvc); err != nil { + return reconcile.Result{}, err + } machineScope.Info("Error state detected, skipping reconciliation") - return reconcile.Result{}, nil } // If the AWSMachine doesn't have our finalizer, add it. @@ -298,7 +320,7 @@ func (r *AWSMachineReconciler) reconcileNormal(ctx context.Context, machineScope ec2svc := r.getEC2Service(clusterScope) // Get or create the instance. - instance, err := r.getOrCreate(machineScope, ec2svc) + instance, err := r.getOrCreate(machineScope, ec2svc, secretSvc) if err != nil { return reconcile.Result{}, err } @@ -324,7 +346,8 @@ func (r *AWSMachineReconciler) reconcileNormal(ctx context.Context, machineScope machineScope.Info("EC2 instance state changed", "state", instance.State, "instance-id", *machineScope.GetInstanceID()) } - machineScope.SetAddresses(instance.Addresses) + // TODO(vincepri): Remove this annotation when clusterctl is no longer relevant. + machineScope.SetAnnotation("cluster-api-provider-aws", "true") switch instance.State { case infrav1.InstanceStatePending, infrav1.InstanceStateStopping, infrav1.InstanceStateStopped: @@ -343,39 +366,69 @@ func (r *AWSMachineReconciler) reconcileNormal(ctx context.Context, machineScope machineScope.SetFailureMessage(errors.Errorf("EC2 instance state %q is undefined", instance.State)) } + // reconcile the deletion of the bootstrap data secret now that we have updated instance state + if err := r.deleteEncryptedBootstrapDataSecret(machineScope, secretSvc); err != nil { + return reconcile.Result{}, err + } + if instance.State == infrav1.InstanceStateTerminated { machineScope.SetFailureReason(capierrors.UpdateMachineError) machineScope.SetFailureMessage(errors.Errorf("EC2 instance state %q is unexpected", instance.State)) } - if err := r.reconcileLBAttachment(machineScope, clusterScope, instance); err != nil { - return reconcile.Result{}, errors.Errorf("failed to reconcile LB attachment: %+v", err) + // tasks that can take place during all known instance states + if machineScope.InstanceIsInKnownState() { + _, err = r.ensureTags(ec2svc, machineScope.AWSMachine, machineScope.GetInstanceID(), machineScope.AdditionalTags()) + if err != nil { + return reconcile.Result{}, errors.Errorf("failed to ensure tags: %+v", err) + } } - // TODO(vincepri): Remove this annotation when clusterctl is no longer relevant. - machineScope.SetAnnotation("cluster-api-provider-aws", "true") + // tasks that can only take place during operational instance states + if machineScope.InstanceIsOperational() { + machineScope.SetAddresses(instance.Addresses) - existingSecurityGroups, err := ec2svc.GetInstanceSecurityGroups(*machineScope.GetInstanceID()) - if err != nil { - return reconcile.Result{}, err - } + if err := r.reconcileLBAttachment(machineScope, clusterScope, instance); err != nil { + return reconcile.Result{}, errors.Errorf("failed to reconcile LB attachment: %+v", err) + } - // Ensure that the security groups are correct. - _, err = r.ensureSecurityGroups(ec2svc, machineScope, machineScope.AWSMachine.Spec.AdditionalSecurityGroups, existingSecurityGroups) - if err != nil { - return reconcile.Result{}, errors.Errorf("failed to apply security groups: %+v", err) - } + existingSecurityGroups, err := ec2svc.GetInstanceSecurityGroups(*machineScope.GetInstanceID()) + if err != nil { + return reconcile.Result{}, err + } - // Ensure that the tags are correct. - _, err = r.ensureTags(ec2svc, machineScope.AWSMachine, machineScope.GetInstanceID(), machineScope.AdditionalTags()) - if err != nil { - return reconcile.Result{}, errors.Errorf("failed to ensure tags: %+v", err) + // Ensure that the security groups are correct. + _, err = r.ensureSecurityGroups(ec2svc, machineScope, machineScope.AWSMachine.Spec.AdditionalSecurityGroups, existingSecurityGroups) + if err != nil { + return reconcile.Result{}, errors.Errorf("failed to apply security groups: %+v", err) + } } return reconcile.Result{}, nil } -func (r *AWSMachineReconciler) getOrCreate(scope *scope.MachineScope, ec2svc services.EC2MachineInterface) (*infrav1.Instance, error) { +func (r *AWSMachineReconciler) deleteEncryptedBootstrapDataSecret(machineScope *scope.MachineScope, secretSvc services.SecretsManagerInterface) error { + // do nothing if there isn't asecret + if machineScope.GetSecretARN() == "" { + return nil + } + + // Do nothing if the AWSMachine is not in a failed state, and is operational from an EC2 perspective, but does not have a node reference + if !machineScope.HasFailed() && machineScope.InstanceIsOperational() && machineScope.Machine.Status.NodeRef == nil && !machineScope.AWSMachineIsDeleted() { + return nil + } + machineScope.Info("Deleting unneeded entry from AWS Secrets Manager", "secretARN", machineScope.GetSecretARN()) + if err := secretSvc.Delete(machineScope); err != nil { + machineScope.Info("Unable to delete entry from AWS Secrets Manager containing encrypted userdata", "secretARN", machineScope.GetSecretARN()) + r.Recorder.Eventf(machineScope.AWSMachine, corev1.EventTypeWarning, "FailedDeleteEncryptedBootstrapDataSecret", "AWS Secret Manager entry containing userdata not deleted") + return err + } + r.Recorder.Eventf(machineScope.AWSMachine, corev1.EventTypeNormal, "SuccessfulDeleteEncryptedBootstrapDataSecret", "AWS Secret Manager entry containing userdata deleted") + machineScope.DeleteSecretARN() + return nil +} + +func (r *AWSMachineReconciler) getOrCreate(scope *scope.MachineScope, ec2svc services.EC2MachineInterface, secretSvc services.SecretsManagerInterface) (*infrav1.Instance, error) { instance, err := r.findInstance(scope, ec2svc) if err != nil { return nil, err @@ -384,7 +437,40 @@ func (r *AWSMachineReconciler) getOrCreate(scope *scope.MachineScope, ec2svc ser if instance == nil { scope.Info("Creating EC2 instance") // Create a new AWSMachine instance if we couldn't find a running instance. - instance, err = ec2svc.CreateInstance(scope) + + userData, err := scope.GetRawBootstrapData() + if err != nil { + r.Recorder.Eventf(scope.AWSMachine, corev1.EventTypeWarning, "FailedGetBootstrapData", err.Error()) + return nil, err + } + + if scope.UseSecretsManager() { + compressedUserData, err := userdata.GzipBytes(userData) + if err != nil { + return nil, err + } + // Do an initial check in case manager was terminated prematurely + if scope.GetSecretARN() == "" { + newSecretARN, err := secretSvc.Create(scope, compressedUserData) + if err != nil { + r.Recorder.Eventf(scope.AWSMachine, corev1.EventTypeWarning, "FailedCreateAWSSecretsManagerEntry", err.Error()) + return nil, err + } + scope.SetSecretARN(newSecretARN) + } + // Register the Secret ARN immediately to avoid orphaning AWS resources on delete + if err := scope.PatchObject(); err != nil { + return nil, err + } + encryptedCloudInit, err := secretsmanager.GenerateCloudInitMIMEDocument(scope.GetSecretARN(), scope.AWSCluster.Spec.Region) + if err != nil { + r.Recorder.Eventf(scope.AWSMachine, corev1.EventTypeWarning, "FailedGenerateAWSSecretsManagerCloudInit", err.Error()) + return nil, err + } + userData = encryptedCloudInit + } + + instance, err = ec2svc.CreateInstance(scope, userData) if err != nil { return nil, errors.Wrapf(err, "failed to create EC2 instance for AWSMachine %s/%s", scope.Namespace(), scope.Name()) } diff --git a/controllers/awsmachine_controller_test.go b/controllers/awsmachine_controller_test.go index 8aaa388465..883696ea18 100644 --- a/controllers/awsmachine_controller_test.go +++ b/controllers/awsmachine_controller_test.go @@ -27,6 +27,7 @@ import ( . "github.com/onsi/gomega" . "github.com/onsi/gomega/gstruct" "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/tools/record" @@ -50,6 +51,7 @@ var _ = Describe("AWSMachineReconciler", func() { ms *scope.MachineScope mockCtrl *gomock.Controller ec2Svc *mock_services.MockEC2MachineInterface + secretSvc *mock_services.MockSecretsManagerInterface recorder *record.FakeRecorder ) @@ -68,9 +70,18 @@ var _ = Describe("AWSMachineReconciler", func() { Spec: infrav1.AWSMachineSpec{}, } + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "bootstrap-data", + }, + Data: map[string][]byte{ + "value": []byte("shell-script"), + }, + } + ms, err = scope.NewMachineScope( scope.MachineScopeParams{ - Client: fake.NewFakeClient([]runtime.Object{awsMachine}...), + Client: fake.NewFakeClient([]runtime.Object{awsMachine, secret}...), Cluster: &clusterv1.Cluster{ Status: clusterv1.ClusterStatus{ InfrastructureReady: true, @@ -99,13 +110,18 @@ var _ = Describe("AWSMachineReconciler", func() { mockCtrl = gomock.NewController(GinkgoT()) ec2Svc = mock_services.NewMockEC2MachineInterface(mockCtrl) + secretSvc = mock_services.NewMockSecretsManagerInterface(mockCtrl) - recorder = record.NewFakeRecorder(1) + // If your test hangs for 9 minutes, increase the value here to the number of events during a reconciliation loop + recorder = record.NewFakeRecorder(2) reconciler = AWSMachineReconciler{ - serviceFactory: func(*scope.ClusterScope) services.EC2MachineInterface { + ec2ServiceFactory: func(*scope.ClusterScope) services.EC2MachineInterface { return ec2Svc }, + secretsManagerServiceFactory: func(*scope.ClusterScope) services.SecretsManagerInterface { + return secretSvc + }, Recorder: recorder, } @@ -130,8 +146,7 @@ var _ = Describe("AWSMachineReconciler", func() { buf := new(bytes.Buffer) klog.SetOutput(buf) - _, err := reconciler.reconcileNormal(context.Background(), ms, cs) - Expect(err).To(BeNil()) + _, _ = reconciler.reconcileNormal(context.Background(), ms, cs) Expect(buf).To(ContainSubstring("Error state detected, skipping reconciliation")) }) @@ -190,7 +205,8 @@ var _ = Describe("AWSMachineReconciler", func() { It("should try to create a new machine if none exists", func() { expectedErr := errors.New("Invalid instance") ec2Svc.EXPECT().InstanceIfExists(gomock.Any()).Return(nil, nil) - ec2Svc.EXPECT().CreateInstance(gomock.Any()).Return(nil, expectedErr) + secretSvc.EXPECT().Create(gomock.Any(), gomock.Any()).Return("testARN", nil).Times(1) + ec2Svc.EXPECT().CreateInstance(gomock.Any(), gomock.Any()).Return(nil, expectedErr) _, err := reconciler.reconcileNormal(context.Background(), ms, cs) Expect(errors.Cause(err)).To(MatchError(expectedErr)) @@ -203,9 +219,11 @@ var _ = Describe("AWSMachineReconciler", func() { instance = &infrav1.Instance{ ID: "myMachine", } + instance.State = infrav1.InstanceStatePending ec2Svc.EXPECT().GetRunningInstanceByTags(gomock.Any()).Return(nil, nil) - ec2Svc.EXPECT().CreateInstance(gomock.Any()).Return(instance, nil) + secretSvc.EXPECT().Create(gomock.Any(), gomock.Any()).Return("testARN", nil).Times(1) + ec2Svc.EXPECT().CreateInstance(gomock.Any(), gomock.Any()).Return(instance, nil) }) Context("instance security group errors", func() { @@ -249,12 +267,12 @@ var _ = Describe("AWSMachineReconciler", func() { It("should error when the instance state is a new unseen one", func() { buf := new(bytes.Buffer) klog.SetOutput(buf) - ec2Svc.EXPECT().GetInstanceSecurityGroups(gomock.Any()).Return(nil, errors.New("stop here")) instance.State = "NewAWSMachineState" + secretSvc.EXPECT().Delete(gomock.Any()).Return(nil).Times(1) _, _ = reconciler.reconcileNormal(context.Background(), ms, cs) Expect(ms.AWSMachine.Status.Ready).To(Equal(false)) Expect(buf.String()).To(ContainSubstring(("EC2 instance state is undefined"))) - Expect(recorder.Events).To(Receive(ContainSubstring("InstanceUnhandledState"))) + Eventually(recorder.Events).Should(Receive(ContainSubstring("InstanceUnhandledState"))) Expect(ms.AWSMachine.Status.FailureMessage).To(PointTo(Equal("EC2 instance state \"NewAWSMachineState\" is undefined"))) }) }) @@ -308,8 +326,8 @@ var _ = Describe("AWSMachineReconciler", func() { buf = new(bytes.Buffer) klog.SetOutput(buf) ec2Svc.EXPECT().GetInstanceSecurityGroups(gomock.Any()). - Return(map[string][]string{"eid": {}}, nil).AnyTimes() - ec2Svc.EXPECT().GetCoreSecurityGroups(gomock.Any()).Return([]string{}, nil).AnyTimes() + Return(map[string][]string{"eid": {}}, nil).Times(1) + ec2Svc.EXPECT().GetCoreSecurityGroups(gomock.Any()).Return([]string{}, nil).Times(1) }) It("should set instance to stopping and unready", func() { @@ -342,9 +360,7 @@ var _ = Describe("AWSMachineReconciler", func() { BeforeEach(func() { buf = new(bytes.Buffer) klog.SetOutput(buf) - ec2Svc.EXPECT().GetInstanceSecurityGroups(gomock.Any()). - Return(map[string][]string{"eid": {}}, nil).AnyTimes() - ec2Svc.EXPECT().GetCoreSecurityGroups(gomock.Any()).Return([]string{}, nil).AnyTimes() + secretSvc.EXPECT().Delete(gomock.Any()).Return(nil).Times(1) }) It("should warn if an instance is shutting-down", func() { @@ -352,7 +368,7 @@ var _ = Describe("AWSMachineReconciler", func() { _, _ = reconciler.reconcileNormal(context.Background(), ms, cs) Expect(ms.AWSMachine.Status.Ready).To(Equal(false)) Expect(buf.String()).To(ContainSubstring(("Unexpected EC2 instance termination"))) - Expect(recorder.Events).To(Receive(ContainSubstring("UnexpectedTermination"))) + Eventually(recorder.Events).Should(Receive(ContainSubstring("UnexpectedTermination"))) }) It("should error when the instance is seen as terminated", func() { @@ -360,13 +376,121 @@ var _ = Describe("AWSMachineReconciler", func() { _, _ = reconciler.reconcileNormal(context.Background(), ms, cs) Expect(ms.AWSMachine.Status.Ready).To(Equal(false)) Expect(buf.String()).To(ContainSubstring(("Unexpected EC2 instance termination"))) - Expect(recorder.Events).To(Receive(ContainSubstring("UnexpectedTermination"))) + Eventually(recorder.Events).Should(Receive(ContainSubstring("UnexpectedTermination"))) Expect(ms.AWSMachine.Status.FailureMessage).To(PointTo(Equal("EC2 instance state \"terminated\" is unexpected"))) }) + }) + }) + }) + + Context("secrets management lifecycle", func() { + var instance *infrav1.Instance + arn := "testARN" + When("creating EC2 instances", func() { + BeforeEach(func() { + ec2Svc.EXPECT().GetRunningInstanceByTags(gomock.Any()).Return(nil, nil).AnyTimes() + secretSvc.EXPECT().Create(gomock.Any(), gomock.Any()).Return(arn, nil).Times(1) + ec2Svc.EXPECT().CreateInstance(gomock.Any(), gomock.Any()).Return(instance, nil).AnyTimes() + }) + + It("should leverage AWS Secrets Manager", func() { + ms.AWSMachine.ObjectMeta.Labels = map[string]string{ + clusterv1.MachineControlPlaneLabelName: "", + } + _, _ = reconciler.reconcileNormal(context.Background(), ms, cs) + Expect(ms.AWSMachine.Spec.CloudInit.SecretARN).To(Equal(arn)) + }) + }) + + When("there's a node ref and a secret ARN", func() { + BeforeEach(func() { + instance = &infrav1.Instance{ + ID: "myMachine", + } + + ms.Machine.Status.NodeRef = &corev1.ObjectReference{ + Kind: "Node", + Name: "myMachine", + APIVersion: "v1", + } + + ms.AWSMachine.Spec.CloudInit = infrav1.CloudInit{ + SecretARN: "secret", + } + ec2Svc.EXPECT().GetRunningInstanceByTags(gomock.Any()).Return(nil, nil).AnyTimes() + ec2Svc.EXPECT().CreateInstance(gomock.Any(), gomock.Any()).Return(instance, nil).AnyTimes() + }) + + It("should delete the secret if the instance is running", func() { + instance.State = infrav1.InstanceStateRunning + ec2Svc.EXPECT().GetInstanceSecurityGroups(gomock.Any()). + Return(map[string][]string{"eid": {}}, nil).Times(1) + ec2Svc.EXPECT().GetCoreSecurityGroups(gomock.Any()).Return([]string{}, nil).Times(1) + secretSvc.EXPECT().Delete(gomock.Any()).Return(nil).Times(1) + _, _ = reconciler.reconcileNormal(context.Background(), ms, cs) + }) + + It("should delete the secret if the instance is terminated", func() { + instance.State = infrav1.InstanceStateTerminated + secretSvc.EXPECT().Delete(gomock.Any()).Return(nil).Times(1) + _, _ = reconciler.reconcileNormal(context.Background(), ms, cs) + }) + + It("should delete the secret if the AWSMachine is deleted", func() { + instance.State = infrav1.InstanceStateRunning + secretSvc.EXPECT().Delete(gomock.Any()).Return(nil).Times(1) + _, _ = reconciler.reconcileDelete(ms, cs) + }) + + It("should delete the secret if the AWSMachine is in a failure condition", func() { + ms.AWSMachine.Status.FailureReason = capierrors.MachineStatusErrorPtr(capierrors.UpdateMachineError) + secretSvc.EXPECT().Delete(gomock.Any()).Return(nil).Times(1) + _, _ = reconciler.reconcileDelete(ms, cs) + }) + + }) + + When("there's only a secret ARN and no node ref", func() { + BeforeEach(func() { + instance = &infrav1.Instance{ + ID: "myMachine", + } + ms.AWSMachine.Spec.CloudInit = infrav1.CloudInit{ + SecretARN: "secret", + } + ec2Svc.EXPECT().GetRunningInstanceByTags(gomock.Any()).Return(nil, nil).AnyTimes() + ec2Svc.EXPECT().CreateInstance(gomock.Any(), gomock.Any()).Return(instance, nil).AnyTimes() + }) + It("should not delete the secret if the instance is running", func() { + instance.State = infrav1.InstanceStateRunning + ec2Svc.EXPECT().GetInstanceSecurityGroups(gomock.Any()). + Return(map[string][]string{"eid": {}}, nil).Times(1) + ec2Svc.EXPECT().GetCoreSecurityGroups(gomock.Any()).Return([]string{}, nil).Times(1) + secretSvc.EXPECT().Delete(gomock.Any()).Return(nil).MaxTimes(0) + _, _ = reconciler.reconcileNormal(context.Background(), ms, cs) + }) + + It("should delete the secret if the instance is terminated", func() { + instance.State = infrav1.InstanceStateTerminated + secretSvc.EXPECT().Delete(gomock.Any()).Return(nil).Times(1) + _, _ = reconciler.reconcileNormal(context.Background(), ms, cs) + }) + + It("should delete the secret if the AWSMachine is deleted", func() { + instance.State = infrav1.InstanceStateRunning + secretSvc.EXPECT().Delete(gomock.Any()).Return(nil).Times(1) + _, _ = reconciler.reconcileDelete(ms, cs) + }) + + It("should delete the secret if the AWSMachine is in a failure condition", func() { + ms.AWSMachine.Status.FailureReason = capierrors.MachineStatusErrorPtr(capierrors.UpdateMachineError) + secretSvc.EXPECT().Delete(gomock.Any()).Return(nil).Times(1) + _, _ = reconciler.reconcileDelete(ms, cs) }) }) + }) Context("deleting an AWSMachine", func() { @@ -396,7 +520,7 @@ var _ = Describe("AWSMachineReconciler", func() { Expect(err).To(BeNil()) Expect(buf.String()).To(ContainSubstring("Unable to locate EC2 instance by ID or tags")) Expect(ms.AWSMachine.Finalizers).To(ConsistOf(metav1.FinalizerDeleteDependents)) - Expect(recorder.Events).To(Receive(ContainSubstring("NoInstanceFound"))) + Eventually(recorder.Events).Should(Receive(ContainSubstring("NoInstanceFound"))) }) It("should ignore instances in shutting down state", func() { @@ -444,7 +568,7 @@ var _ = Describe("AWSMachineReconciler", func() { _, err := reconciler.reconcileDelete(ms, cs) Expect(errors.Cause(err)).To(MatchError(expected)) Expect(buf.String()).To(ContainSubstring("Terminating EC2 instance")) - Expect(recorder.Events).To(Receive(ContainSubstring("FailedTerminate"))) + Eventually(recorder.Events).Should(Receive(ContainSubstring("FailedTerminate"))) }) When("instance can be shut down", func() { diff --git a/docs/README.md b/docs/README.md index 937813f9e5..a918c8acf5 100644 --- a/docs/README.md +++ b/docs/README.md @@ -6,9 +6,11 @@ - [List of AMIs](amis.md) - [Accessing cluster instances](accessing-instances.md) - [Building AMIs with Packer](https://github.com/kubernetes-sigs/image-builder/tree/master/images/capi#make-targets) +- [Userdata Privacy](userdata-privacy.md) ## Special use cases - [Reconcile Cluster-API objects in a restricted namespace](reconcile-in-custom-namespace.md) +- [Creating clusters using cross account role assumption using KIAM](roleassumption.md) ## Project Documentation diff --git a/docs/userdata-privacy.md b/docs/userdata-privacy.md new file mode 100644 index 0000000000..d470a7b605 --- /dev/null +++ b/docs/userdata-privacy.md @@ -0,0 +1,30 @@ +# Userdata Privacy + +Cluster API Provider AWS bootstraps EC2 instances to create and join Kubernetes clusters using instance user data. +Because Kubernetes clusters are secured using TLS using multiple Certificate Authorities, these are generated by +Cluster API and injected into the user data. It is important to note that without the configuring of host firewalls, processes can +retrieve instance userdata from http://169.254.169.254/latest/api/token + +## How Cluster API secures TLS secrets + +In 0.5.x/v1alpha3, by default, Cluster API Provider AWS will use [AWS Secrets Manager](https://aws.amazon.com/secrets-manager/) +to store the actual userdata in encrypted form. The normal unencrypted userdata uses a boot script to download the encrypted userdata secret +using instance profile permissions, immediately deletes it from AWS Secrets Manager, and then executes it. + +To avoid guessing keys in the AWS Secrets Manager key-value store and to prevent collisions, the key is an encoding the +Kubernetes namespace, cluster name and instance name, with a random string appended, providing ~256-bits of entropy. + +Cluster API Provider AWS also stores the secret ARN in the AWSMachine spec, and will delete the secret if it isn't already deleted and +the machine has registered successfully against the workload cluster API server as a node. +Cluster API Provider AWS will also attempt deletion of the secret if the AWSMachine is otherwise deleted or the EC2 instance +is terminated or failed. + +This method is only compatible with operating systems and distributions using +[cloud-init](https://cloudinit.readthedocs.io/en/latest/topics/format.html#mime-multi-part-archive). If you are using a different bootstrap +process, you will need to co-ordinate this externally and set the following in the specification of the AWSMachine types to disable the use +of a cloud-init boothook: + +``` yaml +cloudInit: + disableUserdataPrivacy: true +``` diff --git a/go.mod b/go.mod index dd6d6810da..05099b9461 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.13 require ( github.com/aws/aws-sdk-go v1.25.16 - github.com/awslabs/goformation/v3 v3.0.0 + github.com/awslabs/goformation/v4 v4.1.0 github.com/go-logr/logr v0.1.0 github.com/golang/mock v1.3.1 github.com/onsi/ginkgo v1.11.0 @@ -12,9 +12,10 @@ require ( github.com/pkg/errors v0.8.1 github.com/spf13/cobra v0.0.5 github.com/spf13/pflag v1.0.5 - golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392 + golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 golang.org/x/net v0.0.0-20191021144547-ec77196f6094 - gopkg.in/yaml.v2 v2.2.4 + golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898 // indirect + gopkg.in/yaml.v2 v2.2.7 k8s.io/api v0.0.0-20191121015604-11707872ac1c k8s.io/apimachinery v0.0.0-20191121015412-41065c7a8c2a k8s.io/client-go v0.0.0-20190918160344-1fbdaa4c8d90 diff --git a/go.sum b/go.sum index 3fe94fedcc..01dace1e27 100644 --- a/go.sum +++ b/go.sum @@ -25,8 +25,10 @@ github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:l github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aws/aws-sdk-go v1.25.16 h1:k7Fy6T/uNuLX6zuayU/TJoP7yMgGcJSkZpF7QVjwYpA= github.com/aws/aws-sdk-go v1.25.16/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/awslabs/goformation/v3 v3.0.0 h1:Z5b6t3mVZHpAP195p9LmiLS6kqrOB1DKhnzPyKa73jo= -github.com/awslabs/goformation/v3 v3.0.0/go.mod h1:NWYxOJpRoZtm4np627sv1nToNTbiI9p5bb5wb0qO0Aw= +github.com/awslabs/goformation/v3 v3.1.0/go.mod h1:hQ5RXo3GNm2laHWKizDzU5DsDy+yNcenSca2UxN0850= +github.com/awslabs/goformation/v4 v4.1.0 h1:JRxIW0IjhYpYDrIZOTJGMu2azXKI+OK5dP56ubpywGU= +github.com/awslabs/goformation/v4 v4.1.0/go.mod h1:MBDN7u1lMNDoehbFuO4uPvgwPeolTMA2TzX1yO6KlxI= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -150,6 +152,7 @@ github.com/grpc-ecosystem/go-grpc-middleware v0.0.0-20190222133341-cfaf5686ec79/ github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.3.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.3 h1:YPkqC67at8FYaadspW/6uE0COsBxS2656RLEr8Bppgk= github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= @@ -240,6 +243,7 @@ github.com/prometheus/common v0.6.0 h1:kRhiuYSXR3+uv2IbVbZhUxK5zVD/2pp3Gd2PpvPkp github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNGfs= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/prometheus/procfs v0.0.5 h1:3+auTFlqw+ZaQYJARz6ArODtkaIwtvBTx3N2NehQlL8= @@ -251,6 +255,7 @@ github.com/sanathkr/go-yaml v0.0.0-20170819195128-ed9d249f429b/go.mod h1:8458kAa github.com/sanathkr/yaml v0.0.0-20170819201035-0056894fa522 h1:fOCp11H0yuyAt2wqlbJtbyPzSgaxHTv8uN1pMpkG1t8= github.com/sanathkr/yaml v0.0.0-20170819201035-0056894fa522/go.mod h1:tQTYKOQgxoH3v6dEmdHiz4JG+nbxWwM5fgPQUpSZqVQ= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/soheilhy/cmux v0.1.3/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= @@ -266,6 +271,7 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= @@ -302,6 +308,8 @@ golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392 h1:ACG4HJsFiNMf47Y4PeRoebLNy/2lXT9EtprMuTFWt1M= golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190312203227-4b39c73a6495/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -326,6 +334,7 @@ golang.org/x/net v0.0.0-20190320064053-1272bf9dcd53/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190812203447-cdfb69ac37fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20191004110552-13f9640d40b9 h1:rjwSpXsdiK0dV8/Naq3kAw9ymfAeJIyd0upUIElB+lI= golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191021144547-ec77196f6094 h1:5O4U9trLjNpuhpynaDsqwCk+Tw6seqJz1EbqbnzHrc8= golang.org/x/net v0.0.0-20191021144547-ec77196f6094/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -354,6 +363,7 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456 h1:ng0gs1AKnRRuEMZoTLLlbOd+C17zUDepwGQBb/n+JVg= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191008105621-543471e840be h1:QAcqgptGM8IQBC9K/RC4o+O9YmqEm0diQn9QmZw/0mU= @@ -379,9 +389,12 @@ golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac h1:MQEvx39qSf8vyrx3XRaOe+j1UDIzKwkYOVObRgGPVqI= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 h1:9zdDQZ7Thm29KFXgAX/+yaf3eVbP7djjWp/dXAppNCc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898 h1:/atklqdjdhuosWIl6AIbOeHJjicWYPqR9bpxqxYG2pA= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gomodules.xyz/jsonpatch/v2 v2.0.1 h1:xyiBuvkD2g5n7cYzx6u2sxQvsAy4QJsZFCzGVdzOXZ0= gomodules.xyz/jsonpatch/v2 v2.0.1/go.mod h1:IhYNNY4jnS53ZnfE4PAmpKtDpTCj1JFXc+3mwe7XcUU= gonum.org/v1/gonum v0.0.0-20190331200053-3d26580ed485/go.mod h1:2ltnJ7xHfj0zHS40VVPYEAAMTa3ZGguvHGBSJeRWqE0= @@ -417,6 +430,8 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.7 h1:VUgggvou5XRW9mHwD/yXxIYSMtY0zoKQf/v226p2nyo= +gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -435,6 +450,7 @@ k8s.io/client-go v0.0.0-20190918160344-1fbdaa4c8d90 h1:mLmhKUm1X+pXu0zXMEzNsOF5E k8s.io/client-go v0.0.0-20190918160344-1fbdaa4c8d90/go.mod h1:J69/JveO6XESwVgG53q3Uz5OSfgsv4uxpScmmyYOOlk= k8s.io/cluster-bootstrap v0.0.0-20190516232516-d7d78ab2cfe7 h1:5wvjieVoU4oovHlkeD256q2M2YYi2P01zk6wxSR2zk0= k8s.io/cluster-bootstrap v0.0.0-20190516232516-d7d78ab2cfe7/go.mod h1:iBSm2nwo3OaiuW8VDvc3ySDXK5SKfUrxwPvBloKG7zg= +k8s.io/code-generator v0.0.0-20190912054826-cd179ad6a269 h1:d8Fm55A+7HOczX58+x9x+nJnJ1Devt1aCrWVIPaw/Vg= k8s.io/code-generator v0.0.0-20190912054826-cd179ad6a269/go.mod h1:V5BD6M4CyaN5m+VthcclXWsVcT1Hu+glwa1bi3MIsyE= k8s.io/component-base v0.0.0-20190918160511-547f6c5d7090/go.mod h1:933PBGtQFJky3TEwYx4aEPZ4IxqhWh3R6DCmzqIn1hA= k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= diff --git a/pkg/cloud/converters/tags.go b/pkg/cloud/converters/tags.go index 16b56a0ad6..5a91c930b2 100644 --- a/pkg/cloud/converters/tags.go +++ b/pkg/cloud/converters/tags.go @@ -20,6 +20,7 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" "github.com/aws/aws-sdk-go/service/elb" + "github.com/aws/aws-sdk-go/service/secretsmanager" infrav1 "sigs.k8s.io/cluster-api-provider-aws/api/v1alpha3" ) @@ -76,3 +77,19 @@ func MapToELBTags(src infrav1.Tags) []*elb.Tag { return tags } + +// MapToSecretsManagerTags converts a infrav1.Tags to a []*secretsmanager.Tag +func MapToSecretsManagerTags(src infrav1.Tags) []*secretsmanager.Tag { + tags := make([]*secretsmanager.Tag, 0, len(src)) + + for k, v := range src { + tag := &secretsmanager.Tag{ + Key: aws.String(k), + Value: aws.String(v), + } + + tags = append(tags, tag) + } + + return tags +} diff --git a/pkg/cloud/scope/clients.go b/pkg/cloud/scope/clients.go index 973857f856..ca4c806f61 100644 --- a/pkg/cloud/scope/clients.go +++ b/pkg/cloud/scope/clients.go @@ -20,11 +20,13 @@ import ( "github.com/aws/aws-sdk-go/service/ec2/ec2iface" "github.com/aws/aws-sdk-go/service/elb/elbiface" "github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi/resourcegroupstaggingapiiface" + "github.com/aws/aws-sdk-go/service/secretsmanager/secretsmanageriface" ) // AWSClients contains all the aws clients used by the scopes. type AWSClients struct { EC2 ec2iface.EC2API ELB elbiface.ELBAPI + SecretsManager secretsmanageriface.SecretsManagerAPI ResourceTagging resourcegroupstaggingapiiface.ResourceGroupsTaggingAPIAPI } diff --git a/pkg/cloud/scope/cluster.go b/pkg/cloud/scope/cluster.go index 9157c5aac3..b56da590fd 100644 --- a/pkg/cloud/scope/cluster.go +++ b/pkg/cloud/scope/cluster.go @@ -25,6 +25,7 @@ import ( "github.com/aws/aws-sdk-go/service/ec2" "github.com/aws/aws-sdk-go/service/elb" "github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi" + "github.com/aws/aws-sdk-go/service/secretsmanager" "github.com/go-logr/logr" "github.com/pkg/errors" "k8s.io/apimachinery/pkg/runtime" @@ -91,6 +92,12 @@ func NewClusterScope(params ClusterScopeParams) (*ClusterScope, error) { params.AWSClients.ResourceTagging = resourceTagging } + if params.AWSClients.SecretsManager == nil { + sClient := secretsmanager.New(session) + sClient.Handlers.Complete.PushBack(recordAWSPermissionsIssue(params.AWSCluster)) + params.AWSClients.SecretsManager = sClient + } + helper, err := patch.NewHelper(params.AWSCluster, params.Client) if err != nil { return nil, errors.Wrap(err, "failed to init patch helper") diff --git a/pkg/cloud/scope/machine.go b/pkg/cloud/scope/machine.go index 565c0e7dfd..d473a1a52c 100644 --- a/pkg/cloud/scope/machine.go +++ b/pkg/cloud/scope/machine.go @@ -180,29 +180,62 @@ func (m *MachineScope) SetAnnotation(key, value string) { m.AWSMachine.Annotations[key] = value } +// UseSecretsManager returns the computed value of whether or not +// userdata should be stored using AWS Secrets Manager. +func (m *MachineScope) UseSecretsManager() bool { + return !m.AWSMachine.Spec.CloudInit.InsecureSkipSecretsManager +} + +// GetSecretARN returns the Amazon Resource Name for the secret belonging +// to the AWSMachine in AWS Secrets Manager +func (m *MachineScope) GetSecretARN() string { + return m.AWSMachine.Spec.CloudInit.SecretARN +} + +// SetSecretARN sets the Amazon Resource Name for the secret belonging +// to the AWSMachine in AWS Secrets Manager +func (m *MachineScope) SetSecretARN(value string) { + m.AWSMachine.Spec.CloudInit.SecretARN = value +} + +// DeleteSecretARN deletes the AMazon Resource Name for the secret belonging +// to the AWSMachine in AWS Secrets Manager +func (m *MachineScope) DeleteSecretARN() { + m.AWSMachine.Spec.CloudInit.SecretARN = "" +} + // SetAddresses sets the AWSMachine address status. func (m *MachineScope) SetAddresses(addrs []corev1.NodeAddress) { m.AWSMachine.Status.Addresses = addrs } -// GetBootstrapData returns the bootstrap data from the secret in the Machine's bootstrap.dataSecretName. +// GetBootstrapData returns the bootstrap data from the secret in the Machine's bootstrap.dataSecretName as base64. func (m *MachineScope) GetBootstrapData() (string, error) { + value, err := m.GetRawBootstrapData() + if err != nil { + return "", err + } + return base64.StdEncoding.EncodeToString(value), nil +} + +// GetRawBootstrapData returns the bootstrap data from the secret in the Machine's bootstrap.dataSecretName. +func (m *MachineScope) GetRawBootstrapData() ([]byte, error) { if m.Machine.Spec.Bootstrap.DataSecretName == nil { - return "", errors.New("error retrieving bootstrap data: linked Machine's bootstrap.dataSecretName is nil") + return nil, errors.New("error retrieving bootstrap data: linked Machine's bootstrap.dataSecretName is nil") } secret := &corev1.Secret{} key := types.NamespacedName{Namespace: m.Namespace(), Name: *m.Machine.Spec.Bootstrap.DataSecretName} if err := m.client.Get(context.TODO(), key, secret); err != nil { - return "", errors.Wrapf(err, "failed to retrieve bootstrap data secret for AWSMachine %s/%s", m.Namespace(), m.Name()) + return nil, errors.Wrapf(err, "failed to retrieve bootstrap data secret for AWSMachine %s/%s", m.Namespace(), m.Name()) } value, ok := secret.Data["value"] if !ok { - return "", errors.New("error retrieving bootstrap data: secret value key is missing") + return nil, errors.New("error retrieving bootstrap data: secret value key is missing") } - return base64.StdEncoding.EncodeToString(value), nil + return value, nil } // PatchObject persists the machine spec and status. @@ -227,3 +260,21 @@ func (m *MachineScope) AdditionalTags() infrav1.Tags { return tags } + +func (m *MachineScope) HasFailed() bool { + return m.AWSMachine.Status.FailureReason != nil || m.AWSMachine.Status.FailureMessage != nil +} + +func (m *MachineScope) InstanceIsOperational() bool { + state := m.GetInstanceState() + return state != nil && infrav1.InstanceOperationalStates.Has(string(*state)) +} + +func (m *MachineScope) InstanceIsInKnownState() bool { + state := m.GetInstanceState() + return state != nil && infrav1.InstanceKnownStates.Has(string(*state)) +} + +func (m *MachineScope) AWSMachineIsDeleted() bool { + return !m.AWSMachine.ObjectMeta.DeletionTimestamp.IsZero() +} diff --git a/pkg/cloud/scope/machine_test.go b/pkg/cloud/scope/machine_test.go new file mode 100644 index 0000000000..2e3d69460b --- /dev/null +++ b/pkg/cloud/scope/machine_test.go @@ -0,0 +1,204 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package scope + +import ( + "encoding/base64" + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/utils/pointer" + infrav1 "sigs.k8s.io/cluster-api-provider-aws/api/v1alpha3" + clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func setupScheme() (*runtime.Scheme, error) { + scheme := runtime.NewScheme() + if err := infrav1.AddToScheme(scheme); err != nil { + return nil, err + } + if err := clusterv1.AddToScheme(scheme); err != nil { + return nil, err + } + if err := corev1.AddToScheme(scheme); err != nil { + return nil, err + } + return scheme, nil +} + +func newMachine(clusterName, machineName string) *clusterv1.Machine { + return &clusterv1.Machine{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + clusterv1.ClusterLabelName: clusterName, + }, + ClusterName: clusterName, + Name: machineName, + Namespace: "default", + }, + Spec: clusterv1.MachineSpec{ + Bootstrap: clusterv1.Bootstrap{ + DataSecretName: pointer.StringPtr(machineName), + }, + }, + } +} + +func newCluster(name string) *clusterv1.Cluster { + return &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: "default", + }, + } +} + +func newAWSCluster(name string) *infrav1.AWSCluster { + return &infrav1.AWSCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: "default", + }, + } +} + +func newAWSMachine(clusterName, machineName string) *infrav1.AWSMachine { + return &infrav1.AWSMachine{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + clusterv1.ClusterLabelName: clusterName, + }, + Name: machineName, + Namespace: "default", + }, + } +} + +func newBootstrapSecret(clusterName, machineName string) *corev1.Secret { + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + clusterv1.ClusterLabelName: clusterName, + }, + Name: machineName, + Namespace: "default", + }, + Data: map[string][]byte{ + "value": []byte("user data"), + }, + } +} + +func setupMachineScope() (*MachineScope, error) { + scheme, err := setupScheme() + if err != nil { + return nil, err + } + clusterName := "my-cluster" + cluster := newCluster(clusterName) + machine := newMachine(clusterName, "my-machine-0") + secret := newBootstrapSecret(clusterName, "my-machine-0") + awsMachine := newAWSMachine(clusterName, "my-machine-0") + awsCluster := newAWSCluster(clusterName) + + initObjects := []runtime.Object{ + cluster, machine, secret, awsMachine, awsCluster, + } + + client := fake.NewFakeClientWithScheme(scheme, initObjects...) + return NewMachineScope( + MachineScopeParams{ + Client: client, + Machine: machine, + Cluster: cluster, + AWSCluster: awsCluster, + AWSMachine: awsMachine, + }, + ) +} + +func TestGetBootstrapDataIsBase64Encoded(t *testing.T) { + scope, err := setupMachineScope() + if err != nil { + t.Fatal(err) + } + + userdata, err := scope.GetBootstrapData() + if err != nil { + t.Fatal(err) + } + _, err = base64.StdEncoding.DecodeString(userdata) + if err != nil { + t.Fatalf("GetBootstrapData isn't base 64 encoded: %+v", err) + } +} + +func TestGetRawBootstrapDataIsNotBase64Encoded(t *testing.T) { + scope, err := setupMachineScope() + if err != nil { + t.Fatal(err) + } + + userdata, err := scope.GetRawBootstrapData() + if err != nil { + t.Fatal(err) + } + _, err = base64.StdEncoding.DecodeString(string(userdata)) + if err == nil { + t.Fatalf("GetBootstrapData is base 64 encoded: %+v", err) + } +} + +func TestUseSecretsManagerTrue(t *testing.T) { + scope, err := setupMachineScope() + if err != nil { + t.Fatal(err) + } + + if !scope.UseSecretsManager() { + t.Fatalf("UseSecretsManager should be true") + } +} + +func TestGetSecretARNDefaultIsNil(t *testing.T) { + scope, err := setupMachineScope() + if err != nil { + t.Fatal(err) + } + + if scope.GetSecretARN() != "" { + t.Fatalf("GetSecretARN should be empty string") + } +} + +func TestSetSecretARN(t *testing.T) { + secretARN := "secretARN" + scope, err := setupMachineScope() + if err != nil { + t.Fatal(err) + } + + scope.SetSecretARN(secretARN) + val := scope.GetSecretARN() + + if val != secretARN { + t.Fatalf("GetSecretARN does not equal %s: %s", secretARN, val) + } +} diff --git a/pkg/cloud/services/cloudformation/bootstrap.go b/pkg/cloud/services/cloudformation/bootstrap.go index ebd29b9e67..8502579f3b 100644 --- a/pkg/cloud/services/cloudformation/bootstrap.go +++ b/pkg/cloud/services/cloudformation/bootstrap.go @@ -21,8 +21,8 @@ import ( "io/ioutil" "path" - "github.com/awslabs/goformation/v3/cloudformation" - cfn_iam "github.com/awslabs/goformation/v3/cloudformation/iam" + "github.com/awslabs/goformation/v4/cloudformation" + cfn_iam "github.com/awslabs/goformation/v4/cloudformation/iam" "github.com/pkg/errors" "k8s.io/klog" @@ -69,7 +69,7 @@ func BootstrapTemplate(accountID, partition string, extraControlPlanePolicies, e template.Resources[NodePolicy] = &cfn_iam.ManagedPolicy{ ManagedPolicyName: iam.NewManagedName("nodes"), Description: `For the Kubernetes Cloud Provider AWS nodes`, - PolicyDocument: cloudProviderNodeAwsPolicy(), + PolicyDocument: nodePolicy(), Roles: []string{ cloudformation.Ref("AWSIAMRoleControlPlane"), cloudformation.Ref("AWSIAMRoleNodes"), @@ -233,10 +233,37 @@ func controllersPolicy(accountID, partition string) *iam.PolicyDocument { "iam:PassRole", }, }, + { + Effect: iam.EffectAllow, + Resource: iam.Resources{"arn:aws:secretsmanager:*:*:secret:aws.cluster.x-k8s.io/*"}, + Action: iam.Actions{ + "secretsmanager:CreateSecret", + "secretsmanager:DeleteSecret", + }, + }, }, } } +func bootstrapSecretPolicy() iam.StatementEntry { + return iam.StatementEntry{ + Effect: iam.EffectAllow, + Resource: iam.Resources{"arn:aws:secretsmanager:*:*:secret:aws.cluster.x-k8s.io/*"}, + Action: iam.Actions{ + "secretsmanager:DeleteSecret", + "secretsmanager:GetSecretValue", + }, + } +} + +func nodePolicy() *iam.PolicyDocument { + policyDocument := cloudProviderNodeAwsPolicy() + policyDocument.Statement = append( + policyDocument.Statement, bootstrapSecretPolicy(), + ) + return policyDocument +} + // From https://github.com/kubernetes/cloud-provider-aws func cloudProviderControlPlaneAwsPolicy() *iam.PolicyDocument { return &iam.PolicyDocument{ diff --git a/pkg/cloud/services/ec2/instances.go b/pkg/cloud/services/ec2/instances.go index dc7c5a2fb7..8ceb61fd37 100644 --- a/pkg/cloud/services/ec2/instances.go +++ b/pkg/cloud/services/ec2/instances.go @@ -17,8 +17,6 @@ limitations under the License. package ec2 import ( - "bytes" - "compress/gzip" "context" "encoding/base64" "strings" @@ -36,6 +34,7 @@ import ( "sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/converters" "sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/filter" "sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/scope" + "sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/services/userdata" "sigs.k8s.io/cluster-api-provider-aws/pkg/record" capierrors "sigs.k8s.io/cluster-api/errors" "sigs.k8s.io/cluster-api/util" @@ -103,7 +102,7 @@ func (s *Service) InstanceIfExists(id *string) (*infrav1.Instance, error) { } // CreateInstance runs an ec2 instance. -func (s *Service) CreateInstance(scope *scope.MachineScope) (*infrav1.Instance, error) { +func (s *Service) CreateInstance(scope *scope.MachineScope, userData []byte) (*infrav1.Instance, error) { s.scope.V(2).Info("Creating an instance for a machine") input := &infrav1.Instance{ @@ -193,13 +192,12 @@ func (s *Service) CreateInstance(scope *scope.MachineScope) (*infrav1.Instance, ) } - // Set userdata. - userData, err := scope.GetBootstrapData() + compressedUserData, err := userdata.GzipBytes(userData) if err != nil { - record.Warnf(scope.AWSMachine, corev1.EventTypeWarning, "FailedGetBootstrapData", err.Error()) - return nil, err + return nil, errors.New("failed to gzip userdata") } - input.UserData = pointer.StringPtr(userData) + + input.UserData = pointer.StringPtr(base64.StdEncoding.EncodeToString(compressedUserData)) // Set security groups. ids, err := s.GetCoreSecurityGroups(scope) @@ -313,29 +311,10 @@ func (s *Service) runInstance(role string, i *infrav1.Instance) (*infrav1.Instan EbsOptimized: i.EBSOptimized, MaxCount: aws.Int64(1), MinCount: aws.Int64(1), + UserData: i.UserData, } - if i.UserData != nil { - var buf bytes.Buffer - - decoded, err := base64.StdEncoding.DecodeString(*i.UserData) - if err != nil { - return nil, errors.Wrap(err, "failed to decode bootstrapData") - } - - gz := gzip.NewWriter(&buf) - if _, err := gz.Write(decoded); err != nil { - return nil, errors.Wrap(err, "failed to gzip userdata") - } - - if err := gz.Close(); err != nil { - return nil, errors.Wrap(err, "failed to gzip userdata") - } - - s.scope.V(2).Info("userData size", "bytes", buf.Len(), "role", role) - - input.UserData = aws.String(base64.StdEncoding.EncodeToString(buf.Bytes())) - } + s.scope.V(2).Info("userData size", "bytes", len(*i.UserData), "role", role) if len(i.NetworkInterfaces) > 0 { netInterfaces := make([]*ec2.InstanceNetworkInterfaceSpecification, 0, len(i.NetworkInterfaces)) diff --git a/pkg/cloud/services/ec2/instances_test.go b/pkg/cloud/services/ec2/instances_test.go index e0781c3881..d1e200c7e1 100644 --- a/pkg/cloud/services/ec2/instances_test.go +++ b/pkg/cloud/services/ec2/instances_test.go @@ -938,7 +938,7 @@ func TestCreateInstance(t *testing.T) { } s := NewService(clusterScope) - instance, err := s.CreateInstance(machineScope) + instance, err := s.CreateInstance(machineScope, []byte("userData")) tc.check(instance, err) }) } diff --git a/pkg/cloud/services/interfaces.go b/pkg/cloud/services/interfaces.go index 37312fa1c7..90d4b0f642 100644 --- a/pkg/cloud/services/interfaces.go +++ b/pkg/cloud/services/interfaces.go @@ -26,7 +26,7 @@ import ( type EC2MachineInterface interface { InstanceIfExists(id *string) (*infrav1.Instance, error) TerminateInstance(id string) error - CreateInstance(scope *scope.MachineScope) (*infrav1.Instance, error) + CreateInstance(scope *scope.MachineScope, userData []byte) (*infrav1.Instance, error) GetRunningInstanceByTags(scope *scope.MachineScope) (*infrav1.Instance, error) GetCoreSecurityGroups(machine *scope.MachineScope) ([]string, error) @@ -37,3 +37,10 @@ type EC2MachineInterface interface { TerminateInstanceAndWait(instanceID string) error DetachSecurityGroupsFromNetworkInterface(groups []string, interfaceID string) error } + +// SecretsManagerInterface encapsulated the methods exposed to the +// machine actuator +type SecretsManagerInterface interface { + Delete(m *scope.MachineScope) error + Create(m *scope.MachineScope, data []byte) (string, error) +} diff --git a/pkg/cloud/services/mock_services/doc.go b/pkg/cloud/services/mock_services/doc.go index 538b13b102..8852e16aa6 100644 --- a/pkg/cloud/services/mock_services/doc.go +++ b/pkg/cloud/services/mock_services/doc.go @@ -17,4 +17,6 @@ limitations under the License. // Run go generate to regenerate this mock. //go:generate ../../../../hack/tools/bin/mockgen -destination ec2_machine_interface_mock.go -package mock_services sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/services EC2MachineInterface //go:generate /usr/bin/env bash -c "cat ../../../../hack/boilerplate/boilerplate.generatego.txt ec2_machine_interface_mock.go > _ec2_machine_interface_mock.go && mv _ec2_machine_interface_mock.go ec2_machine_interface_mock.go" +//go:generate ../../../../hack/tools/bin/mockgen -destination secretsmanager_machine_interface_mock.go -package mock_services sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/services SecretsManagerInterface +//go:generate /usr/bin/env bash -c "cat ../../../../hack/boilerplate/boilerplate.generatego.txt secretsmanager_machine_interface_mock.go > _secretsmanager_machine_interface_mock.go && mv _secretsmanager_machine_interface_mock.go secretsmanager_machine_interface_mock.go" package mock_services //nolint diff --git a/pkg/cloud/services/mock_services/ec2_machine_interface_mock.go b/pkg/cloud/services/mock_services/ec2_machine_interface_mock.go index 6889d55401..0c58184154 100644 --- a/pkg/cloud/services/mock_services/ec2_machine_interface_mock.go +++ b/pkg/cloud/services/mock_services/ec2_machine_interface_mock.go @@ -51,18 +51,18 @@ func (m *MockEC2MachineInterface) EXPECT() *MockEC2MachineInterfaceMockRecorder } // CreateInstance mocks base method -func (m *MockEC2MachineInterface) CreateInstance(arg0 *scope.MachineScope) (*v1alpha3.Instance, error) { +func (m *MockEC2MachineInterface) CreateInstance(arg0 *scope.MachineScope, arg1 []byte) (*v1alpha3.Instance, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CreateInstance", arg0) + ret := m.ctrl.Call(m, "CreateInstance", arg0, arg1) ret0, _ := ret[0].(*v1alpha3.Instance) ret1, _ := ret[1].(error) return ret0, ret1 } // CreateInstance indicates an expected call of CreateInstance -func (mr *MockEC2MachineInterfaceMockRecorder) CreateInstance(arg0 interface{}) *gomock.Call { +func (mr *MockEC2MachineInterfaceMockRecorder) CreateInstance(arg0, arg1 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateInstance", reflect.TypeOf((*MockEC2MachineInterface)(nil).CreateInstance), arg0) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateInstance", reflect.TypeOf((*MockEC2MachineInterface)(nil).CreateInstance), arg0, arg1) } // DetachSecurityGroupsFromNetworkInterface mocks base method diff --git a/pkg/cloud/services/mock_services/secretsmanager_machine_interface_mock.go b/pkg/cloud/services/mock_services/secretsmanager_machine_interface_mock.go new file mode 100644 index 0000000000..488bf7014b --- /dev/null +++ b/pkg/cloud/services/mock_services/secretsmanager_machine_interface_mock.go @@ -0,0 +1,79 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by MockGen. DO NOT EDIT. +// Source: sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/services (interfaces: SecretsManagerInterface) + +// Package mock_services is a generated GoMock package. +package mock_services + +import ( + gomock "github.com/golang/mock/gomock" + reflect "reflect" + scope "sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/scope" +) + +// MockSecretsManagerInterface is a mock of SecretsManagerInterface interface +type MockSecretsManagerInterface struct { + ctrl *gomock.Controller + recorder *MockSecretsManagerInterfaceMockRecorder +} + +// MockSecretsManagerInterfaceMockRecorder is the mock recorder for MockSecretsManagerInterface +type MockSecretsManagerInterfaceMockRecorder struct { + mock *MockSecretsManagerInterface +} + +// NewMockSecretsManagerInterface creates a new mock instance +func NewMockSecretsManagerInterface(ctrl *gomock.Controller) *MockSecretsManagerInterface { + mock := &MockSecretsManagerInterface{ctrl: ctrl} + mock.recorder = &MockSecretsManagerInterfaceMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockSecretsManagerInterface) EXPECT() *MockSecretsManagerInterfaceMockRecorder { + return m.recorder +} + +// Create mocks base method +func (m *MockSecretsManagerInterface) Create(arg0 *scope.MachineScope, arg1 []byte) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Create", arg0, arg1) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Create indicates an expected call of Create +func (mr *MockSecretsManagerInterfaceMockRecorder) Create(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Create", reflect.TypeOf((*MockSecretsManagerInterface)(nil).Create), arg0, arg1) +} + +// Delete mocks base method +func (m *MockSecretsManagerInterface) Delete(arg0 *scope.MachineScope) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Delete", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// Delete indicates an expected call of Delete +func (mr *MockSecretsManagerInterfaceMockRecorder) Delete(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockSecretsManagerInterface)(nil).Delete), arg0) +} diff --git a/pkg/cloud/services/secretsmanager/cloudinit.go b/pkg/cloud/services/secretsmanager/cloudinit.go new file mode 100644 index 0000000000..d355fa7430 --- /dev/null +++ b/pkg/cloud/services/secretsmanager/cloudinit.go @@ -0,0 +1,94 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package secretsmanager + +import ( + "bytes" + "fmt" + "html/template" + "mime/multipart" + "net/textproto" + "strings" +) + +const ( + includePart = "file:///etc/secret-userdata.txt\n" +) + +var ( + includeType = textproto.MIMEHeader{ + "content-type": {"text/x-include-url"}, + } + + boothookType = textproto.MIMEHeader{ + "content-type": {"text/cloud-boothook"}, + } + + multipartHeader = strings.Join([]string{ + "MIME-Version: 1.0", + "Content-Type: multipart/mixed; boundary=\"%s\"", + "\n", + }, "\n") + + secretFetchTemplate = template.Must(template.New("secret-fetch-script").Parse(secretFetchScript)) +) + +type scriptVariables struct { + SecretID string + Region string +} + +// GenerateCloudInitMIMEDocument creates a multi-part MIME document including a script boothook to +// download userdata from AWS Secrets Manager and then restart cloud-init, and an include part +// specifying the on disk location of the new userdata +func GenerateCloudInitMIMEDocument(secretID, region string) ([]byte, error) { + var buf bytes.Buffer + mpWriter := multipart.NewWriter(&buf) + buf.WriteString(fmt.Sprintf(multipartHeader, mpWriter.Boundary())) + scriptWriter, err := mpWriter.CreatePart(boothookType) + if err != nil { + return []byte{}, err + } + + scriptVariables := scriptVariables{ + SecretID: secretID, + Region: region, + } + + var scriptBuf bytes.Buffer + secretFetchTemplate.Execute(&scriptBuf, scriptVariables) + _, err = scriptWriter.Write(scriptBuf.Bytes()) + if err != nil { + return []byte{}, err + } + + includeWriter, err := mpWriter.CreatePart(includeType) + if err != nil { + return []byte{}, err + } + + _, err = includeWriter.Write([]byte(includePart)) + if err != nil { + return []byte{}, err + } + + if err := mpWriter.Close(); err != nil { + return []byte{}, err + } + + return buf.Bytes(), nil +} diff --git a/pkg/cloud/services/secretsmanager/cloudinit_test.go b/pkg/cloud/services/secretsmanager/cloudinit_test.go new file mode 100644 index 0000000000..e847e960a5 --- /dev/null +++ b/pkg/cloud/services/secretsmanager/cloudinit_test.go @@ -0,0 +1,33 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package secretsmanager + +import ( + "bytes" + "net/mail" + "testing" +) + +func TestGenerateCloudInitMIMEDocument(t *testing.T) { + secretARN := "secretARN" + doc, _ := GenerateCloudInitMIMEDocument(secretARN, "eu-west-1") + + _, err := mail.ReadMessage(bytes.NewBuffer(doc)) + if err != nil { + t.Fatalf("Cannot parse MIME doc: %+v\n%s", err, string(doc)) + } +} diff --git a/pkg/cloud/services/secretsmanager/secret.go b/pkg/cloud/services/secretsmanager/secret.go new file mode 100644 index 0000000000..5e259b227d --- /dev/null +++ b/pkg/cloud/services/secretsmanager/secret.go @@ -0,0 +1,104 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package secretsmanager + +import ( + "strings" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/secretsmanager" + apirand "k8s.io/apimachinery/pkg/util/rand" + infrav1 "sigs.k8s.io/cluster-api-provider-aws/api/v1alpha3" + "sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/awserrors" + "sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/converters" + "sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/scope" +) + +const ( + parameterPrefix = "aws.cluster.x-k8s.io" + namespacesPrefix = "namespaces" + clustersPrefix = "clusters" + instancesPrefix = "instances" +) + +// Create stores a secret in AWS Secrets Manager for a given machine +func (s *Service) Create(m *scope.MachineScope, data []byte) (string, error) { + // Make sure to use the MachineScope here to get the merger of AWSCluster and AWSMachine tags + additionalTags := m.AdditionalTags() + // Set the cloud provider tag + additionalTags[infrav1.ClusterAWSCloudProviderTagKey(s.scope.Name())] = string(infrav1.ResourceLifecycleOwned) + + tags := infrav1.Build(infrav1.BuildParams{ + ClusterName: s.scope.Name(), + Lifecycle: infrav1.ResourceLifecycleOwned, + Name: aws.String(m.Name()), + Role: aws.String(m.Role()), + Additional: additionalTags, + }) + + name := s.secretName(m) + + resp, err := s.scope.SecretsManager.CreateSecret(&secretsmanager.CreateSecretInput{ + Name: aws.String(name), + SecretBinary: data, + Tags: converters.MapToSecretsManagerTags(tags), + }) + + if err != nil { + return "", err + } + + return aws.StringValue(resp.ARN), nil +} + +// Delete the secret belonging to a machine from AWS Secrets Manager +func (s *Service) Delete(m *scope.MachineScope) error { + secretArn := m.AWSMachine.Spec.CloudInit.SecretARN + if secretArn == "" { + return nil + } + _, err := s.scope.SecretsManager.DeleteSecret(&secretsmanager.DeleteSecretInput{ + SecretId: aws.String(secretArn), + ForceDeleteWithoutRecovery: aws.Bool(true), + }) + + if awserrors.IsNotFound(err) { + return nil + } + + return err +} + +func (s *Service) secretName(m *scope.MachineScope) string { + prefix := strings.Join( + []string{ + parameterPrefix, + namespacesPrefix, + s.scope.Namespace(), + clustersPrefix, + s.scope.Name(), + instancesPrefix, + m.Name(), + }, + "/", + ) + "-" + + // apirand uses 27 runes, 27^54 is closest to 2^256 for 256-bits of entropy. + randomStr := apirand.String(54) + + return prefix + randomStr +} diff --git a/pkg/cloud/services/secretsmanager/secret_fetch_script.go b/pkg/cloud/services/secretsmanager/secret_fetch_script.go new file mode 100644 index 0000000000..22aa0fa203 --- /dev/null +++ b/pkg/cloud/services/secretsmanager/secret_fetch_script.go @@ -0,0 +1,156 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package secretsmanager + +//nolint +const secretFetchScript = `#!/bin/bash + +# Copyright 2020 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +REGION="{{.Region}}" +SECRET_ID="{{.SecretID}}" +FILE="/etc/secret-userdata.txt" + +# Log an error and exit. +# Args: +# $1 Message to log with the error +# $2 The error code to return +log::error_exit() { + local message="${1}" + local code="${2}" + + log::error "${message}" + log::error "aws.cluster.x-k8s.io encrypted cloud-init script $0 exiting with status ${code}" + exit "${code}" +} + +log::success_exit() { + log::info "aws.cluster.x-k8s.io encrypted cloud-init script $0 finished" + exit 0 +} + +# Log an error but keep going. +log::error() { + local message="${1}" + timestamp=$(date --iso-8601=seconds) + echo "!!! [${timestamp}] ${1}" >&2 + shift + for message; do + echo " ${message}" >&2 + done +} + +# Print a status line. Formatted to show up in a stream of output. +log::info() { + timestamp=$(date --iso-8601=seconds) + echo "+++ [${timestamp}] ${1}" + shift + for message; do + echo " ${message}" + done +} + +check_aws_command() { + local command="${1}" + local code="${2}" + local out="${3}" + local sanitised="${out//[$'\t\r\n']/}" + case ${code} in + "0") + log::info "AWS CLI reported successful execution for ${command}" + ;; + "2") + log::error "AWS CLI reported that it could not parse ${command}" + log::error "${sanitised}" + ;; + "130") + log::error "AWS CLI reported SIGINT signal during ${command}" + log::error "${sanitised}" + ;; + "255") + log::error "AWS CLI reported service error for ${command}" + log::error "${sanitised}" + ;; + *) + log::error "AWS CLI reported unknown error ${code} for ${command}" + log::error "${sanitised}" + ;; + esac +} + +delete_secret_value() { + local out + log::info "deleting secret from AWS Secrets Manager" + out=$( + set +e + aws secretsmanager --region ${REGION} delete-secret --force-delete-without-recovery --secret-id ${SECRET_ID} 2>&1 + ) + local delete_return=$? + check_aws_command "SecretsManager::DeleteSecret" "${delete_return}" "${out}" + if [ ${delete_return} -ne 0 ]; then + log::error_exit "Could not delete secret value" 2 + fi +} + +log::info "aws.cluster.x-k8s.io encrypted cloud-init script $0 started" +umask 006 +if test -f "${FILE}"; then + log::info "encrypted userdata already written to disk" + log::success_exit +fi + +log::info "getting userdata from AWS Secrets Manager" +log::info "getting secret value from AWS Secrets Manager" +BINARY_DATA=$( + set +e + aws secretsmanager --region ${REGION} get-secret-value --output text --query 'SecretBinary' --secret-id ${SECRET_ID} 2>&1 +) +GET_RETURN=$? +check_aws_command "SecretsManager::GetSecretValue" "$?" "${BINARY_DATA}" +if [ ${GET_RETURN} -ne 0 ]; then + log::error "could not get secret value, deleting secret" + delete_secret_value + log::error_exit "could not get secret value, but secret was deleted" 1 +fi + +delete_secret_value + +log::info "decoding and decompressing userdata" +UNZIPPED=$(echo "${BINARY_DATA}" | base64 -d | gunzip) +GUNZIP_RETURN=$? +if [ ${GUNZIP_RETURN} -ne 0 ]; then + log::error_exit "could not get unzip data" 4 +fi + +log::info "writing userdata to ${FILE}" +echo -n "${UNZIPPED}" >${FILE} +log::info "restarting cloud-init" +systemctl restart cloud-init +log::success_exit +` diff --git a/pkg/cloud/services/secretsmanager/service.go b/pkg/cloud/services/secretsmanager/service.go new file mode 100644 index 0000000000..ad46ff7914 --- /dev/null +++ b/pkg/cloud/services/secretsmanager/service.go @@ -0,0 +1,35 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package secretsmanager + +import ( + "sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/scope" +) + +// Service holds a collection of interfaces. +// The interfaces are broken down like this to group functions together. +// One alternative is to have a large list of functions from the ec2 client. +type Service struct { + scope *scope.ClusterScope +} + +// NewService returns a new service given the api clients. +func NewService(scope *scope.ClusterScope) *Service { + return &Service{ + scope: scope, + } +} diff --git a/pkg/cloud/services/userdata/utils.go b/pkg/cloud/services/userdata/utils.go index f25897dd84..9205cb2018 100644 --- a/pkg/cloud/services/userdata/utils.go +++ b/pkg/cloud/services/userdata/utils.go @@ -17,9 +17,13 @@ limitations under the License. package userdata import ( + "bytes" + "compress/gzip" "encoding/base64" "strings" "text/template" + + "github.com/pkg/errors" ) var ( @@ -38,3 +42,18 @@ func templateYAMLIndent(i int, input string) string { ident := "\n" + strings.Repeat(" ", i) return strings.Repeat(" ", i) + strings.Join(split, ident) } + +// GzipBytes will gzip a byte array +func GzipBytes(dat []byte) ([]byte, error) { + var buf bytes.Buffer + gz := gzip.NewWriter(&buf) + if _, err := gz.Write(dat); err != nil { + return []byte{}, errors.Wrap(err, "failed to gzip bytes") + } + + if err := gz.Close(); err != nil { + return []byte{}, errors.Wrap(err, "failed to gzip bytes") + } + + return buf.Bytes(), nil +}