Add S3 KMS support to blocks storage client (grafana#3810)

* Add S3 KMS support to blocks storage client Signed-off-by: Marco Pracucci <marco@pracucci.com> * Fixed integration test Signed-off-by: Marco Pracucci <marco@pracucci.com> * Removed named return arguments from parseKMSEncryptionContext() Signed-off-by: Marco Pracucci <marco@pracucci.com> * Update pkg/storage/bucket/s3/config.go Signed-off-by: Marco Pracucci <marco@pracucci.com> Co-authored-by: Jacob Lisi <jlisi@grafana.com> * Rebuilt doc Signed-off-by: Marco Pracucci <marco@pracucci.com> Co-authored-by: Jacob Lisi <jlisi@grafana.com>
cyriltovena · Feb 11, 2021 · 84c0916 · 84c0916
1 parent 19210b9
commit 84c0916
Show file tree

Hide file tree

Showing 3 changed files with 32 additions and 49 deletions.
diff --git a/aws/s3_storage_client.go b/aws/s3_storage_client.go
@@ -27,6 +27,7 @@ import (
 	"github.com/weaveworks/common/instrument"
 
 	"github.com/cortexproject/cortex/pkg/chunk"
+	cortex_s3 "github.com/cortexproject/cortex/pkg/storage/bucket/s3"
 	"github.com/cortexproject/cortex/pkg/util"
 	"github.com/cortexproject/cortex/pkg/util/flagext"
 )
@@ -64,15 +65,15 @@ type S3Config struct {
 	S3ForcePathStyle bool
 
 	BucketNames      string
-	Endpoint         string     `yaml:"endpoint"`
-	Region           string     `yaml:"region"`
-	AccessKeyID      string     `yaml:"access_key_id"`
-	SecretAccessKey  string     `yaml:"secret_access_key"`
-	Insecure         bool       `yaml:"insecure"`
-	SSEEncryption    bool       `yaml:"sse_encryption"`
-	HTTPConfig       HTTPConfig `yaml:"http_config"`
-	SignatureVersion string     `yaml:"signature_version"`
-	SSEConfig        SSEConfig  `yaml:"sse"`
+	Endpoint         string              `yaml:"endpoint"`
+	Region           string              `yaml:"region"`
+	AccessKeyID      string              `yaml:"access_key_id"`
+	SecretAccessKey  string              `yaml:"secret_access_key"`
+	Insecure         bool                `yaml:"insecure"`
+	SSEEncryption    bool                `yaml:"sse_encryption"`
+	HTTPConfig       HTTPConfig          `yaml:"http_config"`
+	SignatureVersion string              `yaml:"signature_version"`
+	SSEConfig        cortex_s3.SSEConfig `yaml:"sse"`
 
 	Inject InjectRequestMiddleware `yaml:"-"`
 }
@@ -165,8 +166,8 @@ func buildSSEParsedConfig(cfg S3Config) (*SSEParsedConfig, error) {
 
 	// deprecated, but if used it assumes SSE-S3 type
 	if cfg.SSEEncryption {
-		return NewSSEParsedConfig(SSEConfig{
-			Type: SSES3,
+		return NewSSEParsedConfig(cortex_s3.SSEConfig{
+			Type: cortex_s3.SSES3,
 		})
 	}
 

diff --git a/aws/sse_config.go b/aws/sse_config.go
@@ -3,20 +3,15 @@ package aws
 import (
 	"encoding/base64"
 	"encoding/json"
-	"flag"
 
 	"github.com/pkg/errors"
+
+	cortex_s3 "github.com/cortexproject/cortex/pkg/storage/bucket/s3"
 )
 
 const (
-	// SSEKMS config type constant to configure S3 server side encryption using KMS
-	// https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingKMSEncryption.html
-	SSEKMS     = "SSE-KMS"
 	sseKMSType = "aws:kms"
-	// SSES3 config type constant to configure S3 server side encryption with AES-256
-	// https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingServerSideEncryption.html
-	SSES3     = "SSE-S3"
-	sseS3Type = "AES256"
+	sseS3Type  = "AES256"
 )
 
 // SSEParsedConfig configures server side encryption (SSE)
@@ -27,29 +22,14 @@ type SSEParsedConfig struct {
 	KMSEncryptionContext *string
 }
 
-// SSEConfig configures S3 server side encryption
-// struct that is going to receive user input (through config file or CLI)
-type SSEConfig struct {
-	Type                 string `yaml:"type"`
-	KMSKeyID             string `yaml:"kms_key_id"`
-	KMSEncryptionContext string `yaml:"kms_encryption_context"`
-}
-
-// RegisterFlagsWithPrefix adds the flags required to config this to the given FlagSet
-func (cfg *SSEConfig) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) {
-	f.StringVar(&cfg.Type, prefix+"type", "", "Enable AWS Server Side Encryption. Only SSE-S3 and SSE-KMS are supported")
-	f.StringVar(&cfg.KMSKeyID, prefix+"kms-key-id", "", "KMS Key ID used to encrypt objects in S3")
-	f.StringVar(&cfg.KMSEncryptionContext, prefix+"kms-encryption-context", "", "KMS Encryption Context used for object encryption. It expects a JSON as a string.")
-}
-
 // NewSSEParsedConfig creates a struct to configure server side encryption (SSE)
-func NewSSEParsedConfig(cfg SSEConfig) (*SSEParsedConfig, error) {
+func NewSSEParsedConfig(cfg cortex_s3.SSEConfig) (*SSEParsedConfig, error) {
 	switch cfg.Type {
-	case SSES3:
+	case cortex_s3.SSES3:
 		return &SSEParsedConfig{
 			ServerSideEncryption: sseS3Type,
 		}, nil
-	case SSEKMS:
+	case cortex_s3.SSEKMS:
 		if cfg.KMSKeyID == "" {
 			return nil, errors.New("KMS key id must be passed when SSE-KMS encryption is selected")
 		}

diff --git a/aws/sse_config_test.go b/aws/sse_config_test.go
@@ -5,6 +5,8 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/stretchr/testify/assert"
+
+	cortex_s3 "github.com/cortexproject/cortex/pkg/storage/bucket/s3"
 )
 
 func TestNewSSEParsedConfig(t *testing.T) {
@@ -15,23 +17,23 @@ func TestNewSSEParsedConfig(t *testing.T) {
 
 	tests := []struct {
 		name        string
-		params      SSEConfig
+		params      cortex_s3.SSEConfig
 		expected    *SSEParsedConfig
 		expectedErr error
 	}{
 		{
 			name: "Test SSE encryption with SSES3 type",
-			params: SSEConfig{
-				Type: SSES3,
+			params: cortex_s3.SSEConfig{
+				Type: cortex_s3.SSES3,
 			},
 			expected: &SSEParsedConfig{
 				ServerSideEncryption: sseS3Type,
 			},
 		},
 		{
 			name: "Test SSE encryption with SSEKMS type without context",
-			params: SSEConfig{
-				Type:     SSEKMS,
+			params: cortex_s3.SSEConfig{
+				Type:     cortex_s3.SSEKMS,
 				KMSKeyID: kmsKeyID,
 			},
 			expected: &SSEParsedConfig{
@@ -41,8 +43,8 @@ func TestNewSSEParsedConfig(t *testing.T) {
 		},
 		{
 			name: "Test SSE encryption with SSEKMS type with context",
-			params: SSEConfig{
-				Type:                 SSEKMS,
+			params: cortex_s3.SSEConfig{
+				Type:                 cortex_s3.SSEKMS,
 				KMSKeyID:             kmsKeyID,
 				KMSEncryptionContext: kmsEncryptionContext,
 			},
@@ -54,23 +56,23 @@ func TestNewSSEParsedConfig(t *testing.T) {
 		},
 		{
 			name: "Test invalid SSE type",
-			params: SSEConfig{
+			params: cortex_s3.SSEConfig{
 				Type: "invalid",
 			},
 			expectedErr: errors.New("SSE type is empty or invalid"),
 		},
 		{
 			name: "Test SSE encryption with SSEKMS type without KMS Key ID",
-			params: SSEConfig{
-				Type:     SSEKMS,
+			params: cortex_s3.SSEConfig{
+				Type:     cortex_s3.SSEKMS,
 				KMSKeyID: "",
 			},
 			expectedErr: errors.New("KMS key id must be passed when SSE-KMS encryption is selected"),
 		},
 		{
 			name: "Test SSE with invalid KMS encryption context JSON",
-			params: SSEConfig{
-				Type:                 SSEKMS,
+			params: cortex_s3.SSEConfig{
+				Type:                 cortex_s3.SSEKMS,
 				KMSKeyID:             kmsKeyID,
 				KMSEncryptionContext: `INVALID_JSON`,
 			},