feat: cross account caching with role (#336)

This allows to have a single account for caching in multi-account environments. Changes: - Added support to assume role - Added the ability to specify access policy - Added the ability to specify lifecycle policy Initially, I had two approaches to make this work: 1) let other accounts create repos and fill them with images 2) use the role in the target account (this PR) While 1) would be preferable, unfortunately, it's doesn't look like it's possible: you can allow other accounts to create repos, but not put any policies.
estahn · Sep 7, 2022 · 98d138e · 98d138e
1 parent 730fcaa
commit 98d138e
Show file tree

Hide file tree

Showing 7 changed files with 228 additions and 31 deletions.
diff --git a/.gitignore b/.gitignore
@@ -15,3 +15,5 @@
 # vendor/
 
 .idea/
+coverage.txt
+k8s-image-swapper
diff --git a/.k8s-image-swapper.yml b/.k8s-image-swapper.yml
@@ -41,6 +41,7 @@ target:
   aws:
     accountId: 123456789
     region: ap-southeast-2
+    role: arn:aws:iam::123456789012:role/roleName
     ecrOptions:
       tags:
         - key: CreatedBy
@@ -51,5 +52,48 @@ target:
       encryptionConfiguration:
         encryptionType: AES256
         kmsKey: string
+      accessPolicy: |
+        {
+          "Version": "2008-10-17",
+          "Statement": [
+            {
+              "Sid": "AllowCrossAccountPull",
+              "Effect": "Allow",
+              "Principal": {
+                "AWS": "*"
+              },
+              "Action": [
+                "ecr:GetDownloadUrlForLayer",
+                "ecr:BatchGetImage",
+                "ecr:BatchCheckLayerAvailability"
+              ],
+              "Condition": {
+                "StringEquals": {
+                  "aws:PrincipalOrgID": [
+                    "o-xxxxxxxx"
+                  ]
+                }
+              }
+            }
+          ]
+        }
+
+      lifecyclePolicy: |
+        {
+          "rules": [
+            {
+              "rulePriority": 1,
+              "description": "Rule 1",
+              "selection": {
+                "tagStatus": "any",
+                "countType": "imageCountMoreThan",
+                "countNumber": 1
+              },
+              "action": {
+                "type": "expire"
+              }
+            }
+          ]
+        }
 #    dockerio:
 #    quayio:
diff --git a/cmd/root.go b/cmd/root.go
@@ -64,7 +64,7 @@ A mutating webhook for Kubernetes, pointing the images to a new location.`,
 		//metricsRec := metrics.NewPrometheus(promReg)
 		log.Trace().Interface("config", cfg).Msg("config")
 
-		rClient, err := registry.NewECRClient(cfg.Target.AWS.Region, cfg.Target.AWS.EcrDomain())
+		rClient, err := registry.NewECRClient(cfg.Target.AWS.Region, cfg.Target.AWS.EcrDomain(), cfg.Target.AWS.AccountID, cfg.Target.AWS.Role, cfg.Target.AWS.AccessPolicy, cfg.Target.AWS.LifecyclePolicy)
 		if err != nil {
 			log.Err(err).Msg("error connecting to registry client")
 			os.Exit(1)

diff --git a/docs/getting-started.md b/docs/getting-started.md
@@ -29,6 +29,91 @@ Choose from one of the strategies below or an alternative if needed.
       --from-literal=aws_secret_access_key=<...>
     ```
 
+#### Using ECR registries cross-account
+
+Although ECR allows creating registry policy that allows reposistories creation from different account, there's no way to push anything to these repositories.
+ECR resource-level policy can not be applied during creation, and to apply it afterwards we need ecr:SetRepositoryPolicy permission, which foreign account doesn't have.
+
+One way out of this conundrum is to assume the role in target account
+
+```yaml
+target:
+  type: aws
+  aws:
+    accountId: 123456789
+    region: ap-southeast-2
+    role: arn:aws:iam::123456789012:role/roleName
+```
+!!! note
+Make sure that target role has proper trust permissions that allow to assume it cross-account
+
+!!! note
+In order te be able to pull images from outside accounts, you will have to apply proper access policy
+
+
+#### Access policy
+
+You can specify the access policy that will be applied to the created repos in config. Policy should be raw json string.
+For example:
+```yaml
+target:
+  aws:
+    accountId: 123456789
+    region: ap-southeast-2
+    role: arn:aws:iam::123456789012:role/roleName
+    accessPolicy: '{
+  "Statement": [
+    {
+      "Sid": "AllowCrossAccountPull",
+      "Effect": "Allow",
+      "Principal": {
+        "AWS": "*"
+      },
+      "Action": [
+        "ecr:GetDownloadUrlForLayer",
+        "ecr:BatchGetImage",
+        "ecr:BatchCheckLayerAvailability"
+      ],
+      "Condition": {
+        "StringEquals": {
+          "aws:PrincipalOrgID": "o-xxxxxxxxxx"
+        }
+      }
+    }
+  ],
+  "Version": "2008-10-17"
+}'
+```
+
+#### Lifecycle policy
+
+Similarly to access policy, lifecycle policy can be specified, for example:
+
+```yaml
+target:
+  aws:
+    accountId: 123456789
+    region: ap-southeast-2
+    role: arn:aws:iam::123456789012:role/roleName
+    accessPolicy: '{
+  "rules": [
+    {
+      "rulePriority": 1,
+      "description": "Rule 1",
+      "selection": {
+        "tagStatus": "any",
+        "countType": "imageCountMoreThan",
+        "countNumber": 1000
+      },
+      "action": {
+        "type": "expire"
+      }
+    }
+  ]
+}
+'
+```
+
 #### Service Account
 
 1. Create an Webidentity IAM role (e.g. `k8s-image-swapper`) with the following trust policy, e.g

diff --git a/pkg/config/config.go b/pkg/config/config.go
@@ -54,8 +54,11 @@ type Target struct {
 }
 
 type AWS struct {
-	AccountID string `yaml:"accountId"`
-	Region    string `yaml:"region"`
+	AccountID       string `yaml:"accountId"`
+	Region          string `yaml:"region"`
+	Role            string `yaml:"role"`
+	AccessPolicy    string `yaml:"accessPolicy"`
+	LifecyclePolicy string `yaml:"lifecyclePolicy"`
 }
 
 func (a *AWS) EcrDomain() string {

diff --git a/pkg/registry/ecr.go b/pkg/registry/ecr.go
@@ -8,6 +8,7 @@ import (
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/awserr"
+	"github.com/aws/aws-sdk-go/aws/credentials/stscreds"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/ecr"
 	"github.com/aws/aws-sdk-go/service/ecr/ecriface"
@@ -19,11 +20,14 @@ import (
 var execCommand = exec.Command
 
 type ECRClient struct {
-	client    ecriface.ECRAPI
-	ecrDomain string
-	authToken []byte
-	cache     *ristretto.Cache
-	scheduler *gocron.Scheduler
+	client          ecriface.ECRAPI
+	ecrDomain       string
+	authToken       []byte
+	cache           *ristretto.Cache
+	scheduler       *gocron.Scheduler
+	targetAccount   string
+	accessPolicy    string
+	lifecyclePolicy string
 }
 
 func (e *ECRClient) Credentials() string {
@@ -41,13 +45,15 @@ func (e *ECRClient) CreateRepository(name string) error {
 			ScanOnPush: aws.Bool(true),
 		},
 		ImageTagMutability: aws.String(ecr.ImageTagMutabilityMutable),
+		RegistryId:         &e.targetAccount,
 		Tags: []*ecr.Tag{
 			{
 				Key:   aws.String("CreatedBy"),
 				Value: aws.String("k8s-image-swapper"),
 			},
 		},
 	})
+
 	if err != nil {
 		if aerr, ok := err.(awserr.Error); ok {
 			switch aerr.Code() {
@@ -63,6 +69,37 @@ func (e *ECRClient) CreateRepository(name string) error {
 		}
 	}
 
+	if len(e.accessPolicy) > 0 {
+		log.Info().Msg("Setting access policy on" + name)
+		log.Debug().Msg("Access policy: \n" + e.accessPolicy)
+		_, err := e.client.SetRepositoryPolicy(&ecr.SetRepositoryPolicyInput{
+			PolicyText:     &e.accessPolicy,
+			RegistryId:     &e.targetAccount,
+			RepositoryName: aws.String(name),
+		})
+
+		if err != nil {
+			log.Err(err).Msg(err.Error())
+			return err
+		}
+	}
+
+	if len(e.lifecyclePolicy) > 0 {
+		log.Info().Msg("Setting lifecycle policy on" + name)
+		log.Debug().Msg("Lifecycle policy: \n" + e.lifecyclePolicy)
+
+		_, err := e.client.PutLifecyclePolicy(&ecr.PutLifecyclePolicyInput{
+			LifecyclePolicyText: &e.lifecyclePolicy,
+			RegistryId:          &e.targetAccount,
+			RepositoryName:      aws.String(name),
+		})
+
+		if err != nil {
+			log.Err(err).Msg(err.Error())
+			return err
+		}
+	}
+
 	e.cache.Set(name, "", 1)
 
 	return nil
@@ -115,7 +152,10 @@ func (e *ECRClient) Endpoint() string {
 
 // requestAuthToken requests and returns an authentication token from ECR with its expiration date
 func (e *ECRClient) requestAuthToken() ([]byte, time.Time, error) {
-	getAuthTokenOutput, err := e.client.GetAuthorizationToken(&ecr.GetAuthorizationTokenInput{})
+	getAuthTokenOutput, err := e.client.GetAuthorizationToken(&ecr.GetAuthorizationTokenInput{
+		RegistryIds: []*string{&e.targetAccount},
+	})
+
 	if err != nil {
 		return []byte(""), time.Time{}, err
 	}
@@ -146,18 +186,33 @@ func (e *ECRClient) scheduleTokenRenewal() error {
 	return nil
 }
 
-func NewECRClient(region string, ecrDomain string) (*ECRClient, error) {
-	sess := session.Must(session.NewSessionWithOptions(session.Options{
+func NewECRClient(region string, ecrDomain string, targetAccount string, role string, accessPolicy string, lifecyclePolicy string) (*ECRClient, error) {
+	var sess *session.Session
+	var config *aws.Config
+	if role != "" {
+		log.Debug().Msg("Role is specified. Assuming " + role)
+		stsSession, _ := session.NewSession(config)
+		creds := stscreds.NewCredentials(stsSession, role)
+		config = aws.NewConfig().
+			WithRegion(region).
+			WithCredentialsChainVerboseErrors(true).
+			WithHTTPClient(&http.Client{
+				Timeout: 3 * time.Second,
+			}).
+			WithCredentials(creds)
+	} else {
+		config = aws.NewConfig().
+			WithRegion(region).
+			WithCredentialsChainVerboseErrors(true).
+			WithHTTPClient(&http.Client{
+				Timeout: 3 * time.Second,
+			})
+	}
+
+	sess = session.Must(session.NewSessionWithOptions(session.Options{
 		SharedConfigState: session.SharedConfigEnable,
+		Config:            (*config),
 	}))
-
-	config := aws.NewConfig().
-		WithRegion(region).
-		WithCredentialsChainVerboseErrors(true).
-		WithHTTPClient(&http.Client{
-			Timeout: 3 * time.Second,
-		})
-
 	ecrClient := ecr.New(sess, config)
 
 	cache, err := ristretto.NewCache(&ristretto.Config{
@@ -173,10 +228,13 @@ func NewECRClient(region string, ecrDomain string) (*ECRClient, error) {
 	scheduler.StartAsync()
 
 	client := &ECRClient{
-		client:    ecrClient,
-		ecrDomain: ecrDomain,
-		cache:     cache,
-		scheduler: scheduler,
+		client:          ecrClient,
+		ecrDomain:       ecrDomain,
+		cache:           cache,
+		scheduler:       scheduler,
+		targetAccount:   targetAccount,
+		accessPolicy:    accessPolicy,
+		lifecyclePolicy: lifecyclePolicy,
 	}
 
 	if err := client.scheduleTokenRenewal(); err != nil {
@@ -186,13 +244,14 @@ func NewECRClient(region string, ecrDomain string) (*ECRClient, error) {
 	return client, nil
 }
 
-func NewMockECRClient(ecrClient ecriface.ECRAPI, region string, ecrDomain string) (*ECRClient, error) {
+func NewMockECRClient(ecrClient ecriface.ECRAPI, region string, ecrDomain string, targetAccount, role string) (*ECRClient, error) {
 	client := &ECRClient{
-		client:    ecrClient,
-		ecrDomain: ecrDomain,
-		cache:     nil,
-		scheduler: nil,
-		authToken: []byte("mock-ecr-client-fake-auth-token"),
+		client:        ecrClient,
+		ecrDomain:     ecrDomain,
+		cache:         nil,
+		scheduler:     nil,
+		targetAccount: targetAccount,
+		authToken:     []byte("mock-ecr-client-fake-auth-token"),
 	}
 
 	return client, nil