Skip to content

Commit

Permalink
feat: Support for imagePullSecrets
Browse files Browse the repository at this point in the history
Add support for pulling images from sources requiring authentication.
The feature adheres to `imagePullSecrets` in `Pod`s and `ServiceAccount`s.

Loosely based on some of @DolevAlgam's work in #92

fixes #19
  • Loading branch information
estahn committed Sep 24, 2021
1 parent ef72c66 commit 9baa2c2
Show file tree
Hide file tree
Showing 12 changed files with 451 additions and 88 deletions.
24 changes: 23 additions & 1 deletion cmd/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ import (

"github.com/estahn/k8s-image-swapper/pkg/config"
"github.com/estahn/k8s-image-swapper/pkg/registry"
"github.com/estahn/k8s-image-swapper/pkg/secrets"
"github.com/estahn/k8s-image-swapper/pkg/types"
"github.com/estahn/k8s-image-swapper/pkg/webhook"
homedir "github.com/mitchellh/go-homedir"
Expand All @@ -42,6 +43,8 @@ import (
kwhhttp "github.com/slok/kubewebhook/v2/pkg/http"
"github.com/spf13/cobra"
"github.com/spf13/viper"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
)

var cfgFile string
Expand Down Expand Up @@ -77,7 +80,9 @@ A mutating webhook for Kubernetes, pointing the images to a new location.`,
log.Err(err)
}

wh, err := webhook.NewImageSwapperWebhook(rClient, cfg.Source.Filters, imageSwapPolicy, imageCopyPolicy)
imagePullSecretProvider := setupImagePullSecretsProvider()

wh, err := webhook.NewImageSwapperWebhook(rClient, imagePullSecretProvider, cfg.Source.Filters, imageSwapPolicy, imageCopyPolicy)
if err != nil {
log.Err(err).Msg("error creating webhook")
os.Exit(1)
Expand Down Expand Up @@ -243,3 +248,20 @@ func initLogger() {
log.Logger = log.With().Caller().Logger()
}
}

// setupImagePullSecretsProvider configures the provider handling secrets
func setupImagePullSecretsProvider() secrets.ImagePullSecretsProvider {
config, err := rest.InClusterConfig()
if err != nil {
log.Warn().Err(err).Msg("failed to configure Kubernetes client, will continue without reading secrets")
return secrets.NewDummyImagePullSecretsProvider()
}

clientset, err := kubernetes.NewForConfig(config)
if err != nil {
log.Warn().Err(err).Msg("failed to configure Kubernetes client, will continue without reading secrets")
return secrets.NewDummyImagePullSecretsProvider()
}

return secrets.NewKubernetesImagePullSecretsProvider(clientset)
}
6 changes: 3 additions & 3 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,9 @@ require (
github.com/containerd/containerd v1.5.2 // indirect
github.com/containers/image/v5 v5.16.0
github.com/containers/libtrust v0.0.0-20200511145503-9c3a6c22cd9a // indirect
github.com/davecgh/go-spew v1.1.1
github.com/dgraph-io/ristretto v0.1.0
github.com/evanphx/json-patch v4.11.0+incompatible
github.com/go-co-op/gocron v1.9.0
github.com/golang/glog v0.0.0-20210429001901-424d2337a529 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
Expand All @@ -27,11 +29,9 @@ require (
github.com/spf13/viper v1.8.1
github.com/stretchr/objx v0.3.0 // indirect
github.com/stretchr/testify v1.7.0
golang.org/x/net v0.0.0-20210614182718-04defd469f4e // indirect
google.golang.org/genproto v0.0.0-20210617175327-b9e0b3197ced // indirect
gopkg.in/yaml.v2 v2.4.0
k8s.io/api v0.22.1
k8s.io/apimachinery v0.22.2
k8s.io/client-go v0.21.2 // indirect
k8s.io/klog/v2 v2.9.0 // indirect
k8s.io/client-go v0.21.2
)
64 changes: 11 additions & 53 deletions go.sum

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion pkg/config/config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import (
"testing"

"github.com/stretchr/testify/assert"
"gopkg.in/yaml.v2"
yaml "gopkg.in/yaml.v2"
)

// TestConfigParses validates if yaml annotation do not overlap
Expand Down
18 changes: 15 additions & 3 deletions pkg/registry/ecr.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,13 +9,14 @@ import (
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/ecr"
"github.com/aws/aws-sdk-go/service/ecr/ecriface"
"github.com/dgraph-io/ristretto"
"github.com/go-co-op/gocron"
"github.com/rs/zerolog/log"
)

type ECRClient struct {
client *ecr.ECR
client ecriface.ECRAPI
ecrDomain string
authToken []byte
cache *ristretto.Cache
Expand Down Expand Up @@ -90,7 +91,7 @@ func (e *ECRClient) ImageExists(ref string) bool {
"inspect",
"--retry-times", "3",
"docker://" + ref,
"--creds", e.Credentials(),
"--creds", string(e.Credentials()),
}

log.Trace().Str("app", app).Strs("args", args).Msg("executing command to inspect image")
Expand Down Expand Up @@ -168,7 +169,18 @@ func NewECRClient(region string, ecrDomain string) (*ECRClient, error) {
}

if err := client.scheduleTokenRenewal(); err != nil {
panic(err)
return nil, err
}

return client, nil
}

func NewMockECRClient(ecrClient ecriface.ECRAPI, region string, ecrDomain string) (*ECRClient, error) {
client := &ECRClient{
client: ecrClient,
ecrDomain: ecrDomain,
cache: nil,
scheduler: nil,
}

return client, nil
Expand Down
15 changes: 15 additions & 0 deletions pkg/secrets/dummy.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
package secrets

import v1 "k8s.io/api/core/v1"

// DummyImagePullSecretsProvider does nothing
type DummyImagePullSecretsProvider struct {
}

func NewDummyImagePullSecretsProvider() ImagePullSecretsProvider {
return &DummyImagePullSecretsProvider{}
}

func (p *DummyImagePullSecretsProvider) GetImagePullSecrets(pod *v1.Pod) (*ImagePullSecretsResult, error) {
return NewImagePullSecretsResult(), nil
}
95 changes: 95 additions & 0 deletions pkg/secrets/kubernetes.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
package secrets

import (
"context"
"io/ioutil"
"os"

jsonpatch "github.com/evanphx/json-patch"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
)

// KubernetesImagePullSecretsProvider retrieves the secrets holding docker auth information from Kubernetes and merges
// them if necessary. Supports Pod secrets as well as ServiceAccount secrets.
type KubernetesImagePullSecretsProvider struct {
kubernetesClient kubernetes.Interface
}

// ImagePullSecretsResult contains the result of GetImagePullSecrets
type ImagePullSecretsResult struct {
Secrets map[string][]byte
Aggregate []byte
}

// NewImagePullSecretsResult initialises ImagePullSecretsResult
func NewImagePullSecretsResult() *ImagePullSecretsResult {
return &ImagePullSecretsResult{
Secrets: map[string][]byte{},
Aggregate: []byte("{}"),
}
}

// Add adds a secrets to internal list and rebuilds the aggregate
func (r *ImagePullSecretsResult) Add(name string, data []byte) {
r.Secrets[name] = data
r.Aggregate, _ = jsonpatch.MergePatch(r.Aggregate, data)
}

// AuthFile provides the aggregate as a file to be used by a docker client
func (r *ImagePullSecretsResult) AuthFile() (*os.File, error) {
tmpfile, err := ioutil.TempFile("", "auth")
if err != nil {
return nil, err
}

if _, err := tmpfile.Write(r.Aggregate); err != nil {
return nil, err
}
if err := tmpfile.Close(); err != nil {
return nil, err
}

return tmpfile, nil
}

func NewKubernetesImagePullSecretsProvider(clientset kubernetes.Interface) ImagePullSecretsProvider {
return &KubernetesImagePullSecretsProvider{
kubernetesClient: clientset,
}
}

// GetImagePullSecrets returns all secrets with their respective content
func (p *KubernetesImagePullSecretsProvider) GetImagePullSecrets(pod *v1.Pod) (*ImagePullSecretsResult, error) {
var secrets = make(map[string][]byte)

// retrieve secret names from pod ServiceAccount (spec.imagePullSecrets)
serviceAccount, err := p.kubernetesClient.CoreV1().
ServiceAccounts(pod.Namespace).
Get(context.TODO(), pod.Spec.ServiceAccountName, metav1.GetOptions{})
if err != nil {
// TODO: Handle error gracefully, dont panic
return nil, err
}

imagePullSecrets := append(pod.Spec.ImagePullSecrets, serviceAccount.ImagePullSecrets...)

result := NewImagePullSecretsResult()
for _, imagePullSecret := range imagePullSecrets {
// fetch a secret only once
if _, exists := secrets[imagePullSecret.Name]; exists {
continue
}

secret, _ := p.kubernetesClient.CoreV1().Secrets(pod.Namespace).Get(context.TODO(), imagePullSecret.Name, metav1.GetOptions{})

if secret.Type != v1.SecretTypeDockerConfigJson {
continue
}

result.Add(imagePullSecret.Name, secret.Data[v1.DockerConfigJsonKey])
}

return result, nil
}
116 changes: 116 additions & 0 deletions pkg/secrets/kubernetes_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,116 @@
package secrets

import (
"context"
"testing"

"github.com/stretchr/testify/assert"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes/fake"
)

//type ExampleTestSuite struct {
// suite.Suite
//}
//
//func (suite *ExampleTestSuite) SetupTest() {
//}
//func (suite *ExampleTestSuite) TestExample() {
// assert.Equal(suite.T(), 5, 1)
//}
//
//func TestExampleTestSuite(t *testing.T) {
// suite.Run(t, new(ExampleTestSuite))
//}

// Test:
//+------------------+-----+----------------+
//| | Pod | ServiceAccount |
//+------------------+-----+----------------+
//| ImagePullSecrets | Y | Y |
//+------------------+-----+----------------+
//| ImagePullSecrets | Y | N |
//+------------------+-----+----------------+
//| ImagePullSecrets | N | Y |
//+------------------+-----+----------------+
//| ImagePullSecrets | N | N |
//+------------------+-----+----------------+
//
// Multple image pull secrets on pod + service account
// Pod secret should override service account secret

func TestKubernetesCredentialProvider_GetImagePullSecrets(t *testing.T) {
clientSet := fake.NewSimpleClientset()

svcAccount := &v1.ServiceAccount{
ObjectMeta: metav1.ObjectMeta{
Name: "my-service-account",
},
ImagePullSecrets: []v1.LocalObjectReference{
{Name: "my-sa-secret"},
},
}
svcAccountSecretDockerConfigJson := []byte(`{"auths":{"my-sa-secret.registry.example.com":{"username":"my-sa-secret","password":"xxxxxxxxxxx","email":"jdoe@example.com","auth":"c3R...zE2"}}}`)
svcAccountSecret := &v1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: "my-sa-secret",
},
Type: v1.SecretTypeDockerConfigJson,
Data: map[string][]byte{
v1.DockerConfigJsonKey: svcAccountSecretDockerConfigJson,
},
}
podSecretDockerConfigJson := []byte(`{"auths":{"my-pod-secret.registry.example.com":{"username":"my-sa-secret","password":"xxxxxxxxxxx","email":"jdoe@example.com","auth":"c3R...zE2"}}}`)
podSecret := &v1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: "my-pod-secret",
},
Type: v1.SecretTypeDockerConfigJson,
Data: map[string][]byte{
v1.DockerConfigJsonKey: podSecretDockerConfigJson,
},
}
pod := &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Namespace: "test-ns",
Name: "my-pod",
},
Spec: v1.PodSpec{
ServiceAccountName: "my-service-account",
ImagePullSecrets: []v1.LocalObjectReference{
{Name: "my-pod-secret"},
},
},
}

_, _ = clientSet.CoreV1().ServiceAccounts("test-ns").Create(context.TODO(), svcAccount, metav1.CreateOptions{})
_, _ = clientSet.CoreV1().Secrets("test-ns").Create(context.TODO(), svcAccountSecret, metav1.CreateOptions{})
_, _ = clientSet.CoreV1().Secrets("test-ns").Create(context.TODO(), podSecret, metav1.CreateOptions{})

provider := NewKubernetesImagePullSecretsProvider(clientSet)
result, err := provider.GetImagePullSecrets(pod)

assert.NoError(t, err)
assert.NotNil(t, result)
assert.Len(t, result.Secrets, 2)
assert.Equal(t, svcAccountSecretDockerConfigJson, result.Secrets["my-sa-secret"])
assert.Equal(t, podSecretDockerConfigJson, result.Secrets["my-pod-secret"])
}

// TestImagePullSecretsResult_Add tests if aggregation works
func TestImagePullSecretsResult_Add(t *testing.T) {
expected := &ImagePullSecretsResult{
Secrets: map[string][]byte{
"foo": []byte("{\"foo\":\"123\"}"),
"bar": []byte("{\"bar\":\"456\"}"),
},
Aggregate: []byte("{\"bar\":\"456\",\"foo\":\"123\"}"),
}

r := NewImagePullSecretsResult()
r.Add("foo", []byte("{\"foo\":\"123\"}"))
r.Add("bar", []byte("{\"bar\":\"456\"}"))

assert.Equal(t, r, expected)
}
7 changes: 7 additions & 0 deletions pkg/secrets/provider.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
package secrets

import v1 "k8s.io/api/core/v1"

type ImagePullSecretsProvider interface {
GetImagePullSecrets(pod *v1.Pod) (*ImagePullSecretsResult, error)
}
Loading

0 comments on commit 9baa2c2

Please sign in to comment.