Skip to content

Commit

Permalink
Include ConfigMaps and Secrets in promotion
Browse files Browse the repository at this point in the history 
- create primary configs and secrets at bootstrap
- copy configs and secrets from canary to primary and update the pod spec on promotion
  • Loading branch information
@stefanprodan
stefanprodan committed Jan 25, 2019
1 parent 0109788 commit 71cd4e0
Show file tree
Hide file tree
Showing 3 changed files with 211 additions and 32 deletions.
5 changes: 5 additions & 0 deletions pkg/controller/controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,11 @@ func NewController(
kubeClient: kubeClient,
istioClient: istioClient,
flaggerClient: flaggerClient,
configTracker: ConfigTracker{
logger: logger,
kubeClient: kubeClient,
flaggerClient: flaggerClient,
},
}

router := CanaryRouter{
Expand Down
28 changes: 25 additions & 3 deletions pkg/controller/deployer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,10 @@ type CanaryDeployer struct {
istioClient istioclientset.Interface
flaggerClient clientset.Interface
logger *zap.SugaredLogger
configTracker ConfigTracker
}

// Promote copies the pod spec from canary to primary
// Promote copies the pod spec, secrets and config maps from canary to primary
func (c *CanaryDeployer) Promote(cd *flaggerv1.Canary) error {
targetName := cd.Spec.TargetRef.Name
primaryName := fmt.Sprintf("%s-primary", targetName)
Expand All @@ -51,12 +52,23 @@ func (c *CanaryDeployer) Promote(cd *flaggerv1.Canary) error {
return fmt.Errorf("deployment %s.%s query error %v", primaryName, cd.Namespace, err)
}

// promote secrets and config maps
configRefs, err := c.configTracker.GetTargetConfigs(cd)
if err != nil {
return err
}
if err := c.configTracker.CreatePrimaryConfigs(cd, configRefs); err != nil {
return err
}

primaryCopy := primary.DeepCopy()
primaryCopy.Spec.ProgressDeadlineSeconds = canary.Spec.ProgressDeadlineSeconds
primaryCopy.Spec.MinReadySeconds = canary.Spec.MinReadySeconds
primaryCopy.Spec.RevisionHistoryLimit = canary.Spec.RevisionHistoryLimit
primaryCopy.Spec.Strategy = canary.Spec.Strategy
primaryCopy.Spec.Template.Spec = canary.Spec.Template.Spec

// update spec with primary secrets and config maps
primaryCopy.Spec.Template.Spec = c.configTracker.ApplyPrimaryConfigs(canary.Spec.Template.Spec, configRefs)

_, err = c.kubeClient.AppsV1().Deployments(cd.Namespace).Update(primaryCopy)
if err != nil {
Expand Down Expand Up @@ -290,6 +302,15 @@ func (c *CanaryDeployer) createPrimaryDeployment(cd *flaggerv1.Canary) error {

primaryDep, err := c.kubeClient.AppsV1().Deployments(cd.Namespace).Get(primaryName, metav1.GetOptions{})
if errors.IsNotFound(err) {
// create primary secrets and config maps
configRefs, err := c.configTracker.GetTargetConfigs(cd)
if err != nil {
return err
}
if err := c.configTracker.CreatePrimaryConfigs(cd, configRefs); err != nil {
return err
}
// create primary deployment
primaryDep = &appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{
Name: primaryName,
Expand Down Expand Up @@ -319,7 +340,8 @@ func (c *CanaryDeployer) createPrimaryDeployment(cd *flaggerv1.Canary) error {
Labels: map[string]string{"app": primaryName},
Annotations: canaryDep.Spec.Template.Annotations,
},
Spec: canaryDep.Spec.Template.Spec,
// update spec with the primary secrets and config maps
Spec: c.configTracker.ApplyPrimaryConfigs(canaryDep.Spec.Template.Spec, configRefs),
},
},
}
Expand Down
210 changes: 181 additions & 29 deletions pkg/controller/tracker.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,11 @@ import (
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/client-go/kubernetes"
)

// ConfigTracker is managing the operations for Kubernetes configmaps and secrets
// ConfigTracker is managing the operations for Kubernetes ConfigMaps and Secrets
type ConfigTracker struct {
kubeClient kubernetes.Interface
flaggerClient clientset.Interface
Expand All @@ -27,24 +28,67 @@ const (
ConfigRefSecret ConfigRefType = "secret"
)

// ConfigRef holds the reference to a tracked configmap or secrets
// ConfigRef holds the reference to a tracked Kubernetes ConfigMap or Secret
type ConfigRef struct {
Name string
Type ConfigRefType
Checksum string
}

// GetName returns the config ref type and name
func (c *ConfigRef) GetName() string {
return fmt.Sprintf("%s/%s", c.Type, c.Name)
}

func checksum(data interface{}) string {
jsonBytes, _ := json.Marshal(data)
hashBytes := sha256.Sum256(jsonBytes)

return fmt.Sprintf("%x", hashBytes[:8])
}

func (ct *ConfigTracker) getTargetConfigs(cd *flaggerv1.Canary) (map[string]ConfigRef, error) {
// getRefFromConfigMap transforms a Kubernetes ConfigMap into a ConfigRef
// and computes the checksum of the ConfigMap data
func (ct *ConfigTracker) getRefFromConfigMap(name string, namespace string) (*ConfigRef, error) {
config, err := ct.kubeClient.CoreV1().ConfigMaps(namespace).Get(name, metav1.GetOptions{})
if err != nil {
return nil, err
}

return &ConfigRef{
Name: config.Name,
Type: ConfigRefMap,
Checksum: checksum(config.Data),
}, nil
}

// getRefFromConfigMap transforms a Kubernetes Secret into a ConfigRef
// and computes the checksum of the Secret data
func (ct *ConfigTracker) getRefFromSecret(name string, namespace string) (*ConfigRef, error) {
secret, err := ct.kubeClient.CoreV1().Secrets(namespace).Get(name, metav1.GetOptions{})
if err != nil {
return nil, err
}

// ignore registry secrets (those should be set via service account)
if secret.Type != corev1.SecretTypeOpaque &&
secret.Type != corev1.SecretTypeBasicAuth &&
secret.Type != corev1.SecretTypeSSHAuth &&
secret.Type != corev1.SecretTypeTLS {
ct.logger.Debugf("ignoring secret %s.%s type not supported %v", name, namespace, secret.Type)
return nil, nil
}

return &ConfigRef{
Name: secret.Name,
Type: ConfigRefSecret,
Checksum: checksum(secret.Data),
}, nil
}

// GetTargetConfigs scans the target deployment for Kubernetes ConfigMaps and Secretes
// and returns a list of config references
func (ct *ConfigTracker) GetTargetConfigs(cd *flaggerv1.Canary) (map[string]ConfigRef, error) {
res := make(map[string]ConfigRef)
targetName := cd.Spec.TargetRef.Name
targetDep, err := ct.kubeClient.AppsV1().Deployments(cd.Namespace).Get(targetName, metav1.GetOptions{})
Expand Down Expand Up @@ -79,7 +123,6 @@ func (ct *ConfigTracker) getTargetConfigs(cd *flaggerv1.Canary) (map[string]Conf
}
}
}

// scan containers
for _, container := range targetDep.Spec.Template.Spec.Containers {
// scan env
Expand Down Expand Up @@ -139,36 +182,145 @@ func (ct *ConfigTracker) getTargetConfigs(cd *flaggerv1.Canary) (map[string]Conf
return res, nil
}

func (ct *ConfigTracker) getRefFromConfigMap(name string, namespace string) (*ConfigRef, error) {
config, err := ct.kubeClient.CoreV1().ConfigMaps(namespace).Get(name, metav1.GetOptions{})
if err != nil {
return nil, err
// CreatePrimaryConfigs syncs the primary Kubernetes ConfigMaps and Secretes
// with those found in the target deployment
func (ct *ConfigTracker) CreatePrimaryConfigs(cd *flaggerv1.Canary, refs map[string]ConfigRef) error {
for _, ref := range refs {
switch ref.Type {
case ConfigRefMap:
config, err := ct.kubeClient.CoreV1().ConfigMaps(cd.Namespace).Get(ref.Name, metav1.GetOptions{})
if err != nil {
return err
}
primaryName := fmt.Sprintf("%s-primary", config.GetName())
primaryConfigMap := &corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: primaryName,
Namespace: cd.Namespace,
Annotations: config.Annotations,
OwnerReferences: []metav1.OwnerReference{
*metav1.NewControllerRef(cd, schema.GroupVersionKind{
Group: flaggerv1.SchemeGroupVersion.Group,
Version: flaggerv1.SchemeGroupVersion.Version,
Kind: flaggerv1.CanaryKind,
}),
},
},
Data: config.Data,
}

// update or insert primary ConfigMap
_, err = ct.kubeClient.CoreV1().ConfigMaps(cd.Namespace).Update(primaryConfigMap)
if err != nil {
if errors.IsNotFound(err) {
_, err = ct.kubeClient.CoreV1().ConfigMaps(cd.Namespace).Create(primaryConfigMap)
if err != nil {
return err
}
} else {
return err
}
}

ct.logger.With("canary", fmt.Sprintf("%s.%s", cd.Name, cd.Namespace)).
Infof("ConfigMap %s synced", primaryConfigMap.GetName())
case ConfigRefSecret:
secret, err := ct.kubeClient.CoreV1().Secrets(cd.Namespace).Get(ref.Name, metav1.GetOptions{})
if err != nil {
return err
}
primaryName := fmt.Sprintf("%s-primary", secret.GetName())
primarySecret := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: primaryName,
Namespace: cd.Namespace,
Annotations: secret.Annotations,
OwnerReferences: []metav1.OwnerReference{
*metav1.NewControllerRef(cd, schema.GroupVersionKind{
Group: flaggerv1.SchemeGroupVersion.Group,
Version: flaggerv1.SchemeGroupVersion.Version,
Kind: flaggerv1.CanaryKind,
}),
},
},
Type: secret.Type,
Data: secret.Data,
}

// update or insert primary Secret
_, err = ct.kubeClient.CoreV1().Secrets(cd.Namespace).Update(primarySecret)
if err != nil {
if errors.IsNotFound(err) {
_, err = ct.kubeClient.CoreV1().Secrets(cd.Namespace).Create(primarySecret)
if err != nil {
return err
}
} else {
return err
}
}

ct.logger.With("canary", fmt.Sprintf("%s.%s", cd.Name, cd.Namespace)).
Infof("Secret %s synced", primarySecret.GetName())
}
}

return &ConfigRef{
Name: config.Name,
Type: ConfigRefMap,
Checksum: checksum(config.Data),
}, nil
return nil
}

func (ct *ConfigTracker) getRefFromSecret(name string, namespace string) (*ConfigRef, error) {
secret, err := ct.kubeClient.CoreV1().Secrets(namespace).Get(name, metav1.GetOptions{})
if err != nil {
return nil, err
}
// ApplyPrimaryConfigs appends the primary suffix to all ConfigMaps and Secretes found in the PodSpec
func (ct *ConfigTracker) ApplyPrimaryConfigs(spec corev1.PodSpec, refs map[string]ConfigRef) corev1.PodSpec {
// update volumes
for i, volume := range spec.Volumes {
if cmv := volume.ConfigMap; cmv != nil {
name := fmt.Sprintf("%s/%s", ConfigRefMap, cmv.Name)
if _, exists := refs[name]; exists {
spec.Volumes[i].ConfigMap.Name += "-primary"
}
}

if secret.Type != corev1.SecretTypeOpaque &&
secret.Type != corev1.SecretTypeBasicAuth &&
secret.Type != corev1.SecretTypeSSHAuth &&
secret.Type != corev1.SecretTypeTLS {
ct.logger.Debugf("ignoring secret %s.%s type not supported %v", name, namespace, secret.Type)
return nil, nil
if sv := volume.Secret; sv != nil {
name := fmt.Sprintf("%s/%s", ConfigRefSecret, sv.SecretName)
if _, exists := refs[name]; exists {
spec.Volumes[i].Secret.SecretName += "-primary"
}
}
}
// update containers
for _, container := range spec.Containers {
// update env
for i, env := range container.Env {
if env.ValueFrom != nil {
switch {
case env.ValueFrom.ConfigMapKeyRef != nil:
name := fmt.Sprintf("%s/%s", ConfigRefMap, env.ValueFrom.ConfigMapKeyRef.Name)
if _, exists := refs[name]; exists {
container.Env[i].ValueFrom.ConfigMapKeyRef.Name += "-primary"
}
case env.ValueFrom.SecretKeyRef != nil:
name := fmt.Sprintf("%s/%s", ConfigRefSecret, env.ValueFrom.SecretKeyRef.Name)
if _, exists := refs[name]; exists {
container.Env[i].ValueFrom.SecretKeyRef.Name += "-primary"
}
}
}
}
// update envFrom
for i, envFrom := range container.EnvFrom {
switch {
case envFrom.ConfigMapRef != nil:
name := fmt.Sprintf("%s/%s", ConfigRefMap, envFrom.ConfigMapRef.Name)
if _, exists := refs[name]; exists {
container.EnvFrom[i].ConfigMapRef.Name += "-primary"
}
case envFrom.SecretRef != nil:
name := fmt.Sprintf("%s/%s", ConfigRefSecret, envFrom.SecretRef.Name)
if _, exists := refs[name]; exists {
container.EnvFrom[i].SecretRef.Name += "-primary"
}
}
}
}

return &ConfigRef{
Name: secret.Name,
Type: ConfigRefSecret,
Checksum: checksum(secret.Data),
}, nil
return spec
}

0 comments on commit 71cd4e0

Please sign in to comment.