From e86c02d6001c192223720587d624894caa5783fa Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Wed, 26 Dec 2018 14:41:35 +0200 Subject: [PATCH] Implement canary external check - do a HTTP POST for each webhook registered in the canary analysis - increment the failed checks counter if a webhook returns a non-2xx status code and log the error and the response body if exists --- pkg/apis/flagger/v1alpha2/types.go | 7 ++ .../flagger/v1alpha2/zz_generated.deepcopy.go | 27 +++++++ pkg/controller/scheduler.go | 15 +++- pkg/controller/webhook.go | 71 +++++++++++++++++++ 4 files changed, 118 insertions(+), 2 deletions(-) create mode 100644 pkg/controller/webhook.go diff --git a/pkg/apis/flagger/v1alpha2/types.go b/pkg/apis/flagger/v1alpha2/types.go index 270ad75a7..fc71ff917 100755 --- a/pkg/apis/flagger/v1alpha2/types.go +++ b/pkg/apis/flagger/v1alpha2/types.go @@ -119,6 +119,13 @@ type CanaryWebhook struct { Metadata *map[string]string `json:"metadata,omitempty"` } +// CanaryWebhookPayload holds the deployment info and metadata sent to webhooks +type CanaryWebhookPayload struct { + Name string `json:"name"` + Namespace string `json:"namespace"` + Metadata *map[string]string `json:"metadata,omitempty"` +} + // GetProgressDeadlineSeconds returns the progress deadline (default 600s) func (c *Canary) GetProgressDeadlineSeconds() int { if c.Spec.ProgressDeadlineSeconds != nil { diff --git a/pkg/apis/flagger/v1alpha2/zz_generated.deepcopy.go b/pkg/apis/flagger/v1alpha2/zz_generated.deepcopy.go index 7df95a245..698ddda3a 100644 --- a/pkg/apis/flagger/v1alpha2/zz_generated.deepcopy.go +++ b/pkg/apis/flagger/v1alpha2/zz_generated.deepcopy.go @@ -223,3 +223,30 @@ func (in *CanaryWebhook) DeepCopy() *CanaryWebhook { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CanaryWebhookPayload) DeepCopyInto(out *CanaryWebhookPayload) { + *out = *in + if in.Metadata != nil { + in, out := &in.Metadata, &out.Metadata + *out = new(map[string]string) + if **in != nil { + in, out := *in, *out + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CanaryWebhookPayload. +func (in *CanaryWebhookPayload) DeepCopy() *CanaryWebhookPayload { + if in == nil { + return nil + } + out := new(CanaryWebhookPayload) + in.DeepCopyInto(out) + return out +} diff --git a/pkg/controller/scheduler.go b/pkg/controller/scheduler.go index ee4af8d55..6552898ca 100644 --- a/pkg/controller/scheduler.go +++ b/pkg/controller/scheduler.go @@ -139,7 +139,7 @@ func (c *Controller) advanceCanary(name string, namespace string) { if canaryRoute.Weight == 0 { c.recordEventInfof(cd, "Starting canary deployment for %s.%s", cd.Name, cd.Namespace) } else { - if ok := c.checkCanaryMetrics(cd); !ok { + if ok := c.analyseCanary(cd); !ok { if err := c.deployer.SetFailedChecks(cd, cd.Status.FailedChecks+1); err != nil { c.recordEventWarningf(cd, "%v", err) return @@ -242,7 +242,8 @@ func (c *Controller) checkCanaryStatus(cd *flaggerv1.Canary, deployer CanaryDepl return false } -func (c *Controller) checkCanaryMetrics(r *flaggerv1.Canary) bool { +func (c *Controller) analyseCanary(r *flaggerv1.Canary) bool { + // run metrics checks for _, metric := range r.Spec.CanaryAnalysis.Metrics { if metric.Name == "istio_requests_total" { val, err := c.observer.GetDeploymentCounter(r.Spec.TargetRef.Name, r.Namespace, metric.Name, metric.Interval) @@ -272,5 +273,15 @@ func (c *Controller) checkCanaryMetrics(r *flaggerv1.Canary) bool { } } + // run external checks + for _, webhook := range r.Spec.CanaryAnalysis.Webhooks { + err := CallWebhook(r.Name, r.Namespace, webhook) + if err != nil { + c.recordEventWarningf(r, "Halt %s.%s advancement external check %s failed %v", + r.Name, r.Namespace, webhook.Name, err) + return false + } + } + return true } diff --git a/pkg/controller/webhook.go b/pkg/controller/webhook.go new file mode 100644 index 000000000..ef4b29005 --- /dev/null +++ b/pkg/controller/webhook.go @@ -0,0 +1,71 @@ +package controller + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + flaggerv1 "github.com/stefanprodan/flagger/pkg/apis/flagger/v1alpha2" + "io/ioutil" + "net/http" + "net/url" + "time" +) + +// CallWebhook does a HTTP POST to an external service and +// returns an error if the response status code is non-2xx +func CallWebhook(name string, namepace string, w flaggerv1.CanaryWebhook) error { + + payload := flaggerv1.CanaryWebhookPayload{ + Name: name, + Namespace: namepace, + Metadata: w.Metadata, + } + + payloadBin, err := json.Marshal(payload) + if err != nil { + return err + } + + hook, err := url.Parse(w.URL) + if err != nil { + return err + } + + req, err := http.NewRequest("POST", hook.String(), bytes.NewBuffer(payloadBin)) + if err != nil { + return err + } + + req.Header.Set("Content-Type", "application/json") + + if len(w.Timeout) < 2 { + w.Timeout = "10s" + } + + timeout, err := time.ParseDuration(w.Timeout) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(req.Context(), timeout) + defer cancel() + + r, err := http.DefaultClient.Do(req.WithContext(ctx)) + if err != nil { + return err + } + defer r.Body.Close() + + b, err := ioutil.ReadAll(r.Body) + if err != nil { + return fmt.Errorf("error reading body: %s", err.Error()) + } + + if r.StatusCode > 202 { + return errors.New(string(b)) + } + + return nil +}