diff --git a/pkg/controller/deployer.go b/pkg/controller/deployer.go index a5d65a3ec..3eed041a0 100644 --- a/pkg/controller/deployer.go +++ b/pkg/controller/deployer.go @@ -82,15 +82,11 @@ func (c *CanaryDeployer) IsPrimaryReady(cd *flaggerv1.Canary) (bool, error) { retriable, err := c.isDeploymentReady(primary, cd.GetProgressDeadlineSeconds()) if err != nil { - if retriable { - return retriable, fmt.Errorf("Halt %s.%s advancement %s", cd.Name, cd.Namespace, err.Error()) - } else { - return retriable, err - } + return retriable, fmt.Errorf("Halt advancement %s.%s %s", primaryName, cd.Namespace, err.Error()) } if primary.Spec.Replicas == int32p(0) { - return true, fmt.Errorf("halt %s.%s advancement primary deployment is scaled to zero", + return true, fmt.Errorf("Halt %s.%s advancement primary deployment is scaled to zero", cd.Name, cd.Namespace) } return true, nil @@ -112,7 +108,7 @@ func (c *CanaryDeployer) IsCanaryReady(cd *flaggerv1.Canary) (bool, error) { retriable, err := c.isDeploymentReady(canary, cd.GetProgressDeadlineSeconds()) if err != nil { if retriable { - return retriable, fmt.Errorf("Halt %s.%s advancement %s", cd.Name, cd.Namespace, err.Error()) + return retriable, fmt.Errorf("Halt advancement %s.%s %s", targetName, cd.Namespace, err.Error()) } else { return retriable, fmt.Errorf("deployment does not have minimum availability for more than %vs", cd.GetProgressDeadlineSeconds()) diff --git a/pkg/controller/job.go b/pkg/controller/job.go index f44a9abc8..ead2415e4 100644 --- a/pkg/controller/job.go +++ b/pkg/controller/job.go @@ -6,7 +6,8 @@ import "time" type CanaryJob struct { Name string Namespace string - function func(name string, namespace string) + SkipTests bool + function func(name string, namespace string, skipTests bool) done chan bool ticker *time.Ticker } @@ -15,11 +16,11 @@ type CanaryJob struct { func (j CanaryJob) Start() { go func() { // run the infra bootstrap on job creation - j.function(j.Name, j.Namespace) + j.function(j.Name, j.Namespace, j.SkipTests) for { select { case <-j.ticker.C: - j.function(j.Name, j.Namespace) + j.function(j.Name, j.Namespace, j.SkipTests) case <-j.done: return } diff --git a/pkg/controller/scheduler.go b/pkg/controller/scheduler.go index 40c7d38bf..0fd9f1c42 100644 --- a/pkg/controller/scheduler.go +++ b/pkg/controller/scheduler.go @@ -69,7 +69,7 @@ func (c *Controller) scheduleCanaries() { } } -func (c *Controller) advanceCanary(name string, namespace string) { +func (c *Controller) advanceCanary(name string, namespace string, skipLivenessChecks bool) { begin := time.Now() // check if the canary exists cd, err := c.flaggerClient.FlaggerV1alpha3().Canaries(namespace).Get(name, v1.GetOptions{}) @@ -104,9 +104,11 @@ func (c *Controller) advanceCanary(name string, namespace string) { } // check primary deployment status - if _, err := c.deployer.IsPrimaryReady(cd); err != nil { - c.recordEventWarningf(cd, "%v", err) - return + if !skipLivenessChecks { + if _, err := c.deployer.IsPrimaryReady(cd); err != nil { + c.recordEventWarningf(cd, "%v", err) + return + } } // check if virtual service exists @@ -155,10 +157,13 @@ func (c *Controller) advanceCanary(name string, namespace string) { }() // check canary deployment status - retriable, err := c.deployer.IsCanaryReady(cd) - if err != nil && retriable { - c.recordEventWarningf(cd, "%v", err) - return + var retriable = true + if !skipLivenessChecks { + retriable, err = c.deployer.IsCanaryReady(cd) + if err != nil && retriable { + c.recordEventWarningf(cd, "%v", err) + return + } } // check if the number of failed checks reached the threshold diff --git a/pkg/controller/scheduler_test.go b/pkg/controller/scheduler_test.go index b796cf5fb..6a4dfc9b1 100644 --- a/pkg/controller/scheduler_test.go +++ b/pkg/controller/scheduler_test.go @@ -67,7 +67,7 @@ func TestScheduler_Init(t *testing.T) { } ctrl.flaggerSynced = alwaysReady - ctrl.advanceCanary("podinfo", "default") + ctrl.advanceCanary("podinfo", "default", false) _, err := kubeClient.AppsV1().Deployments("default").Get("podinfo-primary", metav1.GetOptions{}) if err != nil { @@ -122,7 +122,7 @@ func TestScheduler_NewRevision(t *testing.T) { ctrl.flaggerSynced = alwaysReady // init - ctrl.advanceCanary("podinfo", "default") + ctrl.advanceCanary("podinfo", "default", false) // update dep2 := newTestDeploymentUpdated() @@ -132,7 +132,7 @@ func TestScheduler_NewRevision(t *testing.T) { } // detect changes - ctrl.advanceCanary("podinfo", "default") + ctrl.advanceCanary("podinfo", "default", false) c, err := kubeClient.AppsV1().Deployments("default").Get("podinfo", metav1.GetOptions{}) if err != nil { @@ -191,7 +191,7 @@ func TestScheduler_Rollback(t *testing.T) { ctrl.flaggerSynced = alwaysReady // init - ctrl.advanceCanary("podinfo", "default") + ctrl.advanceCanary("podinfo", "default", true) // update failed checks to max err := deployer.SyncStatus(canary, v1alpha3.CanaryStatus{Phase: v1alpha3.CanaryProgressing, FailedChecks: 11}) @@ -200,7 +200,7 @@ func TestScheduler_Rollback(t *testing.T) { } // detect changes - ctrl.advanceCanary("podinfo", "default") + ctrl.advanceCanary("podinfo", "default", true) c, err := flaggerClient.FlaggerV1alpha3().Canaries("default").Get("podinfo", metav1.GetOptions{}) if err != nil { @@ -211,3 +211,212 @@ func TestScheduler_Rollback(t *testing.T) { t.Errorf("Got canary state %v wanted %v", c.Status.Phase, v1alpha3.CanaryFailed) } } + +func TestScheduler_NewRevisionReset(t *testing.T) { + canary := newTestCanary() + dep := newTestDeployment() + hpa := newTestHPA() + + flaggerClient := fakeFlagger.NewSimpleClientset(canary) + kubeClient := fake.NewSimpleClientset(dep, hpa) + istioClient := fakeIstio.NewSimpleClientset() + + logger, _ := logging.NewLogger("debug") + deployer := CanaryDeployer{ + flaggerClient: flaggerClient, + kubeClient: kubeClient, + logger: logger, + } + router := CanaryRouter{ + flaggerClient: flaggerClient, + kubeClient: kubeClient, + istioClient: istioClient, + logger: logger, + } + observer := CanaryObserver{ + metricsServer: "fake", + } + + flaggerInformerFactory := informers.NewSharedInformerFactory(flaggerClient, noResyncPeriodFunc()) + flaggerInformer := flaggerInformerFactory.Flagger().V1alpha3().Canaries() + + ctrl := &Controller{ + kubeClient: kubeClient, + istioClient: istioClient, + flaggerClient: flaggerClient, + flaggerLister: flaggerInformer.Lister(), + flaggerSynced: flaggerInformer.Informer().HasSynced, + workqueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), controllerAgentName), + eventRecorder: &record.FakeRecorder{}, + logger: logger, + canaries: new(sync.Map), + flaggerWindow: time.Second, + deployer: deployer, + router: router, + observer: observer, + recorder: NewCanaryRecorder(false), + } + ctrl.flaggerSynced = alwaysReady + + // init + ctrl.advanceCanary("podinfo", "default", false) + + // first update + dep2 := newTestDeploymentUpdated() + _, err := kubeClient.AppsV1().Deployments("default").Update(dep2) + if err != nil { + t.Fatal(err.Error()) + } + + // detect changes + ctrl.advanceCanary("podinfo", "default", true) + // advance + ctrl.advanceCanary("podinfo", "default", true) + + primaryRoute, canaryRoute, err := router.GetRoutes(canary) + if err != nil { + t.Fatal(err.Error()) + } + + if primaryRoute.Weight != 90 { + t.Errorf("Got primary route %v wanted %v", primaryRoute.Weight, 90) + } + + if canaryRoute.Weight != 10 { + t.Errorf("Got canary route %v wanted %v", canaryRoute.Weight, 10) + } + + // second update + dep2.Spec.Template.Spec.ServiceAccountName = "test" + _, err = kubeClient.AppsV1().Deployments("default").Update(dep2) + if err != nil { + t.Fatal(err.Error()) + } + + // detect changes + ctrl.advanceCanary("podinfo", "default", true) + + primaryRoute, canaryRoute, err = router.GetRoutes(canary) + if err != nil { + t.Fatal(err.Error()) + } + + if primaryRoute.Weight != 100 { + t.Errorf("Got primary route %v wanted %v", primaryRoute.Weight, 100) + } + + if canaryRoute.Weight != 0 { + t.Errorf("Got canary route %v wanted %v", canaryRoute.Weight, 0) + } +} + +func TestScheduler_Promotion(t *testing.T) { + canary := newTestCanary() + dep := newTestDeployment() + hpa := newTestHPA() + + flaggerClient := fakeFlagger.NewSimpleClientset(canary) + kubeClient := fake.NewSimpleClientset(dep, hpa) + istioClient := fakeIstio.NewSimpleClientset() + + logger, _ := logging.NewLogger("debug") + deployer := CanaryDeployer{ + flaggerClient: flaggerClient, + kubeClient: kubeClient, + logger: logger, + } + router := CanaryRouter{ + flaggerClient: flaggerClient, + kubeClient: kubeClient, + istioClient: istioClient, + logger: logger, + } + observer := CanaryObserver{ + metricsServer: "fake", + } + + flaggerInformerFactory := informers.NewSharedInformerFactory(flaggerClient, noResyncPeriodFunc()) + flaggerInformer := flaggerInformerFactory.Flagger().V1alpha3().Canaries() + + ctrl := &Controller{ + kubeClient: kubeClient, + istioClient: istioClient, + flaggerClient: flaggerClient, + flaggerLister: flaggerInformer.Lister(), + flaggerSynced: flaggerInformer.Informer().HasSynced, + workqueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), controllerAgentName), + eventRecorder: &record.FakeRecorder{}, + logger: logger, + canaries: new(sync.Map), + flaggerWindow: time.Second, + deployer: deployer, + router: router, + observer: observer, + recorder: NewCanaryRecorder(false), + } + ctrl.flaggerSynced = alwaysReady + + // init + ctrl.advanceCanary("podinfo", "default", false) + + // update + dep2 := newTestDeploymentUpdated() + _, err := kubeClient.AppsV1().Deployments("default").Update(dep2) + if err != nil { + t.Fatal(err.Error()) + } + + // detect changes + ctrl.advanceCanary("podinfo", "default", true) + + primaryRoute, canaryRoute, err := router.GetRoutes(canary) + if err != nil { + t.Fatal(err.Error()) + } + + primaryRoute.Weight = 60 + canaryRoute.Weight = 40 + err = ctrl.router.SetRoutes(canary, primaryRoute, canaryRoute) + if err != nil { + t.Fatal(err.Error()) + } + + // advance + ctrl.advanceCanary("podinfo", "default", true) + + // promote + ctrl.advanceCanary("podinfo", "default", true) + + primaryRoute, canaryRoute, err = router.GetRoutes(canary) + if err != nil { + t.Fatal(err.Error()) + } + + if primaryRoute.Weight != 100 { + t.Errorf("Got primary route %v wanted %v", primaryRoute.Weight, 100) + } + + if canaryRoute.Weight != 0 { + t.Errorf("Got canary route %v wanted %v", canaryRoute.Weight, 0) + } + + primaryDep, err := kubeClient.AppsV1().Deployments("default").Get("podinfo-primary", metav1.GetOptions{}) + if err != nil { + t.Fatal(err.Error()) + } + + primaryImage := primaryDep.Spec.Template.Spec.Containers[0].Image + canaryImage := dep2.Spec.Template.Spec.Containers[0].Image + if primaryImage != canaryImage { + t.Errorf("Got primary image %v wanted %v", primaryImage, canaryImage) + } + + c, err := flaggerClient.FlaggerV1alpha3().Canaries("default").Get("podinfo", metav1.GetOptions{}) + if err != nil { + t.Fatal(err.Error()) + } + + if c.Status.Phase != v1alpha3.CanarySucceeded { + t.Errorf("Got canary state %v wanted %v", c.Status.Phase, v1alpha3.CanarySucceeded) + } +} \ No newline at end of file