Skip to content

Commit

Permalink
Add tests for analysis restart and canary promotion
Browse files Browse the repository at this point in the history
  • Loading branch information
@stefanprodan
stefanprodan committed Jan 18, 2019
1 parent aff8b11 commit 6d8a734
Show file tree
Hide file tree
Showing 4 changed files with 234 additions and 23 deletions.
10 changes: 3 additions & 7 deletions pkg/controller/deployer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,15 +82,11 @@ func (c *CanaryDeployer) IsPrimaryReady(cd *flaggerv1.Canary) (bool, error) {

retriable, err := c.isDeploymentReady(primary, cd.GetProgressDeadlineSeconds())
if err != nil {
if retriable {
return retriable, fmt.Errorf("Halt %s.%s advancement %s", cd.Name, cd.Namespace, err.Error())
} else {
return retriable, err
}
return retriable, fmt.Errorf("Halt advancement %s.%s %s", primaryName, cd.Namespace, err.Error())
}

if primary.Spec.Replicas == int32p(0) {
return true, fmt.Errorf("halt %s.%s advancement primary deployment is scaled to zero",
return true, fmt.Errorf("Halt %s.%s advancement primary deployment is scaled to zero",
cd.Name, cd.Namespace)
}
return true, nil
Expand All @@ -112,7 +108,7 @@ func (c *CanaryDeployer) IsCanaryReady(cd *flaggerv1.Canary) (bool, error) {
retriable, err := c.isDeploymentReady(canary, cd.GetProgressDeadlineSeconds())
if err != nil {
if retriable {
return retriable, fmt.Errorf("Halt %s.%s advancement %s", cd.Name, cd.Namespace, err.Error())
return retriable, fmt.Errorf("Halt advancement %s.%s %s", targetName, cd.Namespace, err.Error())
} else {
return retriable, fmt.Errorf("deployment does not have minimum availability for more than %vs",
cd.GetProgressDeadlineSeconds())
Expand Down
7 changes: 4 additions & 3 deletions pkg/controller/job.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,8 @@ import "time"
type CanaryJob struct {
Name string
Namespace string
function func(name string, namespace string)
SkipTests bool
function func(name string, namespace string, skipTests bool)
done chan bool
ticker *time.Ticker
}
Expand All @@ -15,11 +16,11 @@ type CanaryJob struct {
func (j CanaryJob) Start() {
go func() {
// run the infra bootstrap on job creation
j.function(j.Name, j.Namespace)
j.function(j.Name, j.Namespace, j.SkipTests)
for {
select {
case <-j.ticker.C:
j.function(j.Name, j.Namespace)
j.function(j.Name, j.Namespace, j.SkipTests)
case <-j.done:
return
}
Expand Down
21 changes: 13 additions & 8 deletions pkg/controller/scheduler.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ func (c *Controller) scheduleCanaries() {
}
}

func (c *Controller) advanceCanary(name string, namespace string) {
func (c *Controller) advanceCanary(name string, namespace string, skipLivenessChecks bool) {
begin := time.Now()
// check if the canary exists
cd, err := c.flaggerClient.FlaggerV1alpha3().Canaries(namespace).Get(name, v1.GetOptions{})
Expand Down Expand Up @@ -104,9 +104,11 @@ func (c *Controller) advanceCanary(name string, namespace string) {
}

// check primary deployment status
if _, err := c.deployer.IsPrimaryReady(cd); err != nil {
c.recordEventWarningf(cd, "%v", err)
return
if !skipLivenessChecks {
if _, err := c.deployer.IsPrimaryReady(cd); err != nil {
c.recordEventWarningf(cd, "%v", err)
return
}
}

// check if virtual service exists
Expand Down Expand Up @@ -155,10 +157,13 @@ func (c *Controller) advanceCanary(name string, namespace string) {
}()

// check canary deployment status
retriable, err := c.deployer.IsCanaryReady(cd)
if err != nil && retriable {
c.recordEventWarningf(cd, "%v", err)
return
var retriable = true
if !skipLivenessChecks {
retriable, err = c.deployer.IsCanaryReady(cd)
if err != nil && retriable {
c.recordEventWarningf(cd, "%v", err)
return
}
}

// check if the number of failed checks reached the threshold
Expand Down
219 changes: 214 additions & 5 deletions pkg/controller/scheduler_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ func TestScheduler_Init(t *testing.T) {
}
ctrl.flaggerSynced = alwaysReady

ctrl.advanceCanary("podinfo", "default")
ctrl.advanceCanary("podinfo", "default", false)

_, err := kubeClient.AppsV1().Deployments("default").Get("podinfo-primary", metav1.GetOptions{})
if err != nil {
Expand Down Expand Up @@ -122,7 +122,7 @@ func TestScheduler_NewRevision(t *testing.T) {
ctrl.flaggerSynced = alwaysReady

// init
ctrl.advanceCanary("podinfo", "default")
ctrl.advanceCanary("podinfo", "default", false)

// update
dep2 := newTestDeploymentUpdated()
Expand All @@ -132,7 +132,7 @@ func TestScheduler_NewRevision(t *testing.T) {
}

// detect changes
ctrl.advanceCanary("podinfo", "default")
ctrl.advanceCanary("podinfo", "default", false)

c, err := kubeClient.AppsV1().Deployments("default").Get("podinfo", metav1.GetOptions{})
if err != nil {
Expand Down Expand Up @@ -191,7 +191,7 @@ func TestScheduler_Rollback(t *testing.T) {
ctrl.flaggerSynced = alwaysReady

// init
ctrl.advanceCanary("podinfo", "default")
ctrl.advanceCanary("podinfo", "default", true)

// update failed checks to max
err := deployer.SyncStatus(canary, v1alpha3.CanaryStatus{Phase: v1alpha3.CanaryProgressing, FailedChecks: 11})
Expand All @@ -200,7 +200,7 @@ func TestScheduler_Rollback(t *testing.T) {
}

// detect changes
ctrl.advanceCanary("podinfo", "default")
ctrl.advanceCanary("podinfo", "default", true)

c, err := flaggerClient.FlaggerV1alpha3().Canaries("default").Get("podinfo", metav1.GetOptions{})
if err != nil {
Expand All @@ -211,3 +211,212 @@ func TestScheduler_Rollback(t *testing.T) {
t.Errorf("Got canary state %v wanted %v", c.Status.Phase, v1alpha3.CanaryFailed)
}
}

func TestScheduler_NewRevisionReset(t *testing.T) {
canary := newTestCanary()
dep := newTestDeployment()
hpa := newTestHPA()

flaggerClient := fakeFlagger.NewSimpleClientset(canary)
kubeClient := fake.NewSimpleClientset(dep, hpa)
istioClient := fakeIstio.NewSimpleClientset()

logger, _ := logging.NewLogger("debug")
deployer := CanaryDeployer{
flaggerClient: flaggerClient,
kubeClient: kubeClient,
logger: logger,
}
router := CanaryRouter{
flaggerClient: flaggerClient,
kubeClient: kubeClient,
istioClient: istioClient,
logger: logger,
}
observer := CanaryObserver{
metricsServer: "fake",
}

flaggerInformerFactory := informers.NewSharedInformerFactory(flaggerClient, noResyncPeriodFunc())
flaggerInformer := flaggerInformerFactory.Flagger().V1alpha3().Canaries()

ctrl := &Controller{
kubeClient: kubeClient,
istioClient: istioClient,
flaggerClient: flaggerClient,
flaggerLister: flaggerInformer.Lister(),
flaggerSynced: flaggerInformer.Informer().HasSynced,
workqueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), controllerAgentName),
eventRecorder: &record.FakeRecorder{},
logger: logger,
canaries: new(sync.Map),
flaggerWindow: time.Second,
deployer: deployer,
router: router,
observer: observer,
recorder: NewCanaryRecorder(false),
}
ctrl.flaggerSynced = alwaysReady

// init
ctrl.advanceCanary("podinfo", "default", false)

// first update
dep2 := newTestDeploymentUpdated()
_, err := kubeClient.AppsV1().Deployments("default").Update(dep2)
if err != nil {
t.Fatal(err.Error())
}

// detect changes
ctrl.advanceCanary("podinfo", "default", true)
// advance
ctrl.advanceCanary("podinfo", "default", true)

primaryRoute, canaryRoute, err := router.GetRoutes(canary)
if err != nil {
t.Fatal(err.Error())
}

if primaryRoute.Weight != 90 {
t.Errorf("Got primary route %v wanted %v", primaryRoute.Weight, 90)
}

if canaryRoute.Weight != 10 {
t.Errorf("Got canary route %v wanted %v", canaryRoute.Weight, 10)
}

// second update
dep2.Spec.Template.Spec.ServiceAccountName = "test"
_, err = kubeClient.AppsV1().Deployments("default").Update(dep2)
if err != nil {
t.Fatal(err.Error())
}

// detect changes
ctrl.advanceCanary("podinfo", "default", true)

primaryRoute, canaryRoute, err = router.GetRoutes(canary)
if err != nil {
t.Fatal(err.Error())
}

if primaryRoute.Weight != 100 {
t.Errorf("Got primary route %v wanted %v", primaryRoute.Weight, 100)
}

if canaryRoute.Weight != 0 {
t.Errorf("Got canary route %v wanted %v", canaryRoute.Weight, 0)
}
}

func TestScheduler_Promotion(t *testing.T) {
canary := newTestCanary()
dep := newTestDeployment()
hpa := newTestHPA()

flaggerClient := fakeFlagger.NewSimpleClientset(canary)
kubeClient := fake.NewSimpleClientset(dep, hpa)
istioClient := fakeIstio.NewSimpleClientset()

logger, _ := logging.NewLogger("debug")
deployer := CanaryDeployer{
flaggerClient: flaggerClient,
kubeClient: kubeClient,
logger: logger,
}
router := CanaryRouter{
flaggerClient: flaggerClient,
kubeClient: kubeClient,
istioClient: istioClient,
logger: logger,
}
observer := CanaryObserver{
metricsServer: "fake",
}

flaggerInformerFactory := informers.NewSharedInformerFactory(flaggerClient, noResyncPeriodFunc())
flaggerInformer := flaggerInformerFactory.Flagger().V1alpha3().Canaries()

ctrl := &Controller{
kubeClient: kubeClient,
istioClient: istioClient,
flaggerClient: flaggerClient,
flaggerLister: flaggerInformer.Lister(),
flaggerSynced: flaggerInformer.Informer().HasSynced,
workqueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), controllerAgentName),
eventRecorder: &record.FakeRecorder{},
logger: logger,
canaries: new(sync.Map),
flaggerWindow: time.Second,
deployer: deployer,
router: router,
observer: observer,
recorder: NewCanaryRecorder(false),
}
ctrl.flaggerSynced = alwaysReady

// init
ctrl.advanceCanary("podinfo", "default", false)

// update
dep2 := newTestDeploymentUpdated()
_, err := kubeClient.AppsV1().Deployments("default").Update(dep2)
if err != nil {
t.Fatal(err.Error())
}

// detect changes
ctrl.advanceCanary("podinfo", "default", true)

primaryRoute, canaryRoute, err := router.GetRoutes(canary)
if err != nil {
t.Fatal(err.Error())
}

primaryRoute.Weight = 60
canaryRoute.Weight = 40
err = ctrl.router.SetRoutes(canary, primaryRoute, canaryRoute)
if err != nil {
t.Fatal(err.Error())
}

// advance
ctrl.advanceCanary("podinfo", "default", true)

// promote
ctrl.advanceCanary("podinfo", "default", true)

primaryRoute, canaryRoute, err = router.GetRoutes(canary)
if err != nil {
t.Fatal(err.Error())
}

if primaryRoute.Weight != 100 {
t.Errorf("Got primary route %v wanted %v", primaryRoute.Weight, 100)
}

if canaryRoute.Weight != 0 {
t.Errorf("Got canary route %v wanted %v", canaryRoute.Weight, 0)
}

primaryDep, err := kubeClient.AppsV1().Deployments("default").Get("podinfo-primary", metav1.GetOptions{})
if err != nil {
t.Fatal(err.Error())
}

primaryImage := primaryDep.Spec.Template.Spec.Containers[0].Image
canaryImage := dep2.Spec.Template.Spec.Containers[0].Image
if primaryImage != canaryImage {
t.Errorf("Got primary image %v wanted %v", primaryImage, canaryImage)
}

c, err := flaggerClient.FlaggerV1alpha3().Canaries("default").Get("podinfo", metav1.GetOptions{})
if err != nil {
t.Fatal(err.Error())
}

if c.Status.Phase != v1alpha3.CanarySucceeded {
t.Errorf("Got canary state %v wanted %v", c.Status.Phase, v1alpha3.CanarySucceeded)
}
}

0 comments on commit 6d8a734

Please sign in to comment.