-
Notifications
You must be signed in to change notification settings - Fork 727
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #53 from stefanprodan/kind
Add CircleCI workflow for end-to-end testing with Kubernetes Kind
- Loading branch information
Showing
12 changed files
with
352 additions
and
7 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,16 @@ | ||
version: 2.1 | ||
jobs: | ||
e2e-testing: | ||
machine: true | ||
steps: | ||
- checkout | ||
- run: test/e2e-kind.sh | ||
- run: test/e2e-istio.sh | ||
- run: test/e2e-build.sh | ||
- run: test/e2e-tests.sh | ||
|
||
workflows: | ||
version: 2 | ||
build-and-test: | ||
jobs: | ||
- e2e-testing |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,4 @@ | ||
FROM golang:1.11 | ||
|
||
RUN go get -u sigs.k8s.io/kind | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,24 @@ | ||
# Flagger end-to-end testing | ||
|
||
The e2e testing infrastructure is powered by CircleCI and [Kubernetes Kind](https://github.com/kubernetes-sigs/kind). | ||
|
||
CircleCI e2e workflow: | ||
|
||
* install latest stable kubectl [e2e-kind.sh](e2e-kind.sh) | ||
* build Kubernetes Kind from master [e2e-kind.sh](e2e-kind.sh) | ||
* create local Kubernetes cluster with kind [e2e-kind.sh](e2e-kind.sh) | ||
* install latest stable Helm CLI [e2e-istio.sh](e2e-istio.sh) | ||
* deploy Tiller on the local cluster [e2e-istio.sh](e2e-istio.sh) | ||
* install Istio CRDs with Helm [e2e-istio.sh](e2e-istio.sh) | ||
* install Istio control plane and Prometheus with Helm [e2e-istio.sh](e2e-istio.sh) | ||
* build Flagger container image [e2e-build.sh](e2e-build.sh) | ||
* load Flagger image onto the local cluster [e2e-build.sh](e2e-build.sh) | ||
* deploy Flagger in the istio-system namespace [e2e-build.sh](e2e-build.sh) | ||
* create a test namespace with Istio injection enabled [e2e-tests.sh](e2e-tests.sh) | ||
* deploy the load tester in the test namespace [e2e-tests.sh](e2e-tests.sh) | ||
* deploy a demo workload (podinfo) in the test namespace [e2e-tests.sh](e2e-tests.sh) | ||
* test the canary initialization [e2e-tests.sh](e2e-tests.sh) | ||
* test the canary analysis and promotion [e2e-tests.sh](e2e-tests.sh) | ||
|
||
|
||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,15 @@ | ||
#!/usr/bin/env bash | ||
|
||
set -o errexit | ||
|
||
REPO_ROOT=$(git rev-parse --show-toplevel) | ||
export KUBECONFIG="$(kind get kubeconfig-path --name="kind")" | ||
|
||
echo '>>> Building Flagger' | ||
cd ${REPO_ROOT} && docker build -t test/flagger:latest . -f Dockerfile | ||
|
||
echo '>>> Installing Flagger' | ||
kind load docker-image test/flagger:latest | ||
kubectl apply -f ${REPO_ROOT}/artifacts/flagger/ | ||
kubectl -n istio-system set image deployment/flagger flagger=test/flagger:latest | ||
kubectl -n istio-system rollout status deployment/flagger |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,62 @@ | ||
# | ||
# Minimal Istio Configuration required by Flagger | ||
# | ||
|
||
# pilot configuration | ||
pilot: | ||
enabled: true | ||
sidecar: true | ||
|
||
gateways: | ||
enabled: false | ||
istio-ingressgateway: | ||
autoscaleMax: 1 | ||
|
||
# citadel configuration | ||
security: | ||
enabled: true | ||
|
||
# sidecar-injector webhook configuration | ||
sidecarInjectorWebhook: | ||
enabled: true | ||
|
||
# galley configuration | ||
galley: | ||
enabled: false | ||
|
||
# mixer configuration | ||
mixer: | ||
policy: | ||
enabled: false | ||
replicaCount: 1 | ||
autoscaleEnabled: false | ||
telemetry: | ||
enabled: true | ||
replicaCount: 1 | ||
autoscaleEnabled: false | ||
resources: | ||
requests: | ||
cpu: 100m | ||
memory: 128Mi | ||
|
||
# addon prometheus configuration | ||
prometheus: | ||
enabled: true | ||
scrapeInterval: 5s | ||
|
||
# addon jaeger tracing configuration | ||
tracing: | ||
enabled: false | ||
|
||
# Common settings. | ||
global: | ||
proxy: | ||
# Resources for the sidecar. | ||
resources: | ||
requests: | ||
cpu: 100m | ||
memory: 128Mi | ||
limits: | ||
cpu: 2000m | ||
memory: 128Mi | ||
useMCP: false |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,28 @@ | ||
#!/usr/bin/env bash | ||
|
||
set -o errexit | ||
|
||
ISTIO_VER="1.1.0-rc.0" | ||
REPO_ROOT=$(git rev-parse --show-toplevel) | ||
export KUBECONFIG="$(kind get kubeconfig-path --name="kind")" | ||
|
||
echo ">>> Installing Helm" | ||
curl https://github.com/raw/kubernetes/helm/master/scripts/get | bash | ||
|
||
echo '>>> Installing Tiller' | ||
kubectl --namespace kube-system create sa tiller | ||
kubectl create clusterrolebinding tiller-cluster-rule --clusterrole=cluster-admin --serviceaccount=kube-system:tiller | ||
helm init --service-account tiller --upgrade --wait | ||
|
||
echo ">>> Installing Istio ${ISTIO_VER}" | ||
helm repo add istio.io https://storage.googleapis.com/istio-release/releases/${ISTIO_VER}/charts | ||
|
||
echo '>>> Installing Istio CRDs' | ||
helm upgrade -i istio-init istio.io/istio-init --wait --namespace istio-system | ||
|
||
echo '>>> Waiting for Istio CRDs to be ready' | ||
kubectl -n istio-system wait --for=condition=complete job/istio-init-crd-10 | ||
kubectl -n istio-system wait --for=condition=complete job/istio-init-crd-11 | ||
|
||
echo '>>> Installing Istio control plane' | ||
helm upgrade -i istio istio.io/istio --wait --namespace istio-system -f ${REPO_ROOT}/test/e2e-istio-values.yaml |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,25 @@ | ||
#!/usr/bin/env bash | ||
|
||
set -o errexit | ||
|
||
REPO_ROOT=$(git rev-parse --show-toplevel) | ||
|
||
echo ">>> Installing kubectl" | ||
curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl && \ | ||
chmod +x kubectl && \ | ||
sudo mv kubectl /usr/local/bin/ | ||
|
||
echo ">>> Building sigs.k8s.io/kind" | ||
docker build -t kind:src . -f ${REPO_ROOT}/test/Dockerfile.kind | ||
docker create -ti --name dummy kind:src sh | ||
docker cp dummy:/go/bin/kind ./kind | ||
docker rm -f dummy | ||
|
||
echo ">>> Installing kind" | ||
chmod +x kind | ||
sudo mv kind /usr/local/bin/ | ||
kind create cluster --wait 5m | ||
|
||
export KUBECONFIG="$(kind get kubeconfig-path --name="kind")" | ||
kubectl get pods --all-namespaces | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,99 @@ | ||
#!/usr/bin/env bash | ||
|
||
# This script runs e2e tests for Canary initialization, analysis and promotion | ||
# Prerequisites: Kubernetes Kind, Helm and Istio | ||
|
||
set -o errexit | ||
|
||
REPO_ROOT=$(git rev-parse --show-toplevel) | ||
export KUBECONFIG="$(kind get kubeconfig-path --name="kind")" | ||
|
||
echo '>>> Creating test namespace' | ||
kubectl create namespace test | ||
kubectl label namespace test istio-injection=enabled | ||
|
||
echo '>>> Installing the load tester' | ||
kubectl -n test apply -f ${REPO_ROOT}/artifacts/loadtester/ | ||
kubectl -n test rollout status deployment/flagger-loadtester | ||
|
||
echo '>>> Initialising canary' | ||
kubectl apply -f ${REPO_ROOT}/test/e2e-workload.yaml | ||
|
||
cat <<EOF | kubectl apply -f - | ||
apiVersion: flagger.app/v1alpha3 | ||
kind: Canary | ||
metadata: | ||
name: podinfo | ||
namespace: test | ||
spec: | ||
targetRef: | ||
apiVersion: apps/v1 | ||
kind: Deployment | ||
name: podinfo | ||
progressDeadlineSeconds: 60 | ||
service: | ||
port: 9898 | ||
canaryAnalysis: | ||
interval: 15s | ||
threshold: 15 | ||
maxWeight: 50 | ||
stepWeight: 10 | ||
metrics: | ||
- name: istio_requests_total | ||
threshold: 99 | ||
interval: 1m | ||
- name: istio_request_duration_seconds_bucket | ||
threshold: 500 | ||
interval: 30s | ||
webhooks: | ||
- name: load-test | ||
url: http://flagger-loadtester.test/ | ||
timeout: 5s | ||
metadata: | ||
cmd: "hey -z 10m -q 10 -c 2 http://podinfo.test:9898/" | ||
EOF | ||
|
||
echo '>>> Waiting for primary to be ready' | ||
retries=50 | ||
count=0 | ||
ok=false | ||
until ${ok}; do | ||
kubectl -n test get canary/podinfo | grep 'Initialized' && ok=true || ok=false | ||
sleep 5 | ||
count=$(($count + 1)) | ||
if [[ ${count} -eq ${retries} ]]; then | ||
kubectl -n istio-system logs deployment/flagger | ||
echo "No more retries left" | ||
exit 1 | ||
fi | ||
done | ||
|
||
echo '✔ Canary initialization test passed' | ||
|
||
echo '>>> Triggering canary deployment' | ||
kubectl -n test set image deployment/podinfo podinfod=quay.io/stefanprodan/podinfo:1.4.1 | ||
|
||
echo '>>> Waiting for canary promotion' | ||
retries=50 | ||
count=0 | ||
ok=false | ||
until ${ok}; do | ||
kubectl -n test describe deployment/podinfo-primary | grep '1.4.1' && ok=true || ok=false | ||
sleep 10 | ||
kubectl -n istio-system logs deployment/flagger --tail 1 | ||
count=$(($count + 1)) | ||
if [[ ${count} -eq ${retries} ]]; then | ||
kubectl -n test describe deployment/podinfo | ||
kubectl -n test describe deployment/podinfo-primary | ||
kubectl -n istio-system logs deployment/flagger | ||
echo "No more retries left" | ||
exit 1 | ||
fi | ||
done | ||
|
||
echo '✔ Canary promotion test passed' | ||
|
||
kubectl -n istio-system logs deployment/flagger | ||
|
||
echo '✔ All tests passed' |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,67 @@ | ||
apiVersion: apps/v1 | ||
kind: Deployment | ||
metadata: | ||
name: podinfo | ||
namespace: test | ||
labels: | ||
app: podinfo | ||
spec: | ||
minReadySeconds: 5 | ||
revisionHistoryLimit: 5 | ||
progressDeadlineSeconds: 60 | ||
strategy: | ||
rollingUpdate: | ||
maxUnavailable: 0 | ||
type: RollingUpdate | ||
selector: | ||
matchLabels: | ||
app: podinfo | ||
template: | ||
metadata: | ||
annotations: | ||
prometheus.io/scrape: "true" | ||
labels: | ||
app: podinfo | ||
spec: | ||
containers: | ||
- name: podinfod | ||
image: quay.io/stefanprodan/podinfo:1.4.0 | ||
imagePullPolicy: IfNotPresent | ||
ports: | ||
- containerPort: 9898 | ||
name: http | ||
protocol: TCP | ||
command: | ||
- ./podinfo | ||
- --port=9898 | ||
- --level=info | ||
- --random-delay=false | ||
- --random-error=false | ||
env: | ||
- name: PODINFO_UI_COLOR | ||
value: blue | ||
livenessProbe: | ||
exec: | ||
command: | ||
- podcli | ||
- check | ||
- http | ||
- localhost:9898/healthz | ||
initialDelaySeconds: 5 | ||
timeoutSeconds: 5 | ||
readinessProbe: | ||
exec: | ||
command: | ||
- podcli | ||
- check | ||
- http | ||
- localhost:9898/readyz | ||
initialDelaySeconds: 5 | ||
timeoutSeconds: 5 | ||
resources: | ||
limits: | ||
cpu: 1000m | ||
memory: 128Mi | ||
requests: | ||
cpu: 1m | ||
memory: 16Mi |