From a5e6cf60fc89a4958cc59bc07cbfa73b4111f795 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Tue, 11 Dec 2018 13:56:49 -0800 Subject: [PATCH 1/3] Change restart policy on gvisor pod Change the restart policy on the gvisor pod to Always. This way, if a user runs minikube addons enable gvisor minikube stop minikube start when the addon manager tries to restart the gvisor pod, it will be restarted and gvisor will start running automatically. This PR also adds an integration test for this functionality. --- deploy/addons/gvisor/gvisor-pod.yaml | 2 +- test/integration/addons_test.go | 56 +++++++++++++++++++++++----- test/integration/functional_test.go | 1 + 3 files changed, 48 insertions(+), 11 deletions(-) diff --git a/deploy/addons/gvisor/gvisor-pod.yaml b/deploy/addons/gvisor/gvisor-pod.yaml index 50a54bb69336..8cb0d3321977 100644 --- a/deploy/addons/gvisor/gvisor-pod.yaml +++ b/deploy/addons/gvisor/gvisor-pod.yaml @@ -69,4 +69,4 @@ spec: - name: gvisor hostPath: path: /tmp/gvisor - restartPolicy: Never + restartPolicy: Always diff --git a/test/integration/addons_test.go b/test/integration/addons_test.go index 7d65b970222a..e727c1232b86 100644 --- a/test/integration/addons_test.go +++ b/test/integration/addons_test.go @@ -30,6 +30,7 @@ import ( "testing" "time" + "github.com/docker/machine/libmachine/state" "k8s.io/apimachinery/pkg/labels" pkgutil "k8s.io/minikube/pkg/util" "k8s.io/minikube/test/integration/util" @@ -186,7 +187,6 @@ func testServicesList(t *testing.T) { func testGvisor(t *testing.T) { minikubeRunner := NewMinikubeRunner(t) - kubectlRunner := util.NewKubectlRunner(t) minikubeRunner.RunCommand("addons enable gvisor", true) t.Log("waiting for gvisor controller to come up") @@ -194,11 +194,7 @@ func testGvisor(t *testing.T) { t.Fatalf("waiting for gvisor controller to be up: %v", err) } - untrustedPath, _ := filepath.Abs("testdata/nginx-untrusted.yaml") - t.Log("creating pod with untrusted workload annotation") - if _, err := kubectlRunner.RunCommand([]string{"create", "-f", untrustedPath}); err != nil { - t.Fatalf("creating untrusted nginx resource: %v", err) - } + createUntrustedWorkload(t) t.Log("making sure untrusted workload is Running") if err := util.WaitForUntrustedNginxRunning(); err != nil { @@ -212,16 +208,56 @@ func testGvisor(t *testing.T) { t.Fatalf("waiting for gvisor controller to be deleted: %v", err) } - t.Log("recreating untrusted workload pod") - if _, err := kubectlRunner.RunCommand([]string{"replace", "-f", untrustedPath, "--force"}); err != nil { - t.Fatalf("replacing untrusted nginx resource: %v", err) - } + createUntrustedWorkload(t) t.Log("waiting for FailedCreatePodSandBox event") if err := util.WaitForFailedCreatePodSandBoxEvent(); err != nil { t.Fatalf("waiting for FailedCreatePodSandBox event: %v", err) } + deleteUntrustedWorkload(t) +} + +func testGvisorRestart(t *testing.T) { + minikubeRunner := NewMinikubeRunner(t) + minikubeRunner.EnsureRunning() + fmt.Println("enabling gvisor") + minikubeRunner.RunCommand("addons enable gvisor", true) + + t.Log("waiting for gvisor controller to come up") + if err := util.WaitForGvisorControllerRunning(t); err != nil { + t.Fatalf("waiting for gvisor controller to be up: %v", err) + } + + minikubeRunner.RunCommand("delete", false) + minikubeRunner.CheckStatus(state.None.String()) + minikubeRunner.Start() + minikubeRunner.CheckStatus(state.Running.String()) + + t.Log("waiting for gvisor controller to come up") + if err := util.WaitForGvisorControllerRunning(t); err != nil { + t.Fatalf("waiting for gvisor controller to be up: %v", err) + } + + createUntrustedWorkload(t) + t.Log("making sure untrusted workload is Running") + if err := util.WaitForUntrustedNginxRunning(); err != nil { + t.Fatalf("waiting for nginx to be up: %v", err) + } + deleteUntrustedWorkload(t) +} +func createUntrustedWorkload(t *testing.T) { + kubectlRunner := util.NewKubectlRunner(t) + untrustedPath, _ := filepath.Abs("testdata/nginx-untrusted.yaml") + t.Log("creating pod with untrusted workload annotation") + if _, err := kubectlRunner.RunCommand([]string{"replace", "-f", untrustedPath, "--force"}); err != nil { + t.Fatalf("creating untrusted nginx resource: %v", err) + } +} + +func deleteUntrustedWorkload(t *testing.T) { + kubectlRunner := util.NewKubectlRunner(t) + untrustedPath, _ := filepath.Abs("testdata/nginx-untrusted.yaml") if _, err := kubectlRunner.RunCommand([]string{"delete", "-f", untrustedPath}); err != nil { t.Logf("error deleting untrusted nginx resource: %v", err) } diff --git a/test/integration/functional_test.go b/test/integration/functional_test.go index 08ca9e171b8c..42cbdd5ec851 100644 --- a/test/integration/functional_test.go +++ b/test/integration/functional_test.go @@ -65,6 +65,7 @@ func TestFunctionalContainerd(t *testing.T) { minikubeRunner.EnsureRunning() t.Run("Gvisor", testGvisor) + t.Run("GvisorRestart", testGvisorRestart) minikubeRunner.RunCommand("delete", true) } From 15d311ba402ae337aee0a7794b93bcfe085c4181 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Tue, 11 Dec 2018 14:40:38 -0800 Subject: [PATCH 2/3] Test stop and start --- test/integration/addons_test.go | 12 +++++++++--- test/integration/util/util.go | 3 +-- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/test/integration/addons_test.go b/test/integration/addons_test.go index e727c1232b86..bab1162c6b35 100644 --- a/test/integration/addons_test.go +++ b/test/integration/addons_test.go @@ -220,7 +220,6 @@ func testGvisor(t *testing.T) { func testGvisorRestart(t *testing.T) { minikubeRunner := NewMinikubeRunner(t) minikubeRunner.EnsureRunning() - fmt.Println("enabling gvisor") minikubeRunner.RunCommand("addons enable gvisor", true) t.Log("waiting for gvisor controller to come up") @@ -228,8 +227,15 @@ func testGvisorRestart(t *testing.T) { t.Fatalf("waiting for gvisor controller to be up: %v", err) } - minikubeRunner.RunCommand("delete", false) - minikubeRunner.CheckStatus(state.None.String()) + checkStop := func() error { + minikubeRunner.RunCommand("stop", true) + return minikubeRunner.CheckStatusNoFail(state.Stopped.String()) + } + + if err := util.Retry(t, checkStop, 5*time.Second, 6); err != nil { + t.Fatalf("timed out while checking stopped status: %v", err) + } + minikubeRunner.Start() minikubeRunner.CheckStatus(state.Running.String()) diff --git a/test/integration/util/util.go b/test/integration/util/util.go index 5faf473c89bc..a4b4a8d5b19b 100644 --- a/test/integration/util/util.go +++ b/test/integration/util/util.go @@ -126,8 +126,7 @@ func (m *MinikubeRunner) SSH(command string) (string, error) { func (m *MinikubeRunner) Start() { switch r := m.Runtime; r { case constants.ContainerdRuntime: - // TODO: priyawadhwa@ remove iso url once updated iso is being used in integration tests - containerdFlags := "--container-runtime=containerd --network-plugin=cni --docker-opt containerd=/var/run/containerd/containerd.sock --iso-url=https://storage.googleapis.com/k8s-minikube/gvisor-preview.iso" + containerdFlags := "--container-runtime=containerd --network-plugin=cni --docker-opt containerd=/var/run/containerd/containerd.sock" m.RunCommand(fmt.Sprintf("start %s %s %s", m.StartArgs, m.Args, containerdFlags), true) default: m.RunCommand(fmt.Sprintf("start %s %s", m.StartArgs, m.Args), true) From 019ac524c70f62111438fb623fb79f368f1df3d1 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Tue, 11 Dec 2018 15:50:23 -0800 Subject: [PATCH 3/3] Revert test to delete Revert test to delete for now, for some reason "stop" and then "start" is failing both locally and in Jenkins for VirtualBox with a "panic test timed out after 30 min" error --- test/integration/addons_test.go | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/test/integration/addons_test.go b/test/integration/addons_test.go index bab1162c6b35..748894f2e0ba 100644 --- a/test/integration/addons_test.go +++ b/test/integration/addons_test.go @@ -227,15 +227,9 @@ func testGvisorRestart(t *testing.T) { t.Fatalf("waiting for gvisor controller to be up: %v", err) } - checkStop := func() error { - minikubeRunner.RunCommand("stop", true) - return minikubeRunner.CheckStatusNoFail(state.Stopped.String()) - } - - if err := util.Retry(t, checkStop, 5*time.Second, 6); err != nil { - t.Fatalf("timed out while checking stopped status: %v", err) - } - + // TODO: @priyawadhwa to add test for stop as well + minikubeRunner.RunCommand("delete", false) + minikubeRunner.CheckStatus(state.None.String()) minikubeRunner.Start() minikubeRunner.CheckStatus(state.Running.String())