From fcba3d9d61513eadb4d55d9827ff4daac3b0f9f2 Mon Sep 17 00:00:00 2001 From: Mathieu Tortuyaux Date: Wed, 15 Mar 2023 10:48:03 +0100 Subject: [PATCH 1/5] kola: add --devcontainer-url this param allows to override the default base URL for fetching the dev container. Signed-off-by: Mathieu Tortuyaux --- cmd/kola/options.go | 1 + kola/harness.go | 1 + kola/tests/devcontainer/devcontainer.go | 17 +++++++++-------- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/cmd/kola/options.go b/cmd/kola/options.go index ee7bf83d5..8a473f04f 100644 --- a/cmd/kola/options.go +++ b/cmd/kola/options.go @@ -73,6 +73,7 @@ func init() { // general options sv(&outputDir, "output-dir", "", "Temporary output directory for test data and logs") sv(&kola.TorcxManifestFile, "torcx-manifest", "", "Path to a torcx manifest that should be made available to tests") + sv(&kola.DevcontainerURL, "devcontainer-url", "http://bincache.flatcar-linux.net/images/@ARCH@/@VERSION@", "URL to a dev container archive that should be made available to tests") root.PersistentFlags().StringVarP(&kolaPlatform, "platform", "p", "qemu", "VM platform: "+strings.Join(kolaPlatforms, ", ")) root.PersistentFlags().StringVarP(&kolaChannel, "channel", "", "stable", "Channel: "+strings.Join(kolaChannels, ", ")) root.PersistentFlags().StringVarP(&kolaOffering, "offering", "", "basic", "Offering: "+strings.Join(kolaOfferings, ", ")) diff --git a/kola/harness.go b/kola/harness.go index 2a6b61905..02199fa94 100644 --- a/kola/harness.go +++ b/kola/harness.go @@ -76,6 +76,7 @@ var ( TestParallelism int //glue var to set test parallelism from main TAPFile string // if not "", write TAP results here TorcxManifestFile string // torcx manifest to expose to tests, if set + DevcontainerURL string // dev container to expose to tests, if set // TorcxManifest is the unmarshalled torcx manifest file. It is available for // tests to access via `kola.TorcxManifest`. It will be nil if there was no // manifest given to kola. diff --git a/kola/tests/devcontainer/devcontainer.go b/kola/tests/devcontainer/devcontainer.go index 454020a7c..a63cece7c 100644 --- a/kola/tests/devcontainer/devcontainer.go +++ b/kola/tests/devcontainer/devcontainer.go @@ -24,6 +24,7 @@ import ( "github.com/coreos/go-semver/semver" + "github.com/flatcar/mantle/kola" "github.com/flatcar/mantle/kola/cluster" "github.com/flatcar/mantle/kola/register" "github.com/flatcar/mantle/platform" @@ -50,11 +51,6 @@ func trimLeftSpace(contents string) string { } var ( - defaultScriptTemplateParameters = scriptTemplateParameters{ - BinhostURLTemplate: "http://bincache.flatcar-linux.net/boards/@ARCH@-usr/@VERSION@/pkgs", - ImageDirectoryURLTemplate: "http://bincache.flatcar-linux.net/images/@ARCH@/@VERSION@", - } - devContainerScriptTemplate = trimLeftSpace(` #!/bin/bash @@ -229,14 +225,19 @@ func init() { } func withSystemdNspawn(c cluster.TestCluster) { - runDevContainerTest(c, defaultScriptTemplateParameters, systemdNspawnScriptBody) + runDevContainerTest(c, systemdNspawnScriptBody) } func withDocker(c cluster.TestCluster) { - runDevContainerTest(c, defaultScriptTemplateParameters, dockerScriptBody) + runDevContainerTest(c, dockerScriptBody) } -func runDevContainerTest(c cluster.TestCluster, scriptParameters scriptTemplateParameters, scriptBody string) { +func runDevContainerTest(c cluster.TestCluster, scriptBody string) { + scriptParameters := scriptTemplateParameters{ + BinhostURLTemplate: "http://bincache.flatcar-linux.net/boards/@ARCH@-usr/@VERSION@/pkgs", + ImageDirectoryURLTemplate: kola.DevcontainerURL, + } + userdata, err := prepareUserData(scriptParameters, scriptBody) if err != nil { c.Fatalf("preparing user data failed: %v", err) From 22626b76e476f5bae689c1dfa115854d9f3d9e0a Mon Sep 17 00:00:00 2001 From: Mathieu Tortuyaux Date: Wed, 15 Mar 2023 11:58:48 +0100 Subject: [PATCH 2/5] harness: export ScpKolet function this function can be used outside in case one cluster starts without initial size but we want Kolet to be uploaded on the instances. Signed-off-by: Mathieu Tortuyaux --- kola/harness.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/kola/harness.go b/kola/harness.go index 02199fa94..97af8ae67 100644 --- a/kola/harness.go +++ b/kola/harness.go @@ -626,7 +626,7 @@ func runTest(h *harness.H, t *register.Test, pltfrm string, flight platform.Flig // drop kolet binary on machines if t.NativeFuncs != nil { - scpKolet(tcluster, architecture(pltfrm)) + ScpKolet(tcluster, architecture(pltfrm)) } defer func() { @@ -677,8 +677,8 @@ func findExecDir() string { return filepath.Dir(p) } -// scpKolet searches for a kolet binary and copies it to the machine. -func scpKolet(c cluster.TestCluster, mArch string) { +// ScpKolet searches for a kolet binary and copies it to the machine. +func ScpKolet(c cluster.TestCluster, mArch string) { for _, d := range []string{ ".", findExecDir(), From 5065166f1496ef6e54e300d98dbd1f8714ea5cac Mon Sep 17 00:00:00 2001 From: Mathieu Tortuyaux Date: Wed, 15 Mar 2023 11:59:37 +0100 Subject: [PATCH 3/5] local/http: add a simple HTTP implementation it serves a directory /var/www on the port 8080 Signed-off-by: Mathieu Tortuyaux --- platform/local/http.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 platform/local/http.go diff --git a/platform/local/http.go b/platform/local/http.go new file mode 100644 index 000000000..526e8b143 --- /dev/null +++ b/platform/local/http.go @@ -0,0 +1,15 @@ +// Copyright The Mantle Authors +// SPDX-License-Identifier: Apache-2.0 + +package local + +import ( + "net/http" +) + +// SimpleHTTP provides a single http server. +type SimpleHTTP struct{} + +func (s *SimpleHTTP) Serve() error { + return http.ListenAndServe(":8080", http.FileServer(http.Dir("/var/www"))) +} From 9b07a7e5cd06927fb7c7a9d439ba611791bf4242 Mon Sep 17 00:00:00 2001 From: Mathieu Tortuyaux Date: Wed, 15 Mar 2023 12:01:03 +0100 Subject: [PATCH 4/5] kola: add devcontainer-file flag this option allows to pass arbitrary local dev container archive to be uploaded and served by a HTTP server in order to be consumed by the devcontainer test. Signed-off-by: Mathieu Tortuyaux --- cmd/kola/options.go | 1 + kola/harness.go | 1 + kola/tests/devcontainer/devcontainer.go | 53 ++++++++++++++++++++++++- 3 files changed, 54 insertions(+), 1 deletion(-) diff --git a/cmd/kola/options.go b/cmd/kola/options.go index 8a473f04f..1c2fdbc4c 100644 --- a/cmd/kola/options.go +++ b/cmd/kola/options.go @@ -74,6 +74,7 @@ func init() { sv(&outputDir, "output-dir", "", "Temporary output directory for test data and logs") sv(&kola.TorcxManifestFile, "torcx-manifest", "", "Path to a torcx manifest that should be made available to tests") sv(&kola.DevcontainerURL, "devcontainer-url", "http://bincache.flatcar-linux.net/images/@ARCH@/@VERSION@", "URL to a dev container archive that should be made available to tests") + sv(&kola.DevcontainerFile, "devcontainer-file", "", "Path to a dev container archive that should be made available to tests as alternative to devcontainer-url, note that a working devcontainer-binhost-url is still needed") root.PersistentFlags().StringVarP(&kolaPlatform, "platform", "p", "qemu", "VM platform: "+strings.Join(kolaPlatforms, ", ")) root.PersistentFlags().StringVarP(&kolaChannel, "channel", "", "stable", "Channel: "+strings.Join(kolaChannels, ", ")) root.PersistentFlags().StringVarP(&kolaOffering, "offering", "", "basic", "Offering: "+strings.Join(kolaOfferings, ", ")) diff --git a/kola/harness.go b/kola/harness.go index 97af8ae67..8e8c7aba4 100644 --- a/kola/harness.go +++ b/kola/harness.go @@ -77,6 +77,7 @@ var ( TAPFile string // if not "", write TAP results here TorcxManifestFile string // torcx manifest to expose to tests, if set DevcontainerURL string // dev container to expose to tests, if set + DevcontainerFile string // dev container path to expose to tests, if set // TorcxManifest is the unmarshalled torcx manifest file. It is available for // tests to access via `kola.TorcxManifest`. It will be nil if there was no // manifest given to kola. diff --git a/kola/tests/devcontainer/devcontainer.go b/kola/tests/devcontainer/devcontainer.go index a63cece7c..20c4db9f4 100644 --- a/kola/tests/devcontainer/devcontainer.go +++ b/kola/tests/devcontainer/devcontainer.go @@ -18,8 +18,10 @@ import ( "encoding/base64" "errors" "fmt" + "os" "strings" "text/template" + "time" "unicode" "github.com/coreos/go-semver/semver" @@ -29,8 +31,10 @@ import ( "github.com/flatcar/mantle/kola/register" "github.com/flatcar/mantle/platform" "github.com/flatcar/mantle/platform/conf" + "github.com/flatcar/mantle/platform/local" "github.com/flatcar/mantle/platform/machine/qemu" "github.com/flatcar/mantle/platform/machine/unprivqemu" + "github.com/flatcar/mantle/util" ) // Both template parameters may contain @ARCH@ and @VERSION@ @@ -211,6 +215,9 @@ func init() { Platforms: []string{"qemu", "qemu-unpriv"}, Distros: []string{"cl"}, MinVersion: semver.Version{Major: 2592}, + NativeFuncs: map[string]func() error{ + "Http": Serve, + }, }) register.Register(®ister.Test{ Name: "devcontainer.docker", @@ -221,6 +228,9 @@ func init() { Distros: []string{"cl"}, // TODO: Revisit this flag when updating SELinux policies. Flags: []register.Flag{register.NoEnableSelinux}, + NativeFuncs: map[string]func() error{ + "Http": Serve, + }, }) } @@ -233,9 +243,15 @@ func withDocker(c cluster.TestCluster) { } func runDevContainerTest(c cluster.TestCluster, scriptBody string) { + devcontainerURL := kola.DevcontainerURL + if kola.DevcontainerFile != "" { + // This URL is deterministic as it runs on the started machine. + devcontainerURL = "http://localhost:8080" + } + scriptParameters := scriptTemplateParameters{ BinhostURLTemplate: "http://bincache.flatcar-linux.net/boards/@ARCH@-usr/@VERSION@/pkgs", - ImageDirectoryURLTemplate: kola.DevcontainerURL, + ImageDirectoryURLTemplate: devcontainerURL, } userdata, err := prepareUserData(scriptParameters, scriptBody) @@ -246,6 +262,11 @@ func runDevContainerTest(c cluster.TestCluster, scriptBody string) { if err != nil { c.Fatalf("creating a machine failed: %v", err) } + + if kola.DevcontainerFile != "" { + configureHTTPServer(c, machine) + } + if _, err := c.SSH(machine, "/home/core/main-script"); err != nil { c.Fatalf("main script failed: %v", err) } @@ -298,3 +319,33 @@ func newMachineWithLargeDisk(c cluster.TestCluster, userData *conf.UserData) (pl } return nil, errors.New("unknown cluster type, this test should only be running on qemu or qemu-unpriv platforms") } + +func Serve() error { + httpServer := local.SimpleHTTP{} + return httpServer.Serve() +} + +func configureHTTPServer(c cluster.TestCluster, srv platform.Machine) { + // manually copy Kolet on the host, as the initial size cluster is 0. + kola.ScpKolet(c, strings.SplitN(kola.QEMUOptions.Board, "-", 2)[0]) + + in, err := os.Open(kola.DevcontainerFile) + if err != nil { + c.Fatalf("opening dev container file: %v", err) + } + + defer in.Close() + + if err := platform.InstallFile(in, srv, "/var/www/flatcar_developer_container.bin.bz2"); err != nil { + c.Fatalf("copying dev container to HTTP server: %v", err) + } + + c.MustSSH(srv, fmt.Sprintf("sudo systemd-run --quiet ./kolet run %s Http", c.H.Name())) + + if err := util.WaitUntilReady(60*time.Second, 5*time.Second, func() (bool, error) { + _, _, err := srv.SSH(fmt.Sprintf("curl %s:8080", srv.PrivateIP())) + return err == nil, nil + }); err != nil { + c.Fatal("timed out waiting for http server to become active") + } +} From cd05fb3bddcc1499d7a989911ff8f42085f4cdb5 Mon Sep 17 00:00:00 2001 From: Mathieu Tortuyaux Date: Thu, 16 Mar 2023 09:37:04 +0100 Subject: [PATCH 5/5] kola: add --devcontainer-binhost-url this param allows to override the default devcontainer binhost URL for the devcontainer test. Signed-off-by: Mathieu Tortuyaux --- cmd/kola/options.go | 1 + kola/harness.go | 11 ++++++----- kola/tests/devcontainer/devcontainer.go | 2 +- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/cmd/kola/options.go b/cmd/kola/options.go index 1c2fdbc4c..bd44c2e34 100644 --- a/cmd/kola/options.go +++ b/cmd/kola/options.go @@ -75,6 +75,7 @@ func init() { sv(&kola.TorcxManifestFile, "torcx-manifest", "", "Path to a torcx manifest that should be made available to tests") sv(&kola.DevcontainerURL, "devcontainer-url", "http://bincache.flatcar-linux.net/images/@ARCH@/@VERSION@", "URL to a dev container archive that should be made available to tests") sv(&kola.DevcontainerFile, "devcontainer-file", "", "Path to a dev container archive that should be made available to tests as alternative to devcontainer-url, note that a working devcontainer-binhost-url is still needed") + sv(&kola.DevcontainerBinhostURL, "devcontainer-binhost-url", "http://bincache.flatcar-linux.net/boards/@ARCH@-usr/@VERSION@/pkgs", "URL to a binary host that the devcontainer test should use") root.PersistentFlags().StringVarP(&kolaPlatform, "platform", "p", "qemu", "VM platform: "+strings.Join(kolaPlatforms, ", ")) root.PersistentFlags().StringVarP(&kolaChannel, "channel", "", "stable", "Channel: "+strings.Join(kolaChannels, ", ")) root.PersistentFlags().StringVarP(&kolaOffering, "offering", "", "basic", "Offering: "+strings.Join(kolaOfferings, ", ")) diff --git a/kola/harness.go b/kola/harness.go index 8e8c7aba4..82ff02f64 100644 --- a/kola/harness.go +++ b/kola/harness.go @@ -73,11 +73,12 @@ var ( EquinixMetalOptions = equinixmetalapi.Options{Options: &Options} // glue to set platform options from main QEMUOptions = qemu.Options{Options: &Options} // glue to set platform options from main - TestParallelism int //glue var to set test parallelism from main - TAPFile string // if not "", write TAP results here - TorcxManifestFile string // torcx manifest to expose to tests, if set - DevcontainerURL string // dev container to expose to tests, if set - DevcontainerFile string // dev container path to expose to tests, if set + TestParallelism int //glue var to set test parallelism from main + TAPFile string // if not "", write TAP results here + TorcxManifestFile string // torcx manifest to expose to tests, if set + DevcontainerURL string // dev container to expose to tests, if set + DevcontainerBinhostURL string // dev container binhost URL to use in the devcontainer test + DevcontainerFile string // dev container path to expose to tests, if set // TorcxManifest is the unmarshalled torcx manifest file. It is available for // tests to access via `kola.TorcxManifest`. It will be nil if there was no // manifest given to kola. diff --git a/kola/tests/devcontainer/devcontainer.go b/kola/tests/devcontainer/devcontainer.go index 20c4db9f4..e89f37de0 100644 --- a/kola/tests/devcontainer/devcontainer.go +++ b/kola/tests/devcontainer/devcontainer.go @@ -250,7 +250,7 @@ func runDevContainerTest(c cluster.TestCluster, scriptBody string) { } scriptParameters := scriptTemplateParameters{ - BinhostURLTemplate: "http://bincache.flatcar-linux.net/boards/@ARCH@-usr/@VERSION@/pkgs", + BinhostURLTemplate: kola.DevcontainerBinhostURL, ImageDirectoryURLTemplate: devcontainerURL, }