diff --git a/Makefile b/Makefile index 7ac3f751c..a7a858123 100644 --- a/Makefile +++ b/Makefile @@ -1,12 +1,14 @@ PREFIX ?= $(DESTDIR)/usr BINDIR ?= $(DESTDIR)/usr/bin +TAP ?= tap BUILDTAGS= RUNTIME ?= runc COMMIT=$(shell git rev-parse HEAD 2> /dev/null || true) VERSION := ${shell cat ./VERSION} +VALIDATION_TESTS ?= $(patsubst %.go,%.t,$(wildcard validation/*.go)) -all: tool runtimetest +all: tool runtimetest validation-executables tool: go build -tags "$(BUILDTAGS)" -ldflags "-X main.gitCommit=${COMMIT} -X main.version=${VERSION}" -o oci-runtime-tool ./cmd/oci-runtime-tool @@ -35,10 +37,18 @@ uninstall: rm -f $(PREFIX)/share/bash-completion/completions/oci-runtime-tool clean: - rm -f oci-runtime-tool runtimetest *.1 + rm -f oci-runtime-tool runtimetest *.1 $(VALIDATION_TESTS) -localvalidation: runtimetest - RUNTIME=$(RUNTIME) go test -tags "$(BUILDTAGS)" ${TESTFLAGS} -v github.com/opencontainers/runtime-tools/validation +localvalidation: + RUNTIME=$(RUNTIME) $(TAP) $(VALIDATION_TESTS) + +.PHONY: validation-executables +validation-executables: $(VALIDATION_TESTS) + +.PRECIOUS: $(VALIDATION_TESTS) +.PHONY: $(VALIDATION_TESTS) +$(VALIDATION_TESTS): %.t: %.go + go build -tags "$(BUILDTAGS)" ${TESTFLAGS} -o $@ $< .PHONY: test .gofmt .govet .golint diff --git a/README.md b/README.md index bbcafc26e..5f16436bc 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ To build from source code, runtime-tools requires Go 1.7.x or above. [`oci-runtime-tool generate`][generate.1] generates [configuration JSON][config.json] for an [OCI bundle][bundle]. [OCI-compatible runtimes][runtime-spec] like [runC][] expect to read the configuration from `config.json`. -```sh +```console $ oci-runtime-tool generate --output config.json $ cat config.json { @@ -22,7 +22,7 @@ $ cat config.json [`oci-runtime-tool validate`][validate.1] validates an OCI bundle. The error message will be printed if the OCI bundle failed the validation procedure. -```sh +```console $ oci-runtime-tool generate $ oci-runtime-tool validate INFO[0000] Bundle validation succeeded. @@ -30,55 +30,153 @@ INFO[0000] Bundle validation succeeded. ## Testing OCI runtimes -```sh -$ sudo make RUNTIME=runc localvalidation -RUNTIME=runc go test -tags "" -v github.com/opencontainers/runtime-tools/validation -=== RUN TestValidateBasic -TAP version 13 -ok 1 - root filesystem -ok 2 - hostname -ok 3 - mounts -ok 4 - capabilities -ok 5 - default symlinks -ok 6 - default devices -ok 7 - linux devices -ok 8 - linux process -ok 9 - masked paths -ok 10 - oom score adj -ok 11 - read only paths -ok 12 - rlimits -ok 13 - sysctls -ok 14 - uid mappings -ok 15 - gid mappings -1..15 ---- PASS: TestValidateBasic (0.08s) -=== RUN TestValidateSysctls +The runtime validation suite uses [node-tap][], which is packaged for some distributions (for example, it is in [Debian's `node-tap` package][debian-node-tap]). +If your distribution does not package node-tap, you can install [npm][] (for example, from [Gentoo's `nodejs` package][gentoo-nodejs]) and use it: + +```console +$ npm install tap +``` + +```console +$ make runtimetest validation-executables +RUNTIME=runc tap validation/linux_rootfs_propagation_shared.t validation/create.t validation/default.t validation/linux_readonly_paths.t validation/linux_masked_paths.t validation/mounts.t validation/process.t validation/root_readonly_false.t validation/linux_sysctl.t validation/linux_devices.t validation/linux_gid_mappings.t validation/process_oom_score_adj.t validation/process_capabilities.t validation/process_rlimits.t validation/root_readonly_true.t validation/linux_rootfs_propagation_unbindable.t validation/hostname.t validation/linux_uid_mappings.t +validation/linux_rootfs_propagation_shared.t ........ 18/19 + not ok rootfs propagation + +validation/create.t ................................... 4/4 +validation/default.t ................................ 19/19 +validation/linux_readonly_paths.t ................... 19/19 +validation/linux_masked_paths.t ..................... 18/19 + not ok masked paths + +validation/mounts.t ................................... 0/1 + Skipped: 1 + TODO: mounts generation options have not been implemented + +validation/process.t ................................ 19/19 +validation/root_readonly_false.t .................... 19/19 +validation/linux_sysctl.t ........................... 19/19 +validation/linux_devices.t .......................... 19/19 +validation/linux_gid_mappings.t ..................... 18/19 + not ok gid mappings + +validation/process_oom_score_adj.t .................. 19/19 +validation/process_capabilities.t ................... 19/19 +validation/process_rlimits.t ........................ 19/19 +validation/root_readonly_true.t ...................failed to create the container +rootfsPropagation=unbindable is not supported +exit status 1 +validation/root_readonly_true.t ..................... 19/19 +validation/linux_rootfs_propagation_unbindable.t ...... 0/1 + not ok validation/linux_rootfs_propagation_unbindable.t + timeout: 30000 + file: validation/linux_rootfs_propagation_unbindable.t + command: validation/linux_rootfs_propagation_unbindable.t + args: [] + stdio: + - 0 + - pipe + - 2 + cwd: /…/go/src/github.com/opencontainers/runtime-tools + exitCode: 1 + +validation/hostname.t ...................failed to create the container +User namespace mappings specified, but USER namespace isn't enabled in the config +exit status 1 +validation/hostname.t ............................... 19/19 +validation/linux_uid_mappings.t ....................... 0/1 + not ok validation/linux_uid_mappings.t + timeout: 30000 + file: validation/linux_uid_mappings.t + command: validation/linux_uid_mappings.t + args: [] + stdio: + - 0 + - pipe + - 2 + cwd: /…/go/src/github.com/opencontainers/runtime-tools + exitCode: 1 + +total ............................................. 267/273 + + + 267 passing (31s) + 1 pending + 5 failing + +make: *** [Makefile:43: localvalidation] Error 1 +``` + +You can also run an individual test executable directly: + +```console +$ RUNTIME=runc validation/default.t TAP version 13 ok 1 - root filesystem ok 2 - hostname -ok 3 - mounts -ok 4 - capabilities -ok 5 - default symlinks -ok 6 - default devices -ok 7 - linux devices -ok 8 - linux process -ok 9 - masked paths -ok 10 - oom score adj -ok 11 - read only paths -ok 12 - rlimits -ok 13 - sysctls -ok 14 - uid mappings -ok 15 - gid mappings -1..15 ---- PASS: TestValidateSysctls (0.20s) -PASS -ok github.com/opencontainers/runtime-tools/validation 0.281s +ok 3 - process +ok 4 - mounts +ok 5 - user +ok 6 - rlimits +ok 7 - capabilities +ok 8 - default symlinks +ok 9 - default file system +ok 10 - default devices +ok 11 - linux devices +ok 12 - linux process +ok 13 - masked paths +ok 14 - oom score adj +ok 15 - read only paths +ok 16 - rootfs propagation +ok 17 - sysctls +ok 18 - uid mappings +ok 19 - gid mappings +1..19 +``` + +If you cannot install node-tap, you can probably run the test suite with another [TAP consumer][tap-consumers]. +For example, with [`prove`][prove]: + +```console +$ sudo make TAP='prove -Q -j9' RUNTIME=runc localvalidation +RUNTIME=runc prove -Q -j9 validation/linux_rootfs_propagation_shared.t validation/create.t validation/default.t validation/linux_readonly_paths.t validation/linux_masked_paths.t validation/mounts.t validation/process.t validation/root_readonly_false.t validation/linux_sysctl.t validation/linux_devices.t validation/linux_gid_mappings.t validation/process_oom_score_adj.t validation/process_capabilities.t validation/process_rlimits.t validation/root_readonly_true.t validation/linux_rootfs_propagation_unbindable.t validation/hostname.t validation/linux_uid_mappings.t +failed to create the container +rootfsPropagation=unbindable is not supported +exit status 1 +failed to create the container +User namespace mappings specified, but USER namespace isn't enabled in the config +exit status 1 + +Test Summary Report +------------------- +validation/linux_rootfs_propagation_shared.t (Wstat: 0 Tests: 19 Failed: 1) + Failed test: 16 +validation/linux_masked_paths.t (Wstat: 0 Tests: 19 Failed: 1) + Failed test: 13 +validation/linux_rootfs_propagation_unbindable.t (Wstat: 256 Tests: 0 Failed: 0) + Non-zero exit status: 1 + Parse errors: No plan found in TAP output +validation/linux_uid_mappings.t (Wstat: 256 Tests: 0 Failed: 0) + Non-zero exit status: 1 + Parse errors: No plan found in TAP output +validation/linux_gid_mappings.t (Wstat: 0 Tests: 19 Failed: 1) + Failed test: 19 +Files=18, Tests=271, 6 wallclock secs ( 0.06 usr 0.01 sys + 0.59 cusr 0.24 csys = 0.90 CPU) +Result: FAIL +make: *** [Makefile:43: localvalidation] Error 1 ``` [bundle]: https://github.com/opencontainers/runtime-spec/blob/master/bundle.md [config.json]: https://github.com/opencontainers/runtime-spec/blob/master/config.md +[debian-node-tap]: https://packages.debian.org/stretch/node-tap +[debian-nodejs]: https://packages.debian.org/stretch/nodejs +[gentoo-nodejs]: https://packages.gentoo.org/packages/net-libs/nodejs +[node-tap]: http://www.node-tap.org/ +[npm]: https://www.npmjs.com/ +[prove]: http://search.cpan.org/~leont/Test-Harness-3.39/bin/prove [runC]: https://github.com/opencontainers/runc [runtime-spec]: https://github.com/opencontainers/runtime-spec +[tap-consumers]: https://testanything.org/consumers.html [generate.1]: man/oci-runtime-tool-generate.1.md [validate.1]: man/oci-runtime-tool-validate.1.md diff --git a/cmd/runtimetest/main.go b/cmd/runtimetest/main.go index 5b5d488ca..11017e9b5 100644 --- a/cmd/runtimetest/main.go +++ b/cmd/runtimetest/main.go @@ -891,46 +891,43 @@ func run(context *cli.Context) error { complianceLevel = rfc2119.Must logrus.Warningf("%s, using 'MUST' by default.", err.Error()) } - var validationErrors error - for _, v := range defaultValidations { - err := v.test(spec) - t.Ok(err == nil, v.description) - if err != nil { - if e, ok := err.(*specerror.Error); ok && e.Err.Level < complianceLevel { - continue - } - validationErrors = multierror.Append(validationErrors, err) - } - } - if platform == "linux" || platform == "solaris" { - for _, v := range posixValidations { - err := v.test(spec) - t.Ok(err == nil, v.description) - if err != nil { - if e, ok := err.(*specerror.Error); ok && e.Err.Level < complianceLevel { - continue - } - validationErrors = multierror.Append(validationErrors, err) - } - } + validations := defaultValidations + if platform == "linux" { + validations = append(validations, posixValidations...) + validations = append(validations, linuxValidations...) + } else if platform == "solaris" { + validations = append(validations, posixValidations...) } - if platform == "linux" { - for _, v := range linuxValidations { - err := v.test(spec) - t.Ok(err == nil, v.description) - if err != nil { - if e, ok := err.(*specerror.Error); ok && e.Err.Level < complianceLevel { - continue + for _, v := range validations { + err := v.test(spec) + if err == nil { + t.Pass(v.description) + } else { + merr, ok := err.(*multierror.Error) + if ok { + for _, err = range merr.Errors { + if e, ok := err.(*rfc2119.Error); ok { + t.Ok(e.Level < complianceLevel, v.description) + } else { + t.Fail(v.description) + } + t.Diagnostic(err.Error()) + } + } else { + if e, ok := err.(*rfc2119.Error); ok { + t.Ok(e.Level < complianceLevel, v.description) + } else { + t.Fail(v.description) } - validationErrors = multierror.Append(validationErrors, err) + t.Diagnostic(err.Error()) } } } t.AutoPlan() - return validationErrors + return nil } func main() { diff --git a/validation/generate_test.go b/generate/generate_test.go similarity index 98% rename from validation/generate_test.go rename to generate/generate_test.go index fe6893821..a96e3a67e 100644 --- a/validation/generate_test.go +++ b/generate/generate_test.go @@ -1,4 +1,4 @@ -package validation +package generate_test import ( "io/ioutil" diff --git a/validation/.gitignore b/validation/.gitignore new file mode 100644 index 000000000..141f81298 --- /dev/null +++ b/validation/.gitignore @@ -0,0 +1 @@ +/*.t diff --git a/validation/create.go b/validation/create.go new file mode 100644 index 000000000..892311277 --- /dev/null +++ b/validation/create.go @@ -0,0 +1,69 @@ +package main + +import ( + "fmt" + + "github.com/mndrix/tap-go" + rspecs "github.com/opencontainers/runtime-spec/specs-go" + "github.com/opencontainers/runtime-tools/generate" + "github.com/opencontainers/runtime-tools/specerror" + "github.com/opencontainers/runtime-tools/validation/util" + "github.com/satori/go.uuid" +) + +func main() { + t := tap.New() + t.Header(0) + + g := generate.New() + g.SetRootPath(".") + g.SetProcessArgs([]string{"ls"}) + + bundleDir, err := util.PrepareBundle() + if err != nil { + util.Fatal(err) + } + + r, err := util.NewRuntime(util.RuntimeCommand, bundleDir) + if err != nil { + util.Fatal(err) + } + defer r.Clean(true, true) + + err = r.SetConfig(&g) + if err != nil { + util.Fatal(err) + } + + containerID := uuid.NewV4().String() + cases := []struct { + id string + errExpected bool + err error + }{ + {"", false, specerror.NewError(specerror.CreateWithBundlePathAndID, fmt.Errorf("create MUST generate an error if the ID is not provided"), rspecs.Version)}, + {containerID, true, specerror.NewError(specerror.CreateNewContainer, fmt.Errorf("create MUST create a new container"), rspecs.Version)}, + {containerID, false, specerror.NewError(specerror.CreateWithUniqueID, fmt.Errorf("create MUST generate an error if the ID provided is not unique"), rspecs.Version)}, + } + + for _, c := range cases { + r.SetID(c.id) + stderr, err := r.Create() + t.Ok((err == nil) == c.errExpected, c.err.(*specerror.Error).Err.Err.Error()) + t.Diagnostic(c.err.(*specerror.Error).Err.Reference) + if err != nil { + t.Diagnostic(err.Error()) + } + if len(stderr) > 0 { + t.Diagnostic(string(stderr)) + } + + if err == nil { + state, _ := r.State() + t.Ok(state.ID == c.id, "") + t.Diagnosticf("container PID: %d, state ID: %d", c.id, state.ID) + } + } + + t.AutoPlan() +} diff --git a/validation/default.go b/validation/default.go new file mode 100644 index 000000000..07054f7bb --- /dev/null +++ b/validation/default.go @@ -0,0 +1,13 @@ +package main + +import ( + "github.com/opencontainers/runtime-tools/validation/util" +) + +func main() { + g := util.GetDefaultGenerator() + err := util.RuntimeInsideValidate(g, nil) + if err != nil { + util.Fatal(err) + } +} diff --git a/validation/hostname.go b/validation/hostname.go new file mode 100644 index 000000000..139de84cd --- /dev/null +++ b/validation/hostname.go @@ -0,0 +1,14 @@ +package main + +import ( + "github.com/opencontainers/runtime-tools/validation/util" +) + +func main() { + g := util.GetDefaultGenerator() + g.SetHostname("hostname-specific") + err := util.RuntimeInsideValidate(g, nil) + if err != nil { + util.Fatal(err) + } +} diff --git a/validation/linux_devices.go b/validation/linux_devices.go new file mode 100644 index 000000000..a5768adcd --- /dev/null +++ b/validation/linux_devices.go @@ -0,0 +1,55 @@ +package main + +import ( + "os" + + rspecs "github.com/opencontainers/runtime-spec/specs-go" + "github.com/opencontainers/runtime-tools/validation/util" +) + +func main() { + g := util.GetDefaultGenerator() + + // add char device + cdev := rspecs.LinuxDevice{} + cdev.Path = "/dev/test1" + cdev.Type = "c" + cdev.Major = 10 + cdev.Minor = 666 + cmode := os.FileMode(int32(432)) + cdev.FileMode = &cmode + cuid := uint32(0) + cdev.UID = &cuid + cgid := uint32(0) + cdev.GID = &cgid + g.AddDevice(cdev) + + // add block device + bdev := rspecs.LinuxDevice{} + bdev.Path = "/dev/test2" + bdev.Type = "b" + bdev.Major = 8 + bdev.Minor = 666 + bmode := os.FileMode(int32(432)) + bdev.FileMode = &bmode + uid := uint32(0) + bdev.UID = &uid + gid := uint32(0) + bdev.GID = &gid + g.AddDevice(bdev) + + // add fifo device + pdev := rspecs.LinuxDevice{} + pdev.Path = "/dev/test3" + pdev.Type = "p" + pdev.Major = 8 + pdev.Minor = 666 + pmode := os.FileMode(int32(432)) + pdev.FileMode = &pmode + g.AddDevice(pdev) + + err := util.RuntimeInsideValidate(g, nil) + if err != nil { + util.Fatal(err) + } +} diff --git a/validation/linux_gid_mappings.go b/validation/linux_gid_mappings.go new file mode 100644 index 000000000..383d5ce65 --- /dev/null +++ b/validation/linux_gid_mappings.go @@ -0,0 +1,14 @@ +package main + +import ( + "github.com/opencontainers/runtime-tools/validation/util" +) + +func main() { + g := util.GetDefaultGenerator() + g.AddLinuxGIDMapping(uint32(1000), uint32(0), uint32(3200)) + err := util.RuntimeInsideValidate(g, nil) + if err != nil { + util.Fatal(err) + } +} diff --git a/validation/linux_masked_paths.go b/validation/linux_masked_paths.go new file mode 100644 index 000000000..5d97f41ba --- /dev/null +++ b/validation/linux_masked_paths.go @@ -0,0 +1,20 @@ +package main + +import ( + "os" + "path/filepath" + + "github.com/opencontainers/runtime-tools/validation/util" +) + +func main() { + g := util.GetDefaultGenerator() + g.AddLinuxMaskedPaths("/masktest") + err := util.RuntimeInsideValidate(g, func(path string) error { + pathName := filepath.Join(path, "masktest") + return os.MkdirAll(pathName, 0700) + }) + if err != nil { + util.Fatal(err) + } +} diff --git a/validation/linux_readonly_paths.go b/validation/linux_readonly_paths.go new file mode 100644 index 000000000..6c1e37fa1 --- /dev/null +++ b/validation/linux_readonly_paths.go @@ -0,0 +1,20 @@ +package main + +import ( + "os" + "path/filepath" + + "github.com/opencontainers/runtime-tools/validation/util" +) + +func main() { + g := util.GetDefaultGenerator() + g.AddLinuxReadonlyPaths("readonlytest") + err := util.RuntimeInsideValidate(g, func(path string) error { + pathName := filepath.Join(path, "readonlytest") + return os.MkdirAll(pathName, 0700) + }) + if err != nil { + util.Fatal(err) + } +} diff --git a/validation/linux_rootfs_propagation_shared.go b/validation/linux_rootfs_propagation_shared.go new file mode 100644 index 000000000..c7699277b --- /dev/null +++ b/validation/linux_rootfs_propagation_shared.go @@ -0,0 +1,15 @@ +package main + +import ( + "github.com/opencontainers/runtime-tools/validation/util" +) + +func main() { + g := util.GetDefaultGenerator() + g.SetupPrivileged(true) + g.SetLinuxRootPropagation("shared") + err := util.RuntimeInsideValidate(g, nil) + if err != nil { + util.Fatal(err) + } +} diff --git a/validation/linux_rootfs_propagation_unbindable.go b/validation/linux_rootfs_propagation_unbindable.go new file mode 100644 index 000000000..c6ea24c88 --- /dev/null +++ b/validation/linux_rootfs_propagation_unbindable.go @@ -0,0 +1,15 @@ +package main + +import ( + "github.com/opencontainers/runtime-tools/validation/util" +) + +func main() { + g := util.GetDefaultGenerator() + g.SetupPrivileged(true) + g.SetLinuxRootPropagation("unbindable") + err := util.RuntimeInsideValidate(g, nil) + if err != nil { + util.Fatal(err) + } +} diff --git a/validation/linux_sysctl.go b/validation/linux_sysctl.go new file mode 100644 index 000000000..1daded44e --- /dev/null +++ b/validation/linux_sysctl.go @@ -0,0 +1,14 @@ +package main + +import ( + "github.com/opencontainers/runtime-tools/validation/util" +) + +func main() { + g := util.GetDefaultGenerator() + g.AddLinuxSysctl("net.ipv4.ip_forward", "1") + err := util.RuntimeInsideValidate(g, nil) + if err != nil { + util.Fatal(err) + } +} diff --git a/validation/linux_uid_mappings.go b/validation/linux_uid_mappings.go new file mode 100644 index 000000000..856863736 --- /dev/null +++ b/validation/linux_uid_mappings.go @@ -0,0 +1,14 @@ +package main + +import ( + "github.com/opencontainers/runtime-tools/validation/util" +) + +func main() { + g := util.GetDefaultGenerator() + g.AddLinuxUIDMapping(uint32(1000), uint32(0), uint32(3200)) + err := util.RuntimeInsideValidate(g, nil) + if err != nil { + util.Fatal(err) + } +} diff --git a/validation/mounts.go b/validation/mounts.go new file mode 100644 index 000000000..e3df60b21 --- /dev/null +++ b/validation/mounts.go @@ -0,0 +1,9 @@ +package main + +import ( + "github.com/opencontainers/runtime-tools/validation/util" +) + +func main() { + util.Skip("TODO: mounts generation options have not been implemented", "") +} diff --git a/validation/process.go b/validation/process.go new file mode 100644 index 000000000..c132c19e9 --- /dev/null +++ b/validation/process.go @@ -0,0 +1,23 @@ +package main + +import ( + "os" + "path/filepath" + + "github.com/opencontainers/runtime-tools/validation/util" +) + +func main() { + g := util.GetDefaultGenerator() + g.SetProcessCwd("/test") + g.AddProcessEnv("testa", "valuea") + g.AddProcessEnv("testb", "123") + + err := util.RuntimeInsideValidate(g, func(path string) error { + pathName := filepath.Join(path, "test") + return os.MkdirAll(pathName, 0700) + }) + if err != nil { + util.Fatal(err) + } +} diff --git a/validation/process_capabilities.go b/validation/process_capabilities.go new file mode 100644 index 000000000..3c676c554 --- /dev/null +++ b/validation/process_capabilities.go @@ -0,0 +1,22 @@ +package main + +import ( + "os" + "runtime" + + "github.com/opencontainers/runtime-tools/validation/util" +) + +func main() { + if "linux" != runtime.GOOS { + util.Skip("linux-specific process.capabilities test", runtime.GOOS) + os.Exit(0) + } + + g := util.GetDefaultGenerator() + g.SetupPrivileged(true) + err := util.RuntimeInsideValidate(g, nil) + if err != nil { + util.Fatal(err) + } +} diff --git a/validation/process_oom_score_adj.go b/validation/process_oom_score_adj.go new file mode 100644 index 000000000..ee8448c1f --- /dev/null +++ b/validation/process_oom_score_adj.go @@ -0,0 +1,14 @@ +package main + +import ( + "github.com/opencontainers/runtime-tools/validation/util" +) + +func main() { + g := util.GetDefaultGenerator() + g.SetProcessOOMScoreAdj(500) + err := util.RuntimeInsideValidate(g, nil) + if err != nil { + util.Fatal(err) + } +} diff --git a/validation/process_rlimits.go b/validation/process_rlimits.go new file mode 100644 index 000000000..9f146552e --- /dev/null +++ b/validation/process_rlimits.go @@ -0,0 +1,14 @@ +package main + +import ( + "github.com/opencontainers/runtime-tools/validation/util" +) + +func main() { + g := util.GetDefaultGenerator() + g.AddProcessRlimits("RLIMIT_NOFILE", 1024, 1024) + err := util.RuntimeInsideValidate(g, nil) + if err != nil { + util.Fatal(err) + } +} diff --git a/validation/root_readonly_false.go b/validation/root_readonly_false.go new file mode 100644 index 000000000..6349ccd15 --- /dev/null +++ b/validation/root_readonly_false.go @@ -0,0 +1,14 @@ +package main + +import ( + "github.com/opencontainers/runtime-tools/validation/util" +) + +func main() { + g := util.GetDefaultGenerator() + g.SetRootReadonly(false) + err := util.RuntimeInsideValidate(g, nil) + if err != nil { + util.Fatal(err) + } +} diff --git a/validation/root_readonly_true.go b/validation/root_readonly_true.go new file mode 100644 index 000000000..05907a02c --- /dev/null +++ b/validation/root_readonly_true.go @@ -0,0 +1,22 @@ +package main + +import ( + "os" + "runtime" + + "github.com/opencontainers/runtime-tools/validation/util" +) + +func main() { + if "windows" == runtime.GOOS { + util.Skip("non-Windows root.readonly test", runtime.GOOS) + os.Exit(0) + } + + g := util.GetDefaultGenerator() + g.SetRootReadonly(true) + err := util.RuntimeInsideValidate(g, nil) + if err != nil { + util.Fatal(err) + } +} diff --git a/validation/container.go b/validation/util/container.go similarity index 54% rename from validation/container.go rename to validation/util/container.go index 295b81e7d..652c316f8 100644 --- a/validation/container.go +++ b/validation/util/container.go @@ -1,8 +1,11 @@ -package validation +package util import ( "encoding/json" "errors" + "fmt" + "io" + "io/ioutil" "os" "os/exec" "path/filepath" @@ -16,6 +19,8 @@ type Runtime struct { RuntimeCommand string BundleDir string ID string + stdout *os.File + stderr *os.File } // NewRuntime create a runtime by command and the bundle directory @@ -45,7 +50,7 @@ func (r *Runtime) SetID(id string) { } // Create a container -func (r *Runtime) Create() error { +func (r *Runtime) Create() (stderr []byte, err error) { var args []string args = append(args, "create") if r.ID != "" { @@ -58,14 +63,46 @@ func (r *Runtime) Create() error { // } cmd := exec.Command(r.RuntimeCommand, args...) cmd.Dir = r.BundleDir - cmd.Stdin = os.Stdin - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - return cmd.Run() + r.stdout, err = os.OpenFile(filepath.Join(r.BundleDir, fmt.Sprintf("stdout-%s", r.ID)), os.O_CREATE|os.O_EXCL|os.O_RDWR, 0600) + if err != nil { + return []byte(""), err + } + cmd.Stdout = r.stdout + r.stderr, err = os.OpenFile(filepath.Join(r.BundleDir, fmt.Sprintf("stderr-%s", r.ID)), os.O_CREATE|os.O_EXCL|os.O_RDWR, 0600) + if err != nil { + return []byte(""), err + } + cmd.Stderr = r.stderr + + err = cmd.Run() + if err == nil { + return []byte(""), err + } + + stdout, stderr, _ := r.ReadStandardStreams() + if len(stderr) == 0 { + stderr = stdout + } + return stderr, err +} + +// ReadStandardStreams collects content from the stdout and stderr buffers. +func (r *Runtime) ReadStandardStreams() (stdout []byte, stderr []byte, err error) { + _, err = r.stdout.Seek(0, io.SeekStart) + stdout, err2 := ioutil.ReadAll(r.stdout) + if err == nil && err2 != nil { + err = err2 + } + _, err = r.stderr.Seek(0, io.SeekStart) + stderr, err2 = ioutil.ReadAll(r.stderr) + if err == nil && err2 != nil { + err = err2 + } + return stdout, stderr, err } // Start a container -func (r *Runtime) Start() error { +func (r *Runtime) Start() (stderr []byte, err error) { var args []string args = append(args, "start") if r.ID != "" { @@ -73,10 +110,15 @@ func (r *Runtime) Start() error { } cmd := exec.Command(r.RuntimeCommand, args...) - cmd.Stdin = os.Stdin - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - return cmd.Run() + stdout, err := cmd.Output() + if e, ok := err.(*exec.ExitError); ok { + stderr = e.Stderr + } + if err != nil && len(stderr) == 0 { + stderr = stdout + } + + return stderr, err } // State a container information @@ -109,16 +151,19 @@ func (r *Runtime) Delete() error { return cmd.Run() } -// Clean deletes the container and removes the bundle file according to the input parameter -func (r *Runtime) Clean(removeBundle bool) error { +// Clean deletes the container. If removeBundle is set, the bundle +// directory is removed after the container is deleted succesfully or, if +// forceRemoveBundle is true, after the deletion attempt regardless of +// whether it was successful or not. +func (r *Runtime) Clean(removeBundle bool, forceRemoveBundle bool) error { err := r.Delete() - if err != nil { - return err - } - if removeBundle { - os.RemoveAll(r.BundleDir) + if removeBundle && (err == nil || forceRemoveBundle) { + err2 := os.RemoveAll(r.BundleDir) + if err2 != nil && err == nil { + err = err2 + } } - return nil + return err } diff --git a/validation/util/test.go b/validation/util/test.go new file mode 100644 index 000000000..3b544197c --- /dev/null +++ b/validation/util/test.go @@ -0,0 +1,140 @@ +package util + +import ( + "fmt" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "runtime" + "time" + + "github.com/mndrix/tap-go" + "github.com/mrunalp/fileutils" + "github.com/opencontainers/runtime-tools/generate" + "github.com/satori/go.uuid" +) + +var ( + // RuntimeCommand is the default runtime command. + RuntimeCommand = "runc" +) + +// PreFunc initializes the test environment after preparing the bundle +// but before creating the container. +type PreFunc func(string) error + +func init() { + runtimeInEnv := os.Getenv("RUNTIME") + if runtimeInEnv != "" { + RuntimeCommand = runtimeInEnv + } +} + +// Fatal prints a warning to stderr and exits. +func Fatal(err error) { + fmt.Fprintf(os.Stderr, "%+v\n", err) + os.Exit(1) +} + +// Skip skips a full TAP suite. +func Skip(message string, diagnostic string) { + t := tap.New() + t.Header(1) + t.Skip(1, message) + if diagnostic != "" { + t.Diagnostic(diagnostic) + } +} + +// PrepareBundle creates a test bundle in a temporary directory. +func PrepareBundle() (string, error) { + bundleDir, err := ioutil.TempDir("", "ocitest") + if err != nil { + return "", err + } + + // Untar the root fs + untarCmd := exec.Command("tar", "-xf", fmt.Sprintf("rootfs-%s.tar.gz", runtime.GOARCH), "-C", bundleDir) + output, err := untarCmd.CombinedOutput() + if err != nil { + os.Stderr.Write(output) + os.RemoveAll(bundleDir) + return "", err + } + + return bundleDir, nil +} + +// GetDefaultGenerator creates a default configuration generator. +func GetDefaultGenerator() *generate.Generator { + g := generate.New() + g.SetRootPath(".") + g.SetProcessArgs([]string{"/runtimetest", "--path=/"}) + return &g +} + +// RuntimeInsideValidate runs runtimetest inside a container. +func RuntimeInsideValidate(g *generate.Generator, f PreFunc) (err error) { + bundleDir, err := PrepareBundle() + if err != nil { + return err + } + + if f != nil { + if err := f(bundleDir); err != nil { + return err + } + } + + r, err := NewRuntime(RuntimeCommand, bundleDir) + if err != nil { + os.RemoveAll(bundleDir) + return err + } + defer r.Clean(true, true) + err = r.SetConfig(g) + if err != nil { + return err + } + err = fileutils.CopyFile("runtimetest", filepath.Join(r.BundleDir, "runtimetest")) + if err != nil { + return err + } + + r.SetID(uuid.NewV4().String()) + stderr, err := r.Create() + if err != nil { + os.Stderr.WriteString("failed to create the container\n") + os.Stderr.Write(stderr) + return err + } + + // FIXME: why do we need this? Without a sleep here, I get: + // failed to start the container + // container "..." does not exist + time.Sleep(1 * time.Second) + + stderr, err = r.Start() + if err != nil { + os.Stderr.WriteString("failed to start the container\n") + os.Stderr.Write(stderr) + return err + } + + // FIXME: wait until the container exits and collect its exit code. + time.Sleep(1 * time.Second) + + stdout, stderr, err := r.ReadStandardStreams() + if err != nil { + if len(stderr) == 0 { + stderr = stdout + } + os.Stderr.WriteString("failed to read standard streams\n") + os.Stderr.Write(stderr) + return err + } + + os.Stdout.Write(stdout) + return nil +} diff --git a/validation/validation_test.go b/validation/validation_test.go deleted file mode 100644 index edc041b87..000000000 --- a/validation/validation_test.go +++ /dev/null @@ -1,320 +0,0 @@ -package validation - -import ( - "fmt" - "io/ioutil" - "os" - "os/exec" - "path/filepath" - "runtime" - "testing" - - "github.com/mrunalp/fileutils" - rspecs "github.com/opencontainers/runtime-spec/specs-go" - "github.com/satori/go.uuid" - "github.com/stretchr/testify/assert" - - "github.com/opencontainers/runtime-tools/generate" - "github.com/opencontainers/runtime-tools/specerror" -) - -var ( - runtimeCommand = "runc" -) - -// build test environment before running container -type preFunc func(string) error - -func init() { - runtimeInEnv := os.Getenv("RUNTIME") - if runtimeInEnv != "" { - runtimeCommand = runtimeInEnv - } -} - -func prepareBundle() (string, error) { - // Setup a temporary test directory - bundleDir, err := ioutil.TempDir("", "ocitest") - if err != nil { - return "", err - } - - // Untar the root fs - untarCmd := exec.Command("tar", "-xf", fmt.Sprintf("../rootfs-%s.tar.gz", runtime.GOARCH), "-C", bundleDir) - _, err = untarCmd.CombinedOutput() - if err != nil { - os.RemoveAll(bundleDir) - return "", err - } - - return bundleDir, nil -} - -func getDefaultGenerator() *generate.Generator { - g := generate.New() - g.SetRootPath(".") - g.SetProcessArgs([]string{"/runtimetest", "--path=/"}) - return &g -} - -func runtimeInsideValidate(g *generate.Generator, f preFunc) error { - bundleDir, err := prepareBundle() - if err != nil { - return err - } - - if f != nil { - if err := f(bundleDir); err != nil { - return err - } - } - - r, err := NewRuntime(runtimeCommand, bundleDir) - if err != nil { - os.RemoveAll(bundleDir) - return err - } - defer r.Clean(true) - err = r.SetConfig(g) - if err != nil { - return err - } - err = fileutils.CopyFile("../runtimetest", filepath.Join(r.BundleDir, "runtimetest")) - if err != nil { - return err - } - - r.SetID(uuid.NewV4().String()) - err = r.Create() - if err != nil { - return err - } - return r.Start() -} - -func TestValidateBasic(t *testing.T) { - g := getDefaultGenerator() - - assert.Nil(t, runtimeInsideValidate(g, nil)) -} - -// Test whether rootfs Readonly can be applied as false -func TestValidateRootFSReadWrite(t *testing.T) { - g := getDefaultGenerator() - g.SetRootReadonly(false) - - assert.Nil(t, runtimeInsideValidate(g, nil)) -} - -// Test whether rootfs Readonly can be applied as true -func TestValidateRootFSReadonly(t *testing.T) { - if "windows" == runtime.GOOS { - t.Skip("skip this test on windows platform") - } - - g := getDefaultGenerator() - g.SetRootReadonly(true) - - assert.Nil(t, runtimeInsideValidate(g, nil)) -} - -// Test Process -func TestValidateProcess(t *testing.T) { - g := getDefaultGenerator() - g.SetProcessCwd("/test") - g.AddProcessEnv("testa", "valuea") - g.AddProcessEnv("testb", "123") - - assert.Nil(t, runtimeInsideValidate(g, func(path string) error { - pathName := filepath.Join(path, "test") - return os.MkdirAll(pathName, 0700) - })) -} - -// Test whether Capabilites can be applied or not -func TestValidateCapabilities(t *testing.T) { - if "linux" != runtime.GOOS { - t.Skip("skip linux-specific capabilities test") - } - - g := getDefaultGenerator() - g.SetupPrivileged(true) - - assert.Nil(t, runtimeInsideValidate(g, nil)) -} - -// Test whether hostname can be applied or not -func TestValidateHostname(t *testing.T) { - g := getDefaultGenerator() - g.SetHostname("hostname-specific") - - assert.Nil(t, runtimeInsideValidate(g, nil)) -} - -func TestValidateRootfsPropagationPrivate(t *testing.T) { - t.Skip("has not been implemented yet") -} - -func TestValidateRootfsPropagationSlave(t *testing.T) { - t.Skip("has not been implemented yet") -} - -func TestValidateRootfsPropagationShared(t *testing.T) { - g := getDefaultGenerator() - g.SetupPrivileged(true) - g.SetLinuxRootPropagation("shared") - - assert.Nil(t, runtimeInsideValidate(g, nil)) -} - -func TestValidateRootfsPropagationUnbindable(t *testing.T) { - g := getDefaultGenerator() - g.SetupPrivileged(true) - g.SetLinuxRootPropagation("unbindable") - - assert.Nil(t, runtimeInsideValidate(g, nil)) -} - -func TestValidateLinuxDevices(t *testing.T) { - g := getDefaultGenerator() - - // add char device - cdev := rspecs.LinuxDevice{} - cdev.Path = "/dev/test1" - cdev.Type = "c" - cdev.Major = 10 - cdev.Minor = 666 - cmode := os.FileMode(int32(432)) - cdev.FileMode = &cmode - cuid := uint32(0) - cdev.UID = &cuid - cgid := uint32(0) - cdev.GID = &cgid - g.AddDevice(cdev) - // add block device - bdev := rspecs.LinuxDevice{} - bdev.Path = "/dev/test2" - bdev.Type = "b" - bdev.Major = 8 - bdev.Minor = 666 - bmode := os.FileMode(int32(432)) - bdev.FileMode = &bmode - uid := uint32(0) - bdev.UID = &uid - gid := uint32(0) - bdev.GID = &gid - g.AddDevice(bdev) - // add fifo device - pdev := rspecs.LinuxDevice{} - pdev.Path = "/dev/test3" - pdev.Type = "p" - pdev.Major = 8 - pdev.Minor = 666 - pmode := os.FileMode(int32(432)) - pdev.FileMode = &pmode - g.AddDevice(pdev) - - assert.Nil(t, runtimeInsideValidate(g, nil)) -} - -func TestValidateMaskedPaths(t *testing.T) { - g := getDefaultGenerator() - g.AddLinuxMaskedPaths("/masktest") - - assert.Nil(t, runtimeInsideValidate(g, func(path string) error { - pathName := filepath.Join(path, "masktest") - return os.MkdirAll(pathName, 0700) - })) -} - -func TestValidateROPaths(t *testing.T) { - g := getDefaultGenerator() - g.AddLinuxReadonlyPaths("readonlytest") - - assert.Nil(t, runtimeInsideValidate(g, func(path string) error { - pathName := filepath.Join(path, "readonlytest") - return os.MkdirAll(pathName, 0700) - })) -} - -func TestValidateOOMScoreAdj(t *testing.T) { - g := getDefaultGenerator() - g.SetProcessOOMScoreAdj(500) - - assert.Nil(t, runtimeInsideValidate(g, nil)) -} - -func TestValidateUIDMappings(t *testing.T) { - g := getDefaultGenerator() - g.AddLinuxUIDMapping(uint32(1000), uint32(0), uint32(3200)) - - assert.Nil(t, runtimeInsideValidate(g, nil)) -} - -func TestValidateGIDMappings(t *testing.T) { - g := getDefaultGenerator() - g.AddLinuxGIDMapping(uint32(1000), uint32(0), uint32(3200)) - - assert.Nil(t, runtimeInsideValidate(g, nil)) -} - -// Test whether mounts are correctly mounted -func TestValidateMounts(t *testing.T) { - // TODO mounts generation options have not been implemented - // will add it after 'mounts generate' done -} - -// Test whether rlimits can be applied or not -func TestValidateRlimits(t *testing.T) { - g := getDefaultGenerator() - g.AddProcessRlimits("RLIMIT_NOFILE", 1024, 1024) - - assert.Nil(t, runtimeInsideValidate(g, nil)) -} - -// Test whether sysctls can be applied or not -func TestValidateSysctls(t *testing.T) { - g := getDefaultGenerator() - g.AddLinuxSysctl("net.ipv4.ip_forward", "1") - - assert.Nil(t, runtimeInsideValidate(g, nil)) -} - -// Test Create operation -func TestValidateCreate(t *testing.T) { - g := generate.New() - g.SetRootPath(".") - g.SetProcessArgs([]string{"ls"}) - - bundleDir, err := prepareBundle() - assert.Nil(t, err) - - r, err := NewRuntime(runtimeCommand, bundleDir) - assert.Nil(t, err) - defer r.Clean(true) - - err = r.SetConfig(&g) - assert.Nil(t, err) - - containerID := uuid.NewV4().String() - cases := []struct { - id string - errExpected bool - err error - }{ - {"", false, specerror.NewError(specerror.CreateWithBundlePathAndID, fmt.Errorf("create MUST generate an error if the ID is not provided"), rspecs.Version)}, - {containerID, true, specerror.NewError(specerror.CreateNewContainer, fmt.Errorf("create MUST create a new container"), rspecs.Version)}, - {containerID, false, specerror.NewError(specerror.CreateWithUniqueID, fmt.Errorf("create MUST generate an error if the ID provided is not unique"), rspecs.Version)}, - } - - for _, c := range cases { - r.SetID(c.id) - err := r.Create() - assert.Equal(t, c.errExpected, err == nil, c.err.Error()) - - if err == nil { - state, _ := r.State() - assert.Equal(t, c.id, state.ID, c.err.Error()) - } - } -}