cmd/runtimetest/main: Use TAP diagnostics for errors

Makefile

@@ -1,12 +1,14 @@
 PREFIX ?= $(DESTDIR)/usr
 BINDIR ?= $(DESTDIR)/usr/bin
+TAP ?= tap
 
 BUILDTAGS=
 RUNTIME ?= runc
 COMMIT=$(shell git rev-parse HEAD 2> /dev/null || true)
 VERSION := ${shell cat ./VERSION}
+VALIDATION_TESTS ?= $(patsubst %.go,%.t,$(wildcard validation/*.go))
 
-all: tool runtimetest
+all: tool runtimetest validation-executables
 
 tool:
 	go build -tags "$(BUILDTAGS)" -ldflags "-X main.gitCommit=${COMMIT} -X main.version=${VERSION}" -o oci-runtime-tool ./cmd/oci-runtime-tool
@@ -35,10 +37,18 @@ uninstall:
 	rm -f $(PREFIX)/share/bash-completion/completions/oci-runtime-tool
 
 clean:
-	rm -f oci-runtime-tool runtimetest *.1
+	rm -f oci-runtime-tool runtimetest *.1 $(VALIDATION_TESTS)
 
-localvalidation: runtimetest
-	RUNTIME=$(RUNTIME) go test -tags "$(BUILDTAGS)" ${TESTFLAGS} -v github.com/opencontainers/runtime-tools/validation
+localvalidation:
+	RUNTIME=$(RUNTIME) $(TAP) $(VALIDATION_TESTS)
+
+.PHONY: validation-executables
+validation-executables: $(VALIDATION_TESTS)
+
+.PRECIOUS: $(VALIDATION_TESTS)
+.PHONY: $(VALIDATION_TESTS)
+$(VALIDATION_TESTS): %.t: %.go
+	go build -tags "$(BUILDTAGS)" ${TESTFLAGS} -o $@ $<
 
 .PHONY: test .gofmt .govet .golint
 

README.md

@@ -8,7 +8,7 @@ To build from source code, runtime-tools requires Go 1.7.x or above.
 [`oci-runtime-tool generate`][generate.1] generates [configuration JSON][config.json] for an [OCI bundle][bundle].
 [OCI-compatible runtimes][runtime-spec] like [runC][] expect to read the configuration from `config.json`.
 
-```sh
+```console
 $ oci-runtime-tool generate --output config.json
 $ cat config.json
 {
@@ -22,63 +22,161 @@ $ cat config.json
 [`oci-runtime-tool validate`][validate.1] validates an OCI bundle.
 The error message will be printed if the OCI bundle failed the validation procedure.
 
-```sh
+```console
 $ oci-runtime-tool generate
 $ oci-runtime-tool validate
 INFO[0000] Bundle validation succeeded.
 ```
 
 ## Testing OCI runtimes
 
-```sh
-$ sudo make RUNTIME=runc localvalidation
-RUNTIME=runc go test -tags ""  -v github.com/opencontainers/runtime-tools/validation
-=== RUN   TestValidateBasic
-TAP version 13
-ok 1 - root filesystem
-ok 2 - hostname
-ok 3 - mounts
-ok 4 - capabilities
-ok 5 - default symlinks
-ok 6 - default devices
-ok 7 - linux devices
-ok 8 - linux process
-ok 9 - masked paths
-ok 10 - oom score adj
-ok 11 - read only paths
-ok 12 - rlimits
-ok 13 - sysctls
-ok 14 - uid mappings
-ok 15 - gid mappings
-1..15
---- PASS: TestValidateBasic (0.08s)
-=== RUN   TestValidateSysctls
+The runtime validation suite uses [node-tap][], which is packaged for some distributions (for example, it is in [Debian's `node-tap` package][debian-node-tap]).
+If your distribution does not package node-tap, you can install [npm][] (for example, from [Gentoo's `nodejs` package][gentoo-nodejs]) and use it:
+
+```console
+$ npm install tap
+```
+
+```console
+$ make runtimetest validation-executables
+RUNTIME=runc tap validation/linux_rootfs_propagation_shared.t validation/create.t validation/default.t validation/linux_readonly_paths.t validation/linux_masked_paths.t validation/mounts.t validation/process.t validation/root_readonly_false.t validation/linux_sysctl.t validation/linux_devices.t validation/linux_gid_mappings.t validation/process_oom_score_adj.t validation/process_capabilities.t validation/process_rlimits.t validation/root_readonly_true.t validation/linux_rootfs_propagation_unbindable.t validation/hostname.t validation/linux_uid_mappings.t
+validation/linux_rootfs_propagation_shared.t ........ 18/19
+  not ok rootfs propagation
+
+validation/create.t ................................... 4/4
+validation/default.t ................................ 19/19
+validation/linux_readonly_paths.t ................... 19/19
+validation/linux_masked_paths.t ..................... 18/19
+  not ok masked paths
+
+validation/mounts.t ................................... 0/1
+  Skipped: 1
+     TODO: mounts generation options have not been implemented
+
+validation/process.t ................................ 19/19
+validation/root_readonly_false.t .................... 19/19
+validation/linux_sysctl.t ........................... 19/19
+validation/linux_devices.t .......................... 19/19
+validation/linux_gid_mappings.t ..................... 18/19
+  not ok gid mappings
+
+validation/process_oom_score_adj.t .................. 19/19
+validation/process_capabilities.t ................... 19/19
+validation/process_rlimits.t ........................ 19/19
+validation/root_readonly_true.t ...................failed to create the container
+rootfsPropagation=unbindable is not supported
+exit status 1
+validation/root_readonly_true.t ..................... 19/19
+validation/linux_rootfs_propagation_unbindable.t ...... 0/1
+  not ok validation/linux_rootfs_propagation_unbindable.t
+    timeout: 30000
+    file: validation/linux_rootfs_propagation_unbindable.t
+    command: validation/linux_rootfs_propagation_unbindable.t
+    args: []
+    stdio:
+      - 0
+      - pipe
+      - 2
+    cwd: /…/go/src/github.com/opencontainers/runtime-tools
+    exitCode: 1
+
+validation/hostname.t ...................failed to create the container
+User namespace mappings specified, but USER namespace isn't enabled in the config
+exit status 1
+validation/hostname.t ............................... 19/19
+validation/linux_uid_mappings.t ....................... 0/1
+  not ok validation/linux_uid_mappings.t
+    timeout: 30000
+    file: validation/linux_uid_mappings.t
+    command: validation/linux_uid_mappings.t
+    args: []
+    stdio:
+      - 0
+      - pipe
+      - 2
+    cwd: /…/go/src/github.com/opencontainers/runtime-tools
+    exitCode: 1
+
+total ............................................. 267/273
+
+
+  267 passing (31s)
+  1 pending
+  5 failing
+
+make: *** [Makefile:43: localvalidation] Error 1
+```
+
+You can also run an individual test executable directly:
+
+```console
+$ RUNTIME=runc validation/default.t
 TAP version 13
 ok 1 - root filesystem
 ok 2 - hostname
-ok 3 - mounts
-ok 4 - capabilities
-ok 5 - default symlinks
-ok 6 - default devices
-ok 7 - linux devices
-ok 8 - linux process
-ok 9 - masked paths
-ok 10 - oom score adj
-ok 11 - read only paths
-ok 12 - rlimits
-ok 13 - sysctls
-ok 14 - uid mappings
-ok 15 - gid mappings
-1..15
---- PASS: TestValidateSysctls (0.20s)
-PASS
-ok      github.com/opencontainers/runtime-tools/validation      0.281s
+ok 3 - process
+ok 4 - mounts
+ok 5 - user
+ok 6 - rlimits
+ok 7 - capabilities
+ok 8 - default symlinks
+ok 9 - default file system
+ok 10 - default devices
+ok 11 - linux devices
+ok 12 - linux process
+ok 13 - masked paths
+ok 14 - oom score adj
+ok 15 - read only paths
+ok 16 - rootfs propagation
+ok 17 - sysctls
+ok 18 - uid mappings
+ok 19 - gid mappings
+1..19
+```
+
+If you cannot install node-tap, you can probably run the test suite with another [TAP consumer][tap-consumers].
+For example, with [`prove`][prove]:
+
+```console
+$ sudo make TAP='prove -Q -j9' RUNTIME=runc localvalidation
+RUNTIME=runc prove -Q -j9 validation/linux_rootfs_propagation_shared.t validation/create.t validation/default.t validation/linux_readonly_paths.t validation/linux_masked_paths.t validation/mounts.t validation/process.t validation/root_readonly_false.t validation/linux_sysctl.t validation/linux_devices.t validation/linux_gid_mappings.t validation/process_oom_score_adj.t validation/process_capabilities.t validation/process_rlimits.t validation/root_readonly_true.t validation/linux_rootfs_propagation_unbindable.t validation/hostname.t validation/linux_uid_mappings.t
+failed to create the container
+rootfsPropagation=unbindable is not supported
+exit status 1
+failed to create the container
+User namespace mappings specified, but USER namespace isn't enabled in the config
+exit status 1
+
+Test Summary Report
+-------------------
+validation/linux_rootfs_propagation_shared.t    (Wstat: 0 Tests: 19 Failed: 1)
+  Failed test:  16
+validation/linux_masked_paths.t                 (Wstat: 0 Tests: 19 Failed: 1)
+  Failed test:  13
+validation/linux_rootfs_propagation_unbindable.t (Wstat: 256 Tests: 0 Failed: 0)
+  Non-zero exit status: 1
+  Parse errors: No plan found in TAP output
+validation/linux_uid_mappings.t                 (Wstat: 256 Tests: 0 Failed: 0)
+  Non-zero exit status: 1
+  Parse errors: No plan found in TAP output
+validation/linux_gid_mappings.t                 (Wstat: 0 Tests: 19 Failed: 1)
+  Failed test:  19
+Files=18, Tests=271,  6 wallclock secs ( 0.06 usr  0.01 sys +  0.59 cusr  0.24 csys =  0.90 CPU)
+Result: FAIL
+make: *** [Makefile:43: localvalidation] Error 1
 ```
 
 [bundle]: https://github.com/opencontainers/runtime-spec/blob/master/bundle.md
 [config.json]: https://github.com/opencontainers/runtime-spec/blob/master/config.md
+[debian-node-tap]: https://packages.debian.org/stretch/node-tap
+[debian-nodejs]: https://packages.debian.org/stretch/nodejs
+[gentoo-nodejs]: https://packages.gentoo.org/packages/net-libs/nodejs
+[node-tap]: http://www.node-tap.org/
+[npm]: https://www.npmjs.com/
+[prove]: http://search.cpan.org/~leont/Test-Harness-3.39/bin/prove
 [runC]: https://github.com/opencontainers/runc
 [runtime-spec]: https://github.com/opencontainers/runtime-spec
+[tap-consumers]: https://testanything.org/consumers.html
 
 [generate.1]: man/oci-runtime-tool-generate.1.md
 [validate.1]: man/oci-runtime-tool-validate.1.md

cmd/runtimetest/main.go

@@ -891,46 +891,43 @@ func run(context *cli.Context) error {
 		complianceLevel = rfc2119.Must
 		logrus.Warningf("%s, using 'MUST' by default.", err.Error())
 	}
-	var validationErrors error
-	for _, v := range defaultValidations {
-		err := v.test(spec)
-		t.Ok(err == nil, v.description)
-		if err != nil {
-			if e, ok := err.(*specerror.Error); ok && e.Err.Level < complianceLevel {
-				continue
-			}
-			validationErrors = multierror.Append(validationErrors, err)
-		}
-	}
 
-	if platform == "linux" || platform == "solaris" {
-		for _, v := range posixValidations {
-			err := v.test(spec)
-			t.Ok(err == nil, v.description)
-			if err != nil {
-				if e, ok := err.(*specerror.Error); ok && e.Err.Level < complianceLevel {
-					continue
-				}
-				validationErrors = multierror.Append(validationErrors, err)
-			}
-		}
+	validations := defaultValidations
+	if platform == "linux" {
+		validations = append(validations, posixValidations...)
+		validations = append(validations, linuxValidations...)
+	} else if platform == "solaris" {
+		validations = append(validations, posixValidations...)
 	}
 
-	if platform == "linux" {
-		for _, v := range linuxValidations {
-			err := v.test(spec)
-			t.Ok(err == nil, v.description)
-			if err != nil {
-				if e, ok := err.(*specerror.Error); ok && e.Err.Level < complianceLevel {
-					continue
+	for _, v := range validations {
+		err := v.test(spec)
+		if err == nil {
+			t.Pass(v.description)
+		} else {
+			merr, ok := err.(*multierror.Error)
+			if ok {
+				for _, err = range merr.Errors {
+					if e, ok := err.(*rfc2119.Error); ok {
+						t.Ok(e.Level < complianceLevel, v.description)
+					} else {
+						t.Fail(v.description)
+					}
+					t.Diagnostic(err.Error())
+				}
+			} else {
+				if e, ok := err.(*rfc2119.Error); ok {
+					t.Ok(e.Level < complianceLevel, v.description)
+				} else {
+					t.Fail(v.description)
 				}
-				validationErrors = multierror.Append(validationErrors, err)
+				t.Diagnostic(err.Error())
 			}
 		}
 	}
 	t.AutoPlan()
 
-	return validationErrors
+	return nil
 }
 
 func main() {

validation/generate_test.go → generate/generate_test.go

@@ -1,4 +1,4 @@
-package validation
+package generate_test
 
 import (
 	"io/ioutil"

validation/.gitignore

@@ -0,0 +1 @@
+/*.t