validation: Use prove(1) as a TAP harness

Capture stdout and stderr from create invocations, because we don't
want to pollute the TAP output with things like runc's:

  Incorrect Usage.
  ...

(which is for some reason printed to stdout) or:

  runc: "create" requires exactly 1 argument(s)

which is printed to stderr.  Instead, show the captured stderr as a
diagnostic, and hide the stdout completely.  Unless stderr is empty,
in which case show stdout in case it contains something useful.

Most of these tests are broken because we aren't collecting the
container exit code or post-start stdout.  But the tests haven't been
doing that since the create/start split in 15577bd (add runtime
struct; add create test, 2017-08-24, #447) anyway [1].  This commit
just makes that more obvious.

The patsubst and wildcard Makefile syntax is documented in [2].  The
$(VALIDATION_TESTS) rule uses the static pattern rule syntax [3].

And use 'console' instead of 'sh' in the README, because these are
shell sessions, not scripts.  See [4,5].  I don't have a working runc
locally, so the mock results based on a dummy runtime script.

[1]: #447 (comment)
[2]: https://www.gnu.org/software/make/manual/html_node/Wildcard-Function.html
[3]: https://www.gnu.org/software/make/manual/html_node/Static-Usage.html#Static-Usage
[4]: https://help.github.com/articles/creating-and-highlighting-code-blocks/#syntax-highlighting
[5]: https://github.com/github/linguist/blob/v5.3.3/lib/linguist/languages.yml#L4249-L4260

Signed-off-by: W. Trevor King <wking@tremily.us>

Author: wking

Makefile

@@ -5,6 +5,7 @@ BUILDTAGS=
 RUNTIME ?= runc
 COMMIT=$(shell git rev-parse HEAD 2> /dev/null || true)
 VERSION := ${shell cat ./VERSION}
+VALIDATION_TESTS ?= $(patsubst %.go,%.t,$(wildcard validation/*.go))
 
 all: tool runtimetest
 
@@ -35,10 +36,14 @@ uninstall:
 	rm -f $(PREFIX)/share/bash-completion/completions/oci-runtime-tool
 
 clean:
-	rm -f oci-runtime-tool runtimetest *.1
+	rm -f oci-runtime-tool runtimetest *.1 $(VALIDATION_TESTS)
 
-localvalidation: runtimetest
-	RUNTIME=$(RUNTIME) go test -tags "$(BUILDTAGS)" ${TESTFLAGS} -v github.com/opencontainers/runtime-tools/validation
+localvalidation: runtimetest $(VALIDATION_TESTS)
+	RUNTIME=$(RUNTIME) prove $(VALIDATION_TESTS)
+
+.PHONY: $(VALIDATION_TESTS)
+$(VALIDATION_TESTS): %.t: %.go
+	go build -tags "$(BUILDTAGS)" ${TESTFLAGS} -o $@ $<
 
 .PHONY: test .gofmt .govet .golint
 

README.md

@@ -8,7 +8,7 @@ To build from source code, runtime-tools requires Go 1.7.x or above.
 [`oci-runtime-tool generate`][generate.1] generates [configuration JSON][config.json] for an [OCI bundle][bundle].
 [OCI-compatible runtimes][runtime-spec] like [runC][] expect to read the configuration from `config.json`.
 
-```sh
+```console
 $ oci-runtime-tool generate --output config.json
 $ cat config.json
 {
@@ -22,63 +22,59 @@ $ cat config.json
 [`oci-runtime-tool validate`][validate.1] validates an OCI bundle.
 The error message will be printed if the OCI bundle failed the validation procedure.
 
-```sh
+```console
 $ oci-runtime-tool generate
 $ oci-runtime-tool validate
 INFO[0000] Bundle validation succeeded.
 ```
 
 ## Testing OCI runtimes
 
-```sh
+The runtime validation suite uses [`prove`][prove], which is packaged for most distributions (for example, it is in [Debian's `perl` package][debian-perl] and [Gentoo's `dev-lang/perl` package][gentoo-perl]).
+If you cannot install `prove`, you can probably run the test suite with another [TAP consumer][tap-consumers], although you'll have to edit the [`Makefile`](Makefile) to replace `prove`.
+
+```console
 $ sudo make RUNTIME=runc localvalidation
-RUNTIME=runc go test -tags ""  -v github.com/opencontainers/runtime-tools/validation
-=== RUN   TestValidateBasic
-TAP version 13
-ok 1 - root filesystem
-ok 2 - hostname
-ok 3 - mounts
-ok 4 - capabilities
-ok 5 - default symlinks
-ok 6 - default devices
-ok 7 - linux devices
-ok 8 - linux process
-ok 9 - masked paths
-ok 10 - oom score adj
-ok 11 - read only paths
-ok 12 - rlimits
-ok 13 - sysctls
-ok 14 - uid mappings
-ok 15 - gid mappings
-1..15
---- PASS: TestValidateBasic (0.08s)
-=== RUN   TestValidateSysctls
+CGO_ENABLED=0 go build -installsuffix cgo -tags "" -ldflags "-X main.gitCommit=b2677cb843676285b0504fa49324dc7bb722044f -X main.version=0.3.0" -o runtimetest ./cmd/runtimetest
+go build -tags ""  -o validation/linux_rootfs_propagation_shared.t validation/linux_rootfs_propagation_shared.go
+…
+go build -tags ""  -o validation/linux_uid_mappings.t validation/linux_uid_mappings.go
+RUNTIME=runc prove validation/linux_rootfs_propagation_unbindable.t validation/mounts.t validation/root_readonly_true.t validation/linux_devices.t validation/process_rlimits.t validation/linux_masked_paths.t validation/linux_readonly_paths.t validation/root_readonly_false.t validation/process_capabilities.t validation/process_oom_score_adj.t validation/process.t validation/create.t validation/hostname.t validation/linux_rootfs_propagation_shared.t validation/linux_sysctl.t validation/linux_uid_mappings.t validation/default.t validation/linux_gid_mappings.t
+…
+Files=18, Tests=21,  1 wallclock secs ( 0.06 usr  0.01 sys +  0.37 cusr  0.13 csys =  0.57 CPU)
+Result: FAIL
+make: *** [Makefile:42: localvalidation] Error 1
+```
+
+If you are confident that the `validation/*.t` are current, you can run `prove` (or your preferred TAP consumer) directly to avoid unnecessary rebuilds:
+
+```console
+$ sudo RUNTIME=runc prove -Q validation/*.t
+
+Test Summary Report
+-------------------
+…
+Files=18, Tests=21,  1 wallclock secs ( 0.05 usr  0.01 sys +  0.37 cusr  0.12 csys =  0.55 CPU)
+Result: FAIL
+```
+
+All of those failures are currently variations of our test suite not waiting on the container process:
+
+```console
+$ RUNTIME=runc validation/default.t
 TAP version 13
-ok 1 - root filesystem
-ok 2 - hostname
-ok 3 - mounts
-ok 4 - capabilities
-ok 5 - default symlinks
-ok 6 - default devices
-ok 7 - linux devices
-ok 8 - linux process
-ok 9 - masked paths
-ok 10 - oom score adj
-ok 11 - read only paths
-ok 12 - rlimits
-ok 13 - sysctls
-ok 14 - uid mappings
-ok 15 - gid mappings
-1..15
---- PASS: TestValidateSysctls (0.20s)
-PASS
-ok      github.com/opencontainers/runtime-tools/validation      0.281s
+1..1
+not ok 1 - FIXME: we don't wait for the container to exit or collect its stdout
 ```
 
 [bundle]: https://github.com/opencontainers/runtime-spec/blob/master/bundle.md
 [config.json]: https://github.com/opencontainers/runtime-spec/blob/master/config.md
+[debian-perl]: https://packages.debian.org/stretch/perl
+[gentoo-perl]: https://packages.gentoo.org/packages/dev-lang/perl
+[prove]: http://search.cpan.org/~leont/Test-Harness-3.39/bin/prove
 [runC]: https://github.com/opencontainers/runc
 [runtime-spec]: https://github.com/opencontainers/runtime-spec
+[tap-consumers]: https://testanything.org/consumers.html
 
 [generate.1]: man/oci-runtime-tool-generate.1.md
 [validate.1]: man/oci-runtime-tool-validate.1.md

validation/.gitignore

@@ -0,0 +1 @@
+/*.t

validation/create.go

@@ -0,0 +1,68 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/mndrix/tap-go"
+	rspecs "github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/opencontainers/runtime-tools/generate"
+	"github.com/opencontainers/runtime-tools/specerror"
+	"github.com/opencontainers/runtime-tools/validation/util"
+	"github.com/satori/go.uuid"
+)
+
+func main() {
+	t := tap.New()
+
+	g := generate.New()
+	g.SetRootPath(".")
+	g.SetProcessArgs([]string{"ls"})
+
+	bundleDir, err := util.PrepareBundle()
+	if err != nil {
+		util.Fatal(err)
+	}
+
+	r, err := util.NewRuntime(util.RuntimeCommand, bundleDir)
+	if err != nil {
+		util.Fatal(err)
+	}
+	defer r.Clean(true)
+
+	err = r.SetConfig(&g)
+	if err != nil {
+		util.Fatal(err)
+	}
+
+	containerID := uuid.NewV4().String()
+	cases := []struct {
+		id          string
+		errExpected bool
+		err         error
+	}{
+		{"", false, specerror.NewError(specerror.CreateWithBundlePathAndID, fmt.Errorf("create MUST generate an error if the ID is not provided"), rspecs.Version)},
+		{containerID, true, specerror.NewError(specerror.CreateNewContainer, fmt.Errorf("create MUST create a new container"), rspecs.Version)},
+		{containerID, false, specerror.NewError(specerror.CreateWithUniqueID, fmt.Errorf("create MUST generate an error if the ID provided is not unique"), rspecs.Version)},
+	}
+
+	for _, c := range cases {
+		r.SetID(c.id)
+		stderr, err := r.Create()
+		t.Ok((err == nil) == c.errExpected, c.err.(*specerror.Error).Err.Err.Error())
+		t.Diagnostic(c.err.(*specerror.Error).Err.Reference)
+		if err != nil {
+			t.Diagnostic(err.Error())
+		}
+		if len(stderr) > 0 {
+			t.Diagnostic(string(stderr))
+		}
+
+		if err == nil {
+			state, _ := r.State()
+			t.Ok(state.ID == c.id, "")
+			t.Diagnosticf("container PID: %d, state ID: %d", c.id, state.ID)
+		}
+	}
+
+	t.AutoPlan()
+}

validation/default.go

@@ -0,0 +1,13 @@
+package main
+
+import (
+	"github.com/opencontainers/runtime-tools/validation/util"
+)
+
+func main() {
+	g := util.GetDefaultGenerator()
+	err := util.RuntimeInsideValidate(g, nil)
+	if err != nil {
+		util.Fatal(err)
+	}
+}

validation/hostname.go

@@ -0,0 +1,14 @@
+package main
+
+import (
+	"github.com/opencontainers/runtime-tools/validation/util"
+)
+
+func main() {
+	g := util.GetDefaultGenerator()
+	g.SetHostname("hostname-specific")
+	err := util.RuntimeInsideValidate(g, nil)
+	if err != nil {
+		util.Fatal(err)
+	}
+}

validation/linux_devices.go

@@ -0,0 +1,55 @@
+package main
+
+import (
+	"os"
+
+	rspecs "github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/opencontainers/runtime-tools/validation/util"
+)
+
+func main() {
+	g := util.GetDefaultGenerator()
+
+	// add char device
+	cdev := rspecs.LinuxDevice{}
+	cdev.Path = "/dev/test1"
+	cdev.Type = "c"
+	cdev.Major = 10
+	cdev.Minor = 666
+	cmode := os.FileMode(int32(432))
+	cdev.FileMode = &cmode
+	cuid := uint32(0)
+	cdev.UID = &cuid
+	cgid := uint32(0)
+	cdev.GID = &cgid
+	g.AddDevice(cdev)
+
+	// add block device
+	bdev := rspecs.LinuxDevice{}
+	bdev.Path = "/dev/test2"
+	bdev.Type = "b"
+	bdev.Major = 8
+	bdev.Minor = 666
+	bmode := os.FileMode(int32(432))
+	bdev.FileMode = &bmode
+	uid := uint32(0)
+	bdev.UID = &uid
+	gid := uint32(0)
+	bdev.GID = &gid
+	g.AddDevice(bdev)
+
+	// add fifo device
+	pdev := rspecs.LinuxDevice{}
+	pdev.Path = "/dev/test3"
+	pdev.Type = "p"
+	pdev.Major = 8
+	pdev.Minor = 666
+	pmode := os.FileMode(int32(432))
+	pdev.FileMode = &pmode
+	g.AddDevice(pdev)
+
+	err := util.RuntimeInsideValidate(g, nil)
+	if err != nil {
+		util.Fatal(err)
+	}
+}

validation/linux_gid_mappings.go

@@ -0,0 +1,14 @@
+package main
+
+import (
+	"github.com/opencontainers/runtime-tools/validation/util"
+)
+
+func main() {
+	g := util.GetDefaultGenerator()
+	g.AddLinuxGIDMapping(uint32(1000), uint32(0), uint32(3200))
+	err := util.RuntimeInsideValidate(g, nil)
+	if err != nil {
+		util.Fatal(err)
+	}
+}

validation/linux_masked_paths.go

@@ -0,0 +1,20 @@
+package main
+
+import (
+	"os"
+	"path/filepath"
+
+	"github.com/opencontainers/runtime-tools/validation/util"
+)
+
+func main() {
+	g := util.GetDefaultGenerator()
+	g.AddLinuxMaskedPaths("/masktest")
+	err := util.RuntimeInsideValidate(g, func(path string) error {
+		pathName := filepath.Join(path, "masktest")
+		return os.MkdirAll(pathName, 0700)
+	})
+	if err != nil {
+		util.Fatal(err)
+	}
+}

validation/linux_readonly_paths.go

@@ -0,0 +1,20 @@
+package main
+
+import (
+	"os"
+	"path/filepath"
+
+	"github.com/opencontainers/runtime-tools/validation/util"
+)
+
+func main() {
+	g := util.GetDefaultGenerator()
+	g.AddLinuxReadonlyPaths("readonlytest")
+	err := util.RuntimeInsideValidate(g, func(path string) error {
+		pathName := filepath.Join(path, "readonlytest")
+		return os.MkdirAll(pathName, 0700)
+	})
+	if err != nil {
+		util.Fatal(err)
+	}
+}