Introduce OCI Conformance Test Suite

Added new conformance directory in the project root, with a number of
test files written in Go. Tests can be compiled by running `go test -c`
in the conformance directory and executing the created conformance.test
file.

In order for the tests to run, registry providers will need to set up
certain environment variables with the root url, the namespace of a
repository, and authentication information. Additionally, the OCI_DEBUG
variable can be set to "true" for more detailed output.

The tests create two report files: report.html and junit.xml. The html
report is expandable if more detailed information is needed on failures.

Related to opencontainers#24

Signed-off-by: Peter Engelbert <pmengelbert@gmail.com>

.gitignore

@@ -1,2 +1,5 @@
+.idea/
+.vscode/
 output
 header.html
+tags

conformance/.gitignore

@@ -0,0 +1,6 @@
+vendor/
+junit.xml
+report.html
+conformance.test
+tags
+env.sh

conformance/00_conformance_suite_test.go

@@ -0,0 +1,25 @@
+package conformance
+
+import (
+	"testing"
+
+	g "github.com/onsi/ginkgo"
+	"github.com/onsi/ginkgo/reporters"
+	. "github.com/onsi/gomega"
+)
+
+func TestConformance(t *testing.T) {
+	g.Describe(suiteDescription, func() {
+		test01BaseAPIRoute()
+		test02BlobUploadStreamed()
+		test03BlobUploadMonolithic()
+		test04BlobUploadChunked()
+		test05ManifestUpload()
+		test06TagsList()
+		test07ManifestDelete()
+		test08BlobDelete()
+	})
+	RegisterFailHandler(g.Fail)
+	reporters := []g.Reporter{newHTMLReporter(reportHTMLFilename), reporters.NewJUnitReporter(reportJUnitFilename)}
+	g.RunSpecsWithDefaultAndCustomReporters(t, suiteDescription, reporters)
+}

conformance/01_base_api_route_test.go

@@ -0,0 +1,20 @@
+package conformance
+
+import (
+	"net/http"
+
+	"github.com/bloodorangeio/reggie"
+	g "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var test01BaseAPIRoute = func() {
+	g.Context("Base API Route", func() {
+		g.Specify("GET request to base API route must return 200 response", func() {
+			req := client.NewRequest(reggie.GET, "/v2/")
+			resp, err := client.Do(req)
+			Expect(err).To(BeNil())
+			Expect(resp.StatusCode()).To(Equal(http.StatusOK))
+		})
+	})
+}

conformance/02_blob_upload_streamed_test.go

@@ -0,0 +1,38 @@
+package conformance
+
+import (
+	"net/http"
+
+	"github.com/bloodorangeio/reggie"
+	g "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var test02BlobUploadStreamed = func() {
+	g.Context("Blob Upload Streamed", func() {
+		g.Specify("PATCH request with blob in body should yield 202 response", func() {
+			req := client.NewRequest(reggie.POST, "/v2/<name>/blobs/uploads/")
+			resp, err := client.Do(req)
+			Expect(err).To(BeNil())
+			lastResponse = resp
+
+			req = client.NewRequest(reggie.PATCH, lastResponse.GetRelativeLocation()).
+				SetHeader("Content-Type", "application/octet-stream").
+				SetBody(blobA)
+			resp, err = client.Do(req)
+			Expect(err).To(BeNil())
+			Expect(resp.StatusCode()).To(Equal(http.StatusAccepted))
+			lastResponse = resp
+		})
+
+		g.Specify("PUT request to session URL with digest should yield 201 response", func() {
+			req := client.NewRequest(reggie.PUT, lastResponse.GetRelativeLocation()).
+				SetQueryParam("digest", blobADigest).
+				SetHeader("Content-Type", "application/octet-stream").
+				SetHeader("Content-Length", blobALength)
+			resp, err := client.Do(req)
+			Expect(err).To(BeNil())
+			Expect(resp.StatusCode()).To(Equal(http.StatusCreated))
+		})
+	})
+}

conformance/03_blob_upload_monolithic_test.go

@@ -0,0 +1,47 @@
+package conformance
+
+import (
+	"net/http"
+
+	"github.com/bloodorangeio/reggie"
+	g "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var test03BlobUploadMonolithic = func() {
+	g.Context("Blob Upload Monolithic", func() {
+		g.Specify("GET nonexistent blob should result in 404 response", func() {
+			req := client.NewRequest(reggie.GET, "/v2/<name>/blobs/<digest>",
+				reggie.WithDigest(dummyDigest))
+			resp, err := client.Do(req)
+			Expect(err).To(BeNil())
+			Expect(resp.StatusCode()).To(Equal(http.StatusNotFound))
+		})
+
+		g.Specify("POST request should yield a session ID", func() {
+			req := client.NewRequest(reggie.POST, "/v2/<name>/blobs/uploads/")
+			resp, err := client.Do(req)
+			Expect(err).To(BeNil())
+			Expect(resp.StatusCode()).To(Equal(http.StatusAccepted))
+			lastResponse = resp
+		})
+
+		g.Specify("PUT upload of a blob should yield a 201 Response", func() {
+			req := client.NewRequest(reggie.PUT, lastResponse.GetRelativeLocation()).
+				SetHeader("Content-Length", configContentLength).
+				SetHeader("Content-Type", "application/octet-stream").
+				SetQueryParam("digest", configDigest).
+				SetBody(configContent)
+			resp, err := client.Do(req)
+			Expect(err).To(BeNil())
+			Expect(resp.StatusCode()).To(Equal(http.StatusCreated))
+		})
+
+		g.Specify("GET request to existing blob should yield 200 response", func() {
+			req := client.NewRequest(reggie.GET, "/v2/<name>/blobs/<digest>", reggie.WithDigest(configDigest))
+			resp, err := client.Do(req)
+			Expect(err).To(BeNil())
+			Expect(resp.StatusCode()).To(Equal(http.StatusOK))
+		})
+	})
+}

conformance/04_blob_upload_chunked_test.go

@@ -0,0 +1,43 @@
+package conformance
+
+import (
+	"net/http"
+
+	"github.com/bloodorangeio/reggie"
+	g "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var test04BlobUploadChunked = func() {
+	g.Context("Blob Upload Chunked", func() {
+		g.Specify("PATCH request with first chunk should return 202", func() {
+			req := client.NewRequest(reggie.POST, "/v2/<name>/blobs/uploads/").
+				SetHeader("Content-Length", "0")
+			resp, err := client.Do(req)
+			Expect(err).To(BeNil())
+			lastResponse = resp
+
+			req = client.NewRequest(reggie.PATCH, lastResponse.GetRelativeLocation()).
+				SetHeader("Content-Type", "application/octet-stream").
+				SetHeader("Content-Length", blobBChunk1Length).
+				SetHeader("Content-Range", blobBChunk1Range).
+				SetBody(blobBChunk1)
+			resp, err = client.Do(req)
+			Expect(err).To(BeNil())
+			Expect(resp.StatusCode()).To(Equal(http.StatusAccepted))
+			lastResponse = resp
+		})
+
+		g.Specify("PUT request with final chunk should return 201", func() {
+			req := client.NewRequest(reggie.PUT, lastResponse.GetRelativeLocation()).
+				SetHeader("Content-Length", blobBChunk2Length).
+				SetHeader("Content-Range", blobBChunk2Range).
+				SetHeader("Content-Type", "application/octet-stream").
+				SetQueryParam("digest", blobBDigest).
+				SetBody(blobBChunk2)
+			resp, err := client.Do(req)
+			Expect(err).To(BeNil())
+			Expect(resp.StatusCode()).To(Equal(http.StatusCreated))
+		})
+	})
+}

conformance/05_manifest_upload_test.go

@@ -0,0 +1,42 @@
+package conformance
+
+import (
+	"fmt"
+	"net/http"
+
+	"github.com/bloodorangeio/reggie"
+	g "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var test05ManifestUpload = func() {
+	g.Context("Manifest Upload", func() {
+		g.Specify("GET nonexistent manifest should return 404", func() {
+			req := client.NewRequest(reggie.GET, "/v2/<name>/manifests/<reference>",
+				reggie.WithReference(nonexistentManifest))
+			resp, err := client.Do(req)
+			Expect(err).To(BeNil())
+			Expect(resp.StatusCode()).To(Equal(http.StatusNotFound))
+		})
+
+		g.Specify("PUT should accept a manifest upload", func() {
+			for i := 0; i < 4; i++ {
+				req := client.NewRequest(reggie.PUT, "/v2/<name>/manifests/<reference>",
+					reggie.WithReference(fmt.Sprintf("test%d", i))).
+					SetHeader("Content-Type", "application/vnd.oci.image.manifest.v1+json").
+					SetBody(manifestContent)
+				resp, err := client.Do(req)
+				Expect(err).To(BeNil())
+				Expect(resp.StatusCode()).To(Equal(http.StatusCreated))
+			}
+		})
+
+		g.Specify("GET request to manifest URL (digest) should yield 200 response", func() {
+			req := client.NewRequest(reggie.GET, "/v2/<name>/manifests/<digest>", reggie.WithDigest(manifestDigest)).
+				SetHeader("Accept", "application/vnd.oci.image.manifest.v1+json")
+			resp, err := client.Do(req)
+			Expect(err).To(BeNil())
+			Expect(resp.StatusCode()).To(Equal(http.StatusOK))
+		})
+	})
+}

conformance/06_tags_list_test.go

@@ -0,0 +1,72 @@
+package conformance
+
+import (
+	"encoding/json"
+	"net/http"
+	"strconv"
+
+	"github.com/bloodorangeio/reggie"
+	g "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var test06TagsList = func() {
+	g.Context("Tags List", func() {
+		g.Specify("GET request to list tags should yield 200 response", func() {
+			req := client.NewRequest(reggie.GET, "/v2/<name>/tags/list")
+			resp, err := client.Do(req)
+			Expect(err).To(BeNil())
+			Expect(resp.StatusCode()).To(Equal(http.StatusOK))
+			lastResponse = resp
+			tagList := &TagList{}
+			jsonData := []byte(resp.String())
+			err = json.Unmarshal(jsonData, tagList)
+			Expect(err).To(BeNil())
+			numTags = len(tagList.Tags)
+		})
+
+		g.Specify("GET request to manifest URL (tag) should yield 200 response", func() {
+			tl := &TagList{}
+			jsonData := lastResponse.Body()
+			err := json.Unmarshal(jsonData, tl)
+			Expect(err).To(BeNil())
+			Expect(tl.Tags).ToNot(BeEmpty())
+			req := client.NewRequest(reggie.GET, "/v2/<name>/manifests/<reference>",
+				reggie.WithReference(tl.Tags[0])).
+				SetHeader("Accept", "application/vnd.oci.image.manifest.v1+json")
+			resp, err := client.Do(req)
+			Expect(err).To(BeNil())
+			Expect(resp.StatusCode()).To(Equal(http.StatusOK))
+		})
+
+		g.Specify("GET number of tags should be limitable by `n` query parameter", func() {
+			numResults := numTags / 2
+			req := client.NewRequest(reggie.GET, "/v2/<name>/tags/list").
+				SetQueryParam("n", strconv.Itoa(numResults))
+			resp, err := client.Do(req)
+			Expect(err).To(BeNil())
+			Expect(resp.StatusCode()).To(Equal(http.StatusOK))
+			jsonData := resp.Body()
+			tagList := &TagList{}
+			err = json.Unmarshal(jsonData, tagList)
+			Expect(err).To(BeNil())
+			Expect(len(tagList.Tags)).To(Equal(numResults))
+			lastTagList = *tagList
+		})
+
+		g.Specify("GET start of tag is set by `last` query parameter", func() {
+			numResults := numTags / 2
+			req := client.NewRequest(reggie.GET, "/v2/<name>/tags/list").
+				SetQueryParam("n", strconv.Itoa(numResults)).
+				SetQueryParam("last", lastTagList.Tags[numResults-1])
+			resp, err := client.Do(req)
+			Expect(err).To(BeNil())
+			Expect(resp.StatusCode()).To(Equal(http.StatusOK))
+			jsonData := resp.Body()
+			tagList := &TagList{}
+			err = json.Unmarshal(jsonData, tagList)
+			Expect(err).To(BeNil())
+			Expect(tagList.Tags).To(ContainElement("test3"))
+		})
+	})
+}

conformance/07_manifest_delete_test.go

@@ -0,0 +1,40 @@
+package conformance
+
+import (
+	"encoding/json"
+	"net/http"
+
+	"github.com/bloodorangeio/reggie"
+	g "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var test07ManifestDelete = func() {
+	g.Context("Manifest Delete", func() {
+		g.Specify("DELETE request to manifest URL should yield 202 response", func() {
+			req := client.NewRequest(reggie.DELETE, "/v2/<name>/manifests/<digest>", reggie.WithDigest(manifestDigest))
+			resp, err := client.Do(req)
+			Expect(err).To(BeNil())
+			Expect(resp.StatusCode()).To(Equal(http.StatusAccepted))
+		})
+
+		g.Specify("GET request to deleted manifest URL should yield 404 response", func() {
+			req := client.NewRequest(reggie.GET, "/v2/<name>/manifests/<digest>", reggie.WithDigest(manifestDigest))
+			resp, err := client.Do(req)
+			Expect(err).To(BeNil())
+			Expect(resp.StatusCode()).To(Equal(http.StatusNotFound))
+		})
+
+		g.Specify("GET request to tags list should reflect manifest deletion", func() {
+			req := client.NewRequest(reggie.GET, "/v2/<name>/tags/list")
+			resp, err := client.Do(req)
+			Expect(err).To(BeNil())
+			Expect(resp.StatusCode()).To(Equal(http.StatusOK))
+			tagList := &TagList{}
+			jsonData := []byte(resp.String())
+			err = json.Unmarshal(jsonData, tagList)
+			Expect(err).To(BeNil())
+			Expect(len(tagList.Tags)).To(BeNumerically("<", numTags))
+		})
+	})
+}

conformance/08_blob_delete_test.go

@@ -0,0 +1,27 @@
+package conformance
+
+import (
+	"net/http"
+
+	"github.com/bloodorangeio/reggie"
+	g "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var test08BlobDelete = func() {
+	g.Context("Blob Delete", func() {
+		g.Specify("DELETE request to blob URL should yield 202 response", func() {
+			req := client.NewRequest(reggie.DELETE, "/v2/<name>/blobs/<digest>", reggie.WithDigest(configDigest))
+			resp, err := client.Do(req)
+			Expect(err).To(BeNil())
+			Expect(resp.StatusCode()).To(Equal(http.StatusAccepted))
+		})
+
+		g.Specify("GET request to deleted blob URL should yield 404 response", func() {
+			req := client.NewRequest(reggie.GET, "/v2/<name>/blobs/<digest>", reggie.WithDigest(configDigest))
+			resp, err := client.Do(req)
+			Expect(err).To(BeNil())
+			Expect(resp.StatusCode()).To(Equal(http.StatusNotFound))
+		})
+	})
+}