From 303f0767301a1899cf22b4991187645e0f2cd1a6 Mon Sep 17 00:00:00 2001 From: Peter Engelbert Date: Thu, 16 Jan 2020 17:54:33 -0600 Subject: [PATCH] Introduce OCI Conformance Test Suite Added new conformance directory in the project root, with a number of test files written in Go. Tests can be compiled by running `go test -c` in the conformance directory and executing the created conformance.test file. In order for the tests to run, registry providers will need to set up certain environment variables with the root url, the namespace of a repository, and authentication information. Additionally, the OCI_DEBUG variable can be set to "true" for more detailed output. The tests create two report files: report.html and junit.xml. The html report is expandable if more detailed information is needed on failures. Related to #24 Signed-off-by: Peter Engelbert --- .gitignore | 3 + conformance/.gitignore | 6 + conformance/00_conformance_suite_test.go | 25 ++ conformance/01_base_api_route_test.go | 20 ++ conformance/02_blob_upload_streamed_test.go | 39 +++ conformance/03_blob_upload_monolithic_test.go | 47 +++ conformance/04_blob_upload_chunked_test.go | 44 +++ conformance/05_manifest_upload_test.go | 42 +++ conformance/06_tags_list_test.go | 72 +++++ conformance/07_manifest_delete_test.go | 40 +++ conformance/08_blob_delete_test.go | 27 ++ conformance/README.md | 30 ++ conformance/go.mod | 10 + conformance/go.sum | 42 +++ conformance/reporter.go | 287 ++++++++++++++++++ conformance/setup.go | 93 ++++++ 16 files changed, 827 insertions(+) create mode 100644 conformance/.gitignore create mode 100644 conformance/00_conformance_suite_test.go create mode 100644 conformance/01_base_api_route_test.go create mode 100644 conformance/02_blob_upload_streamed_test.go create mode 100644 conformance/03_blob_upload_monolithic_test.go create mode 100644 conformance/04_blob_upload_chunked_test.go create mode 100644 conformance/05_manifest_upload_test.go create mode 100644 conformance/06_tags_list_test.go create mode 100644 conformance/07_manifest_delete_test.go create mode 100644 conformance/08_blob_delete_test.go create mode 100644 conformance/README.md create mode 100644 conformance/go.mod create mode 100644 conformance/go.sum create mode 100644 conformance/reporter.go create mode 100644 conformance/setup.go diff --git a/.gitignore b/.gitignore index 7674b69b..ed66dcd3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,5 @@ +.idea/ +.vscode/ output header.html +tags diff --git a/conformance/.gitignore b/conformance/.gitignore new file mode 100644 index 00000000..f5a764ee --- /dev/null +++ b/conformance/.gitignore @@ -0,0 +1,6 @@ +vendor/ +junit.xml +report.html +conformance.test +tags +env.sh diff --git a/conformance/00_conformance_suite_test.go b/conformance/00_conformance_suite_test.go new file mode 100644 index 00000000..33a45ba9 --- /dev/null +++ b/conformance/00_conformance_suite_test.go @@ -0,0 +1,25 @@ +package conformance + +import ( + "testing" + + g "github.com/onsi/ginkgo" + "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" +) + +func TestConformance(t *testing.T) { + g.Describe(suiteDescription, func() { + test01BaseAPIRoute() + test02BlobUploadStreamed() + test03BlobUploadMonolithic() + test04BlobUploadChunked() + test05ManifestUpload() + test06TagsList() + test07ManifestDelete() + test08BlobDelete() + }) + RegisterFailHandler(g.Fail) + reporters := []g.Reporter{newHTMLReporter(reportHTMLFilename), reporters.NewJUnitReporter(reportJUnitFilename)} + g.RunSpecsWithDefaultAndCustomReporters(t, suiteDescription, reporters) +} diff --git a/conformance/01_base_api_route_test.go b/conformance/01_base_api_route_test.go new file mode 100644 index 00000000..36e895a4 --- /dev/null +++ b/conformance/01_base_api_route_test.go @@ -0,0 +1,20 @@ +package conformance + +import ( + "net/http" + + "github.com/bloodorangeio/reggie" + g "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var test01BaseAPIRoute = func() { + g.Context("Base API Route", func() { + g.Specify("GET request to base API route must return 200 response", func() { + req := client.NewRequest(reggie.GET, "/v2/") + resp, err := client.Do(req) + Expect(err).To(BeNil()) + Expect(resp.StatusCode()).To(Equal(http.StatusOK)) + }) + }) +} diff --git a/conformance/02_blob_upload_streamed_test.go b/conformance/02_blob_upload_streamed_test.go new file mode 100644 index 00000000..3a8a356d --- /dev/null +++ b/conformance/02_blob_upload_streamed_test.go @@ -0,0 +1,39 @@ +package conformance + +import ( + "net/http" + "strconv" + + "github.com/bloodorangeio/reggie" + g "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var test02BlobUploadStreamed = func() { + g.Context("Blob Upload Streamed", func() { + g.Specify("PATCH request with blob in body should yield 202 response", func() { + req := client.NewRequest(reggie.POST, "/v2//blobs/uploads/") + resp, err := client.Do(req) + Expect(err).To(BeNil()) + lastResponse = resp + + req = client.NewRequest(reggie.PATCH, lastResponse.GetRelativeLocation()). + SetHeader("Content-Type", "application/octet-stream"). + SetBody(blobA) + resp, err = client.Do(req) + Expect(err).To(BeNil()) + Expect(resp.StatusCode()).To(Equal(http.StatusAccepted)) + lastResponse = resp + }) + + g.Specify("PUT request to session URL with digest should yield 201 response", func() { + req := client.NewRequest(reggie.PUT, lastResponse.GetRelativeLocation()). + SetQueryParam("digest", blobADigest). + SetHeader("Content-Type", "application/octet-stream"). + SetHeader("Content-Length", strconv.Itoa(len(blobA))) + resp, err := client.Do(req) + Expect(err).To(BeNil()) + Expect(resp.StatusCode()).To(Equal(http.StatusCreated)) + }) + }) +} diff --git a/conformance/03_blob_upload_monolithic_test.go b/conformance/03_blob_upload_monolithic_test.go new file mode 100644 index 00000000..81134bea --- /dev/null +++ b/conformance/03_blob_upload_monolithic_test.go @@ -0,0 +1,47 @@ +package conformance + +import ( + "net/http" + + "github.com/bloodorangeio/reggie" + g "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var test03BlobUploadMonolithic = func() { + g.Context("Blob Upload Monolithic", func() { + g.Specify("GET nonexistent blob should result in 404 response", func() { + req := client.NewRequest(reggie.GET, "/v2//blobs/", + reggie.WithDigest("sha256:a948904f2f0f479b8f8197694b30184b0d2ed1c1cd2a1ec0fb85d299a192a447")) + resp, err := client.Do(req) + Expect(err).To(BeNil()) + Expect(resp.StatusCode()).To(Equal(http.StatusNotFound)) + }) + + g.Specify("POST request should yield a session ID", func() { + req := client.NewRequest(reggie.POST, "/v2//blobs/uploads/") + resp, err := client.Do(req) + Expect(err).To(BeNil()) + Expect(resp.StatusCode()).To(Equal(http.StatusAccepted)) + lastResponse = resp + }) + + g.Specify("PUT upload of a blob should yield a 201 Response", func() { + req := client.NewRequest(reggie.PUT, lastResponse.GetRelativeLocation()). + SetHeader("Content-Length", configContentLength). + SetHeader("Content-Type", "application/octet-stream"). + SetQueryParam("digest", configDigest). + SetBody(configContent) + resp, err := client.Do(req) + Expect(err).To(BeNil()) + Expect(resp.StatusCode()).To(Equal(http.StatusCreated)) + }) + + g.Specify("GET request to existing blob should yield 200 response", func() { + req := client.NewRequest(reggie.GET, "/v2//blobs/", reggie.WithDigest(configDigest)) + resp, err := client.Do(req) + Expect(err).To(BeNil()) + Expect(resp.StatusCode()).To(Equal(http.StatusOK)) + }) + }) +} diff --git a/conformance/04_blob_upload_chunked_test.go b/conformance/04_blob_upload_chunked_test.go new file mode 100644 index 00000000..8b67fce0 --- /dev/null +++ b/conformance/04_blob_upload_chunked_test.go @@ -0,0 +1,44 @@ +package conformance + +import ( + "fmt" + "net/http" + + "github.com/bloodorangeio/reggie" + g "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var test04BlobUploadChunked = func() { + g.Context("Blob Upload Chunked", func() { + g.Specify("PATCH request with first chunk should return 202", func() { + req := client.NewRequest(reggie.POST, "/v2//blobs/uploads/"). + SetHeader("Content-Length", "0") + resp, err := client.Do(req) + Expect(err).To(BeNil()) + lastResponse = resp + + req = client.NewRequest(reggie.PATCH, lastResponse.GetRelativeLocation()). + SetHeader("Content-Type", "application/octet-stream"). + SetHeader("Content-Length", fmt.Sprintf("%d", len(blobBChunk1))). + SetHeader("Content-Range", blobBChunk1Range). + SetBody(blobBChunk1) + resp, err = client.Do(req) + Expect(err).To(BeNil()) + Expect(resp.StatusCode()).To(Equal(http.StatusAccepted)) + lastResponse = resp + }) + + g.Specify("PUT request with final chunk should return 201", func() { + req := client.NewRequest(reggie.PUT, lastResponse.GetRelativeLocation()). + SetHeader("Content-Length", fmt.Sprintf("%d", len(blobBChunk2))). + SetHeader("Content-Range", blobBChunk2Range). + SetHeader("Content-Type", "application/octet-stream"). + SetQueryParam("digest", blobBDigest). + SetBody(blobBChunk2) + resp, err := client.Do(req) + Expect(err).To(BeNil()) + Expect(resp.StatusCode()).To(Equal(http.StatusCreated)) + }) + }) +} diff --git a/conformance/05_manifest_upload_test.go b/conformance/05_manifest_upload_test.go new file mode 100644 index 00000000..1254283b --- /dev/null +++ b/conformance/05_manifest_upload_test.go @@ -0,0 +1,42 @@ +package conformance + +import ( + "fmt" + "net/http" + + "github.com/bloodorangeio/reggie" + g "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var test05ManifestUpload = func() { + g.Context("Manifest Upload", func() { + g.Specify("GET nonexistent manifest should return 404", func() { + req := client.NewRequest(reggie.GET, "/v2//manifests/", + reggie.WithReference(nonexistentManifest)) + resp, err := client.Do(req) + Expect(err).To(BeNil()) + Expect(resp.StatusCode()).To(Equal(http.StatusNotFound)) + }) + + g.Specify("PUT should accept a manifest upload", func() { + for i := 0; i < 4; i++ { + req := client.NewRequest(reggie.PUT, "/v2//manifests/", + reggie.WithReference(fmt.Sprintf("test%d", i))). + SetHeader("Content-Type", "application/vnd.oci.image.manifest.v1+json"). + SetBody(manifestContent) + resp, err := client.Do(req) + Expect(err).To(BeNil()) + Expect(resp.StatusCode()).To(Equal(http.StatusCreated)) + } + }) + + g.Specify("GET request to manifest URL (digest) should yield 200 response", func() { + req := client.NewRequest(reggie.GET, "/v2//manifests/", reggie.WithDigest(manifestDigest)). + SetHeader("Accept", "application/vnd.oci.image.manifest.v1+json") + resp, err := client.Do(req) + Expect(err).To(BeNil()) + Expect(resp.StatusCode()).To(Equal(http.StatusOK)) + }) + }) +} diff --git a/conformance/06_tags_list_test.go b/conformance/06_tags_list_test.go new file mode 100644 index 00000000..6ed073d7 --- /dev/null +++ b/conformance/06_tags_list_test.go @@ -0,0 +1,72 @@ +package conformance + +import ( + "encoding/json" + "net/http" + "strconv" + + "github.com/bloodorangeio/reggie" + g "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var test06TagsList = func() { + g.Context("Tags List", func() { + g.Specify("GET request to list tags should yield 200 response", func() { + req := client.NewRequest(reggie.GET, "/v2//tags/list") + resp, err := client.Do(req) + Expect(err).To(BeNil()) + Expect(resp.StatusCode()).To(Equal(http.StatusOK)) + lastResponse = resp + tagList := &TagList{} + jsonData := []byte(resp.String()) + err = json.Unmarshal(jsonData, tagList) + Expect(err).To(BeNil()) + numTags = len(tagList.Tags) + }) + + g.Specify("GET request to manifest URL (tag) should yield 200 response", func() { + tl := &TagList{} + jsonData := lastResponse.Body() + err := json.Unmarshal(jsonData, tl) + Expect(err).To(BeNil()) + Expect(tl.Tags).ToNot(BeEmpty()) + req := client.NewRequest(reggie.GET, "/v2//manifests/", + reggie.WithReference(tl.Tags[0])). + SetHeader("Accept", "application/vnd.oci.image.manifest.v1+json") + resp, err := client.Do(req) + Expect(err).To(BeNil()) + Expect(resp.StatusCode()).To(Equal(http.StatusOK)) + }) + + g.Specify("GET number of tags should be limitable by `n` query parameter", func() { + numResults := numTags / 2 + req := client.NewRequest(reggie.GET, "/v2//tags/list"). + SetQueryParam("n", strconv.Itoa(numResults)) + resp, err := client.Do(req) + Expect(err).To(BeNil()) + Expect(resp.StatusCode()).To(Equal(http.StatusOK)) + jsonData := resp.Body() + tagList := &TagList{} + err = json.Unmarshal(jsonData, tagList) + Expect(err).To(BeNil()) + Expect(len(tagList.Tags)).To(Equal(numResults)) + lastTagList = *tagList + }) + + g.Specify("GET start of tag is set by `last` query parameter", func() { + numResults := numTags / 2 + req := client.NewRequest(reggie.GET, "/v2//tags/list"). + SetQueryParam("n", strconv.Itoa(numResults)). + SetQueryParam("last", lastTagList.Tags[numResults-1]) + resp, err := client.Do(req) + Expect(err).To(BeNil()) + Expect(resp.StatusCode()).To(Equal(http.StatusOK)) + jsonData := resp.Body() + tagList := &TagList{} + err = json.Unmarshal(jsonData, tagList) + Expect(err).To(BeNil()) + Expect(tagList.Tags).To(ContainElement("test3")) + }) + }) +} diff --git a/conformance/07_manifest_delete_test.go b/conformance/07_manifest_delete_test.go new file mode 100644 index 00000000..de41018e --- /dev/null +++ b/conformance/07_manifest_delete_test.go @@ -0,0 +1,40 @@ +package conformance + +import ( + "encoding/json" + "net/http" + + "github.com/bloodorangeio/reggie" + g "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var test07ManifestDelete = func() { + g.Context("Manifest Delete", func() { + g.Specify("DELETE request to manifest URL should yield 202 response", func() { + req := client.NewRequest(reggie.DELETE, "/v2//manifests/", reggie.WithDigest(manifestDigest)) + resp, err := client.Do(req) + Expect(err).To(BeNil()) + Expect(resp.StatusCode()).To(Equal(http.StatusAccepted)) + }) + + g.Specify("GET request to deleted manifest URL should yield 404 response", func() { + req := client.NewRequest(reggie.GET, "/v2//manifests/", reggie.WithDigest(manifestDigest)) + resp, err := client.Do(req) + Expect(err).To(BeNil()) + Expect(resp.StatusCode()).To(Equal(http.StatusNotFound)) + }) + + g.Specify("GET request to tags list should reflect manifest deletion", func() { + req := client.NewRequest(reggie.GET, "/v2//tags/list") + resp, err := client.Do(req) + Expect(err).To(BeNil()) + Expect(resp.StatusCode()).To(Equal(http.StatusOK)) + tagList := &TagList{} + jsonData := []byte(resp.String()) + err = json.Unmarshal(jsonData, tagList) + Expect(err).To(BeNil()) + Expect(len(tagList.Tags)).To(BeNumerically("<", numTags)) + }) + }) +} diff --git a/conformance/08_blob_delete_test.go b/conformance/08_blob_delete_test.go new file mode 100644 index 00000000..376d1d5f --- /dev/null +++ b/conformance/08_blob_delete_test.go @@ -0,0 +1,27 @@ +package conformance + +import ( + "net/http" + + "github.com/bloodorangeio/reggie" + g "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var test08BlobDelete = func() { + g.Context("Blob Delete", func() { + g.Specify("DELETE request to blob URL should yield 202 response", func() { + req := client.NewRequest(reggie.DELETE, "/v2//blobs/", reggie.WithDigest(configDigest)) + resp, err := client.Do(req) + Expect(err).To(BeNil()) + Expect(resp.StatusCode()).To(Equal(http.StatusAccepted)) + }) + + g.Specify("GET request to deleted blob URL should yield 404 response", func() { + req := client.NewRequest(reggie.GET, "/v2//blobs/", reggie.WithDigest(configDigest)) + resp, err := client.Do(req) + Expect(err).To(BeNil()) + Expect(resp.StatusCode()).To(Equal(http.StatusNotFound)) + }) + }) +} diff --git a/conformance/README.md b/conformance/README.md new file mode 100644 index 00000000..d57eabe6 --- /dev/null +++ b/conformance/README.md @@ -0,0 +1,30 @@ +## Conformance Tests + +### How to Run + +Requires Go 1.13+. + +In this directory, build the test binary: +``` +go test -c +``` + +This will produce an executable at `conformance.test`. + +Next, set environment variables with your registry details: +``` +export OCI_ROOT_URL="https://r.myreg.io" +export OCI_NAMESPACE="myorg/myrepo" +export OCI_USERNAME="myuser" +export OCI_PASSWORD="mypass" +export OCI_DEBUG="true" +``` + +Lastly, run the tests: +``` +./conformance.test +``` + +This will produce `junit.xml` and `report.html` with the results. + +Note: for some registries, you may need to create `OCI_NAMESPACE` ahead of time. diff --git a/conformance/go.mod b/conformance/go.mod new file mode 100644 index 00000000..de654041 --- /dev/null +++ b/conformance/go.mod @@ -0,0 +1,10 @@ +module github.com/opencontainers/distribution-spec/conformance + +go 1.13 + +require ( + github.com/bloodorangeio/reggie v0.3.1 + github.com/onsi/ginkgo v1.11.0 + github.com/onsi/gomega v1.8.1 + github.com/opencontainers/go-digest v1.0.0-rc1 +) diff --git a/conformance/go.sum b/conformance/go.sum new file mode 100644 index 00000000..c2899bfa --- /dev/null +++ b/conformance/go.sum @@ -0,0 +1,42 @@ +github.com/bloodorangeio/reggie v0.3.1 h1:RQSDByCdJ6fyiOZ7PiNShG7H8CwK2YvXGFADhw2qDlw= +github.com/bloodorangeio/reggie v0.3.1/go.mod h1:u7HqihAZy812d6ysiuDawpHJYtpGSNL2E/4Cyu/mtZg= +github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/go-resty/resty/v2 v2.1.0 h1:Z6IefCpUMfnvItVJaJXWv/pMiiD11So35QgwEELsldE= +github.com/go-resty/resty/v2 v2.1.0/go.mod h1:dZGr0i9PLlaaTD4H/hoZIDjQ+r6xq8mgbRzHZf7f2J8= +github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.11.0 h1:JAKSXpt1YjtLA7YpPiqO9ss6sNXEsPfSGdwN0UHqzrw= +github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.8.1 h1:C5Dqfs/LeauYDX0jJXIe2SWmwCbGzx9yF8C8xy3Lh34= +github.com/onsi/gomega v1.8.1/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= +github.com/opencontainers/distribution-spec v1.0.0-rc0.0.20200108182153-219f20cbcfa1 h1:QMs7PvjIHfWNHJFgY2BTMSDBV14dHRqb82L2BvjO8w0= +github.com/opencontainers/distribution-spec v1.0.0-rc0.0.20200108182153-219f20cbcfa1/go.mod h1:copR2flp+jTEvQIFMb6MIx45OkrxzqyjszPDT3hx/5Q= +github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ= +github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7 h1:rTIdg5QFRR7XCaK4LCjBiPbx8j4DQRpdYMnGn/bJUEU= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 h1:9zdDQZ7Thm29KFXgAX/+yaf3eVbP7djjWp/dXAppNCc= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/conformance/reporter.go b/conformance/reporter.go new file mode 100644 index 00000000..0061f542 --- /dev/null +++ b/conformance/reporter.go @@ -0,0 +1,287 @@ +package conformance + +import ( + "bytes" + "fmt" + "html/template" + "io" + "log" + "os" + "path/filepath" + "regexp" + + "github.com/onsi/ginkgo/config" + "github.com/onsi/ginkgo/types" +) + +const ( + htmlTemplate string = ` + + OCI Distribution Conformance Tests + + + + +

OCI Distribution Conformance Tests

+
+ {{with .SpecSummaryMap}} + {{$x := .M}} + {{range $i, $k := .Keys}} +

{{$k}}

+ {{$v := index $x $k}} + {{range $z, $s := $v}} + {{if eq $s.State 4}} +
+
+
+

{{$s.Title}}

+
+
+ +
+ 
{{$s.Failure.Message}}
+
+
+
+ {{else if eq $s.State 3}} +
+
+
+

{{$s.Title}}

+
+ +
+ {{end}} + {{end}} + {{end}} + {{end}} + + +` +) + +type ( + summaryMap struct { + M map[string][]specSnapshot + Keys []string + Size int + } + + specSnapshot struct { + types.SpecSummary + ID int + Title string + } + + httpDebugWriter struct { + CapturedOutput []string + debug bool + } + + httpDebugLogger struct { + l *log.Logger + w io.Writer + } +) + +func (sm *summaryMap) Add(key string, sum *specSnapshot) { + sm.M[key] = append(sm.M[key], *sum) + sm.Size++ + + if !sm.containsKey(key) { + sm.Keys = append(sm.Keys, key) + } +} + +func (sm *summaryMap) containsKey(key string) bool { + var containsKey bool + for _, k := range sm.Keys { + if k == key { + containsKey = true + break + } + } + return containsKey +} + +func newSpecSnapshot(sum *types.SpecSummary, id int) *specSnapshot { + return &specSnapshot{SpecSummary: *sum, Title: sum.ComponentTexts[3], ID: id} +} + +func newHTTPDebugWriter(debug bool) *httpDebugWriter { + return &httpDebugWriter{debug: debug} +} + +func (writer *httpDebugWriter) Write(b []byte) (int, error) { + s := string(b) + writer.CapturedOutput = append(writer.CapturedOutput, s) + if writer.debug { + fmt.Println(s) + } + + return len(b), nil +} + +func newHTTPDebugLogger(f io.Writer) *httpDebugLogger { + debugLogger := &httpDebugLogger{w: f, l: log.New(f, "", log.Ldate|log.Lmicroseconds)} + return debugLogger +} + +func (l *httpDebugLogger) Errorf(format string, v ...interface{}) { + l.output("ERROR "+format, v...) +} + +func (l *httpDebugLogger) Warnf(format string, v ...interface{}) { + l.output("WARN "+format, v...) +} + +func (l *httpDebugLogger) Debugf(format string, v ...interface{}) { + re := regexp.MustCompile("(?i)(\"?\\w*(authorization|token|state)\\w*\"?(:|=)\\s*)(\")?\\s*((bearer|basic)? )?[^\\s&\"]*(\")?") + format = re.ReplaceAllString(format, "$1$4$5*****$7") + l.output("DEBUG "+format, v...) +} + +func (l *httpDebugLogger) output(format string, v ...interface{}) { + if len(v) == 0 { + l.l.Print(format) + return + } + l.w.Write([]byte(fmt.Sprintf(format, v...))) +} + +type ( + HTMLReporter struct { + htmlReportFilename string + SpecSummaryMap summaryMap + config config.DefaultReporterConfigType + debugLogger *httpDebugWriter + debugIndex int + } +) + +func newHTMLReporter(htmlReportFilename string) *HTMLReporter { + return &HTMLReporter{ + htmlReportFilename: htmlReportFilename, + debugLogger: httpWriter, + SpecSummaryMap: summaryMap{M: make(map[string][]specSnapshot)}, + } +} + +func (reporter *HTMLReporter) SpecDidComplete(specSummary *types.SpecSummary) { + b := new(bytes.Buffer) + for _, co := range httpWriter.CapturedOutput[reporter.debugIndex:] { + b.WriteString(co) + b.WriteString("\n") + } + specSummary.CapturedOutput = b.String() + + header := specSummary.ComponentTexts[2] + summary := newSpecSnapshot(specSummary, reporter.SpecSummaryMap.Size) + reporter.SpecSummaryMap.Add(header, summary) + reporter.debugIndex = len(reporter.debugLogger.CapturedOutput) +} + +func (reporter *HTMLReporter) SpecSuiteDidEnd(summary *types.SuiteSummary) { + t, err := template.New("report").Parse(htmlTemplate) + if err != nil { + log.Fatal(err) + } + + htmlReportFilenameAbsPath, err := filepath.Abs(reporter.htmlReportFilename) + if err != nil { + log.Fatal(err) + } + + htmlReportFile, err := os.Create(htmlReportFilenameAbsPath) + if err != nil { + log.Fatal(err) + } + defer htmlReportFile.Close() + + err = t.ExecuteTemplate(htmlReportFile, "report", &reporter) + + if err != nil { + log.Fatal(err) + } + + fmt.Printf("HTML report was created: %s", htmlReportFilenameAbsPath) +} + +//unused by HTML reporter +func (reporter *HTMLReporter) SpecSuiteWillBegin(config config.GinkgoConfigType, summary *types.SuiteSummary) { +} + +func (reporter *HTMLReporter) BeforeSuiteDidRun(setupSummary *types.SetupSummary) { +} + +func (reporter *HTMLReporter) SpecWillRun(specSummary *types.SpecSummary) { +} + +func (reporter *HTMLReporter) AfterSuiteDidRun(setupSummary *types.SetupSummary) { +} diff --git a/conformance/setup.go b/conformance/setup.go new file mode 100644 index 00000000..8a75bcd1 --- /dev/null +++ b/conformance/setup.go @@ -0,0 +1,93 @@ +package conformance + +import ( + "fmt" + "os" + "strconv" + + "github.com/bloodorangeio/reggie" + godigest "github.com/opencontainers/go-digest" +) + +// TODO: import from opencontainers/distribution-spec +type ( + TagList struct { + Name string `json:"name"` + Tags []string `json:"tags"` + } +) + +const ( + nonexistentManifest string = ".INVALID_MANIFEST_NAME" +) + +var ( + blobA []byte + blobADigest string + blobB []byte + blobBDigest string + blobBChunk1 []byte + blobBChunk2 []byte + blobBChunk1Range string + blobBChunk2Range string + client *reggie.Client + configContent []byte + configContentLength string + configDigest string + lastResponse *reggie.Response + lastTagList TagList + manifestContent []byte + manifestDigest string + numTags int + reportJUnitFilename string + reportHTMLFilename string + httpWriter *httpDebugWriter + suiteDescription string +) + +func init() { + hostname := os.Getenv("OCI_ROOT_URL") + namespace := os.Getenv("OCI_NAMESPACE") + username := os.Getenv("OCI_USERNAME") + password := os.Getenv("OCI_PASSWORD") + debug := os.Getenv("OCI_DEBUG") == "true" + + var err error + + httpWriter = newHTTPDebugWriter(debug) + logger := newHTTPDebugLogger(httpWriter) + client, err = reggie.NewClient(hostname, + reggie.WithDefaultName(namespace), + reggie.WithUsernamePassword(username, password), + reggie.WithDebug(true), + reggie.WithUserAgent("distribution-spec-conformance-tests")) + client.SetLogger(logger) + if err != nil { + panic(err) + } + + configContent = []byte("{}\n") + configContentLength = strconv.Itoa(len(configContent)) + configDigest = godigest.FromBytes(configContent).String() + + manifestContent = []byte(fmt.Sprintf( + "{ \"mediaType\": \"application/vnd.oci.image.manifest.v1+json\", \"config\": { \"digest\": \"%s\", "+ + "\"mediaType\": \"application/vnd.oci.image.config.v1+json\","+" \"size\": %s }, \"layers\": [], "+ + "\"schemaVersion\": 2 }", + configDigest, configContentLength)) + manifestDigest = godigest.FromBytes(manifestContent).String() + + blobA = []byte("NBA Jam on my NBA toast") + blobADigest = godigest.FromBytes(blobA).String() + + blobB = []byte("Hello, how are you today?") + blobBDigest = godigest.FromBytes(blobB).String() + blobBChunk1 = blobB[:3] + blobBChunk1Range = fmt.Sprintf("0-%d", len(blobBChunk1)-1) + blobBChunk2 = blobB[3:] + blobBChunk2Range = fmt.Sprintf("%d-%d", len(blobBChunk1), len(blobB)-1) + + reportJUnitFilename = "junit.xml" + reportHTMLFilename = "report.html" + suiteDescription = "OCI Distribution Conformance Tests" +}