feat: with specs dashboard

munge_aggregates.js

@@ -18,6 +18,51 @@ if (!dbFile || !hugoOutput) {
     process.exit(1);
 }
 
+/**
+ * @param {string} u a spec URL like "specs.ipfs.tech/http-gateways/path-gateway/#if-none-match-request-header"
+ * @returns the spec's parent, or "null" if it's a top-level spec
+ */
+const computeParent = (u) => {
+    const url = new URL(u);
+    const segments = url.pathname.split('/').filter(Boolean);
+
+    // if there's a hash, consider it as a segment
+    if (url.hash) segments.push(url.hash.substring(1));
+
+    if (segments.length <= 1) {
+        return "null";
+    }
+
+    const parent = segments.slice(0, -1).join('/');
+    return `${url.protocol}//${url.host}/${parent}`
+};
+
+/**
+ * @param {string} u a spec URL like "specs.ipfs.tech/http-gateways/path-gateway/#if-none-match-request-header"
+ * @returns the spec's name, or the hash if it's a top-level spec and whether it was found in a hash
+ */
+const computeName = (u) => {
+    const url = new URL(u);
+
+    if (url.hash) {
+        return {
+            isHashed: true,
+            name: url.hash.substring(1),
+        };
+    }
+
+    const segments = url.pathname.split('/').filter(Boolean);
+
+    if (segments.length === 0) {
+        throw new Error(`Invalid spec URL: ${u}`);
+    }
+
+    return {
+        isHashed: false,
+        name: segments[segments.length - 1],
+    };
+};
+
 const main = async () => {
     let db = new sqlite3.Database(dbFile, (err) => {
         if (err) {
@@ -58,6 +103,8 @@ const main = async () => {
     `;
     const testsRows = await all(testsQuery);
     const groups = {};
+    const flatTestGroups = {}; // used for specs generation.
+
     for (const row of testsRows) {
         const { versions, full_name, name, parent_test_full_name } = row;
         const slug = slugify(full_name);
@@ -66,10 +113,115 @@ const main = async () => {
             groups[parent_test_full_name] = {};
         }
 
-        groups[parent_test_full_name][full_name] = { versions: versions?.split(',') || [], name, full_name, slug };
+        const g = { versions: versions?.split(',') || [], name, full_name, slug };
+
+        groups[parent_test_full_name][full_name] = g;
+        flatTestGroups[full_name] = g;
     }
     outputJSON("data/testgroups.json", groups);
 
+    // Query to fetch all test specs
+    const specsQuery = `
+        SELECT
+            spec_url as full_name,
+            GROUP_CONCAT(DISTINCT test_run_version) AS versions
+        FROM TestSpecs
+        GROUP BY full_name
+        ORDER BY full_name
+    `;
+    const specsRows = await all(specsQuery);
+    const specs = {};
+    const flatSpecs = {};
+
+    for (const row of specsRows) {
+        const { versions, full_name } = row;
+        let current = full_name;
+
+        while (current !== "null") {
+            const slug = slugify(current);
+            const parent = computeParent(current);
+            const { name, isHashed } = computeName(current)
+
+            if (!specs[parent]) {
+                specs[parent] = {};
+            }
+
+            flatSpecs[current] = true
+
+            specs[parent][current] = {
+                versions: versions?.split(',') || [],
+                spec_full_name: current,
+                slug,
+                name,
+                isHashed,
+            };
+
+            current = parent;
+        }
+    }
+    outputJSON("data/specs.json", specs);
+
+    const descendTheSpecsTree = (current, path) => {
+        Object.entries(specs[current] || {})
+            .forEach(([key, spec]) => {
+                const addSpecs = (current) => {
+                    let hashes = [...(current.specs || []), spec.name];
+                    hashes = [...new Set(hashes)]; // deduplicate
+                    return { ...current, hashes }
+                };
+
+                // To reproduce the structure of URLs and hashes, we update existing specs pages
+                if (spec.isHashed) {
+                    const p = path.join("/");
+                    outputFrontmatter(
+                        `content/specs/${p}/_index.md`,
+                        addSpecs
+                    );
+                    // We assume there are no recursion / children for hashes
+                    return
+                }
+
+                const newPath = [...path, spec.name];
+                const p = newPath.join("/");
+
+                outputFrontmatter(`content/specs/${p}/_index.md`, {
+                    ...spec,
+                    title: spec.name,
+                });
+
+                descendTheSpecsTree(key, newPath);
+            })
+    }
+
+    descendTheSpecsTree("null", [])
+
+    // Aggregate test results per specs
+    const specsTestGroups = {};
+
+    for (const fullName of Object.keys(flatSpecs)) {
+        // list all the test names for a given spec.
+        // we prefix search the database for spec_urls starting with the spec name
+        const specsQuery = `
+            SELECT
+                test_full_name
+            FROM TestSpecs
+            WHERE spec_url LIKE ?
+            ORDER BY test_full_name
+        `;
+        const tests = await all(specsQuery, [fullName + '%']);
+
+        const s = tests.map(x => x.test_full_name)
+            .reduce((acc, name) => {
+                return {
+                    ...acc,
+                    [name]: flatTestGroups[name]
+                }
+            }, {});
+        specsTestGroups[fullName] = s;
+    }
+
+    outputJSON("data/specsgroups.json", specsTestGroups);
+
     // Query to fetch all stdouts
     const logsQuery = `
         SELECT
@@ -269,6 +421,7 @@ const slugify = (str) => {
         .replace(/_+/g, '_') // remove consecutive underscores
         .replace(/[\/]/g, "__")
         .replace(/[^a-z0-9 -]/g, '-') // remove non-alphanumeric characters
+        .replace(/-+/g, '-') // remove consecutive dashes
 }
 
 const outputJSON = (p, data) => {
@@ -283,7 +436,7 @@ const outputJSON = (p, data) => {
     fs.writeFileSync(fullPath, json);
 }
 
-const outputFrontmatter = (p, data) => {
+const outputFrontmatter = (p, dataOrUpdate) => {
     const fullPath = `${hugoOutput}/${p}`;
 
     // TODO: implement update frontmatter
@@ -303,7 +456,12 @@ const outputFrontmatter = (p, data) => {
         content.content = existing.content;
         content.data = existing.data;
     }
-    content.data = { ...content.data, ...data };
+
+    if (typeof dataOrUpdate === "function") {
+        content.data = dataOrUpdate(content.data);
+    } else {
+        content.data = { ...content.data, ...dataOrUpdate };
+    }
 
     const md = matter.stringify(content.content, content.data);
     fs.writeFileSync(fullPath, md);
@@ -322,4 +480,4 @@ main()
     .catch((e) => {
         console.error(e);
         process.exit(1);
-    })
+    })

munge_sql.js

@@ -100,6 +100,28 @@ const main = async () => {
         );
     `)
 
+    // Create the SPECS
+    await run(`
+    CREATE TABLE IF NOT EXISTS TestSpecs (
+        test_run_implementation_id TEXT,
+        test_run_version TEXT,
+        test_full_name TEXT,
+
+        spec_url TEXT,
+
+        PRIMARY KEY (test_run_implementation_id, test_run_version, test_full_name, spec_url),
+
+        -- test run
+        FOREIGN KEY (test_run_implementation_id, test_run_version)
+            REFERENCES TestRun (implementation_id, version),
+
+        -- test result
+        FOREIGN KEY (test_run_implementation_id, test_run_version, test_full_name)
+            REFERENCES TestResult (test_run_implementation_id, test_run_version, full_name)
+    );
+    `);
+
+
     for (const file of files) {
         const fileName = file.split("/").slice(-1)[0].split(".")[0];
         const implemId = fileName;
@@ -146,6 +168,16 @@ const main = async () => {
                 VALUES (?, ?, ?, ?)
             `, [implemId, version, fullName, test.output]);
 
+            const specsArray = test.meta?.specs || [];
+            for (const specUrl of specsArray) {
+                // add `https://` if the specs don't have it
+                const cleanSpecUrl = specUrl.startsWith("http") ? specUrl : `https://${specUrl}`;
+
+                await run(`
+                    INSERT INTO TestSpecs (test_run_implementation_id, test_run_version, test_full_name, spec_url)
+                    VALUES (?, ?, ?, ?)
+                `, [implemId, version, fullName, cleanSpecUrl]);
+            }
         }
     }
 

tests/path_gateway_dag_test.go

@@ -42,7 +42,7 @@ func TestGatewayJsonCbor(t *testing.T) {
 		},
 		{
 			Name: "GET UnixFS file with JSON bytes is returned with application/json Content-Type - with headers",
-			Spec: "specs.ipfs.tech/http-gateways/path-gateway/#accept-request-header",
+			Spec: "https://specs.ipfs.tech/http-gateways/path-gateway/#accept-request-header",
 			Hint: `
 			## Quick regression check for JSON stored on UnixFS:
 			## it has nothing to do with DAG-JSON and JSON codecs,
@@ -479,7 +479,7 @@ func TestNativeDag(t *testing.T) {
 				Response: Expect().
 					Headers(
 						Header("Content-Type").Hint("expected Content-Type").Equals("application/vnd.ipld.dag-{{format}}", row.Format),
-						Header("Content-Length").Spec("specs.ipfs.tech/http-gateways/path-gateway/#content-disposition-response-header").Hint("includes Content-Length").Equals("{{length}}", len(dagTraversal.RawData())),
+						Header("Content-Length").Spec("https://specs.ipfs.tech/http-gateways/path-gateway/#content-disposition-response-header").Hint("includes Content-Length").Equals("{{length}}", len(dagTraversal.RawData())),
 						Header("Content-Disposition").Hint("includes Content-Disposition").Contains(`{{disposition}}; filename="{{cid}}.{{format}}"`, row.Disposition, dagTraversalCID, row.Format),
 						Header("X-Content-Type-Options").Hint("includes nosniff hint").Contains("nosniff"),
 					),
@@ -553,7 +553,7 @@ func TestNativeDag(t *testing.T) {
 			},
 			{
 				Name: Fmt("HEAD {{name}} with only-if-cached for missing block returns HTTP 412 Precondition Failed", row.Name),
-				Spec: "specs.ipfs.tech/http-gateways/path-gateway/#only-if-cached",
+				Spec: "https://specs.ipfs.tech/http-gateways/path-gateway/#only-if-cached",
 				Request: Request().
 					Path("/ipfs/{{cid}}", missingCID).
 					Header("Cache-Control", "only-if-cached").

tests/path_gateway_tar_test.go

@@ -76,7 +76,7 @@ func TestTar(t *testing.T) {
 		},
 		{
 			Name: "GET TAR with explicit ?filename= succeeds with modified Content-Disposition header",
-			Spec: "specs.ipfs.tech/http-gateways/path-gateway/#content-disposition-response-header",
+			Spec: "https://specs.ipfs.tech/http-gateways/path-gateway/#content-disposition-response-header",
 			Request: Request().
 				Path("/ipfs/{{cid}}", dirCID).
 				Query("filename", "testтест.tar").

tests/path_gateway_unixfs_test.go

@@ -202,7 +202,7 @@ func TestGatewayCache(t *testing.T) {
 		// ==========
 		{
 			Name: "GET for /ipfs/ file with matching Etag in If-None-Match returns 304 Not Modified",
-			Spec: "specs.ipfs.tech/http-gateways/path-gateway/#if-none-match-request-header",
+			Spec: "https://specs.ipfs.tech/http-gateways/path-gateway/#if-none-match-request-header",
 			Request: Request().
 				Path("/ipfs/{{cid}}/root2/root3/root4/index.html", fixture.MustGetCid()).
 				Headers(
@@ -213,7 +213,7 @@ func TestGatewayCache(t *testing.T) {
 		},
 		{
 			Name: "GET for /ipfs/ dir with index.html file with matching Etag in If-None-Match returns 304 Not Modified",
-			Spec: "specs.ipfs.tech/http-gateways/path-gateway/#if-none-match-request-header",
+			Spec: "https://specs.ipfs.tech/http-gateways/path-gateway/#if-none-match-request-header",
 			Request: Request().
 				Path("/ipfs/{{cid}}/root2/root3/root4/", fixture.MustGetCid()).
 				Headers(
@@ -224,7 +224,7 @@ func TestGatewayCache(t *testing.T) {
 		},
 		{
 			Name: "GET for /ipfs/ file with matching third Etag in If-None-Match returns 304 Not Modified",
-			Spec: "specs.ipfs.tech/http-gateways/path-gateway/#if-none-match-request-header",
+			Spec: "https://specs.ipfs.tech/http-gateways/path-gateway/#if-none-match-request-header",
 			Request: Request().
 				Path("/ipfs/{{cid}}/root2/root3/root4/index.html", fixture.MustGetCid()).
 				Headers(
@@ -235,7 +235,7 @@ func TestGatewayCache(t *testing.T) {
 		},
 		{
 			Name: "GET for /ipfs/ file with matching weak Etag in If-None-Match returns 304 Not Modified",
-			Spec: "specs.ipfs.tech/http-gateways/path-gateway/#if-none-match-request-header",
+			Spec: "https://specs.ipfs.tech/http-gateways/path-gateway/#if-none-match-request-header",
 			Request: Request().
 				Path("/ipfs/{{cid}}/root2/root3/root4/index.html", fixture.MustGetCid()).
 				Headers(
@@ -246,7 +246,7 @@ func TestGatewayCache(t *testing.T) {
 		},
 		{
 			Name: "GET for /ipfs/ file with wildcard Etag in If-None-Match returns 304 Not Modified",
-			Spec: "specs.ipfs.tech/http-gateways/path-gateway/#if-none-match-request-header",
+			Spec: "https://specs.ipfs.tech/http-gateways/path-gateway/#if-none-match-request-header",
 			Request: Request().
 				Path("/ipfs/{{cid}}/root2/root3/root4/index.html", fixture.MustGetCid()).
 				Headers(
@@ -257,7 +257,7 @@ func TestGatewayCache(t *testing.T) {
 		},
 		{
 			Name: "GET for /ipfs/ dir listing with matching weak Etag in If-None-Match returns 304 Not Modified",
-			Spec: "specs.ipfs.tech/http-gateways/path-gateway/#if-none-match-request-header",
+			Spec: "https://specs.ipfs.tech/http-gateways/path-gateway/#if-none-match-request-header",
 			Request: Request().
 				Path("/ipfs/{{cid}}/root2/root3/", fixture.MustGetCid()).
 				Headers(

tests/redirects_file_test.go

@@ -15,7 +15,7 @@ import (
 )
 
 func TestRedirectsFileSupport(t *testing.T) {
-	tooling.LogSpecs(t, "specs.ipfs.tech/http-gateways/web-redirects-file/")
+	tooling.LogSpecs(t, "https://specs.ipfs.tech/http-gateways/web-redirects-file/")
 	fixture := car.MustOpenUnixfsCar("redirects_file/redirects.car")
 	redirectDir := fixture.MustGetNode("examples")
 	redirectDirCID := redirectDir.Base32Cid()
@@ -166,8 +166,8 @@ func TestRedirectsFileSupport(t *testing.T) {
 							Contains("could not parse _redirects:"),
 							Contains(`forced redirects (or "shadowing") are not supported`),
 						),
-					).Spec("specs.ipfs.tech/http-gateways/web-redirects-file/#no-forced-redirects"),
-				Spec: "specs.ipfs.tech/http-gateways/web-redirects-file/#error-handling",
+					).Spec("https://specs.ipfs.tech/http-gateways/web-redirects-file/#no-forced-redirects"),
+				Spec: "https://specs.ipfs.tech/http-gateways/web-redirects-file/#error-handling",
 			},
 			{
 				Name: "invalid file: request for $TOO_LARGE_REDIRECTS_DIR_HOSTNAME/not-found returns error about too large redirects file",
@@ -182,7 +182,7 @@ func TestRedirectsFileSupport(t *testing.T) {
 							Contains("redirects file size cannot exceed"),
 						),
 					),
-				Spec: "specs.ipfs.tech/http-gateways/web-redirects-file/#max-file-size",
+				Spec: "https://specs.ipfs.tech/http-gateways/web-redirects-file/#max-file-size",
 			},
 		}...)
 

tests/trustless_gateway_car_test.go

@@ -408,7 +408,7 @@ func TestTrustlessCarDagScopeAll(t *testing.T) {
 
 func TestTrustlessCarEntityBytes(t *testing.T) {
 	tooling.LogTestGroup(t, GroupBlockCar)
-	tooling.LogSpecs(t, "specs.ipfs.tech/http-gateways/trustless-gateway/#entity-bytes-request-query-parameter")
+	tooling.LogSpecs(t, "https://specs.ipfs.tech/http-gateways/trustless-gateway/#entity-bytes-request-query-parameter")
 
 	singleLayerHamtMultiBlockFilesFixture := car.MustOpenUnixfsCar("trustless_gateway_car/single-layer-hamt-with-multi-block-files.car")
 	subdirWithMixedBlockFiles := car.MustOpenUnixfsCar("trustless_gateway_car/subdir-with-mixed-block-files.car")

tests/trustless_gateway_raw_test.go

@@ -14,7 +14,7 @@ import (
 
 func TestTrustlessRaw(t *testing.T) {
 	tooling.LogTestGroup(t, GroupBlockCar)
-	tooling.LogSpecs(t, "specs.ipfs.tech/http-gateways/trustless-gateway/#block-responses-application-vnd-ipld-raw")
+	tooling.LogSpecs(t, "https://specs.ipfs.tech/http-gateways/trustless-gateway/#block-responses-application-vnd-ipld-raw")
 
 	fixture := car.MustOpenUnixfsCar("gateway-raw-block.car")