feat: add car and car generator

Makefile

@@ -1,13 +1,30 @@
+JS_DIR=js
+GO_DIR=go
+RUST_DIR=rust
+
+.PHONY: all testjs testgo testrust _build car build clean
+
 all: testjs testgo testrust
 
-js/node_modules:
-	cd js && npm install
+$(JS_DIR)/node_modules:
+	cd $(JS_DIR) && npm install
 
-testjs: js/node_modules
-	cd js && npm test
+testjs: $(JS_DIR)/node_modules
+	cd $(JS_DIR) && npm test
 
 testgo:
-	cd go && go test
+	cd $(GO_DIR) && go test
 
 testrust:
-	cd rust && cargo test -- --nocapture
+	cd $(RUST_DIR) && cargo test -- --nocapture
+
+_build:
+	cd $(JS_DIR) && npm run build
+
+car: $(JS_DIR)/node_modules
+	cd $(JS_DIR) && npm run car
+
+build: $(JS_DIR)/node_modules _build car
+
+clean:
+	rm -rf $(JS_DIR)/node_modules

README.md

@@ -8,7 +8,9 @@ This repository contains fixtures for standard IPLD codecs. It is used to verify
 
 The [fixtures](./fixtures/) directory contains a suite of test data, where each subdirectory comprises an encoded IPLD block in the formats that are supported for that data. A file containing the binary encoded form of that block has the name `<CID>.<codec-name>`, where the `CID` is the CIDv1 using a SHA2-256 multihash of the block for that codec. The `codec-name` is the standard codec name as found in the [multicodec table](https://github.com/multiformats/multicodec/blob/master/table.csv).
 
-Implementations are expected to be able to:
+Fixtures are also available in CAR format, in the [./fixtures.car](./fixtures.car) file, with every valid variation stored as a separate block. (Note that this file includes blocks that may not be readable on some systems, e.g. Kubo because of the currently [unsupported Go edge cases](./go/special_cases.go)).
+
+Codec implementations are expected to be able to:
 
 1. Read and decode the IPLD block from these files
 2. Re-encode the IPLD block using the supported codecs
@@ -18,7 +20,7 @@ Since the block is encoded in different forms, by re-encoding each decoded form
 
 ## Adding fixtures
 
-The [_fixtures_src](./_fixtures_src/) directory contains the source of each of fixtures contained in the [fixtures](./fixtures/) directory. Each file in [_fixtures_src](./_fixtures_src/) contains an encoded form of a block using one of the supported codecs. The name of the file is `<fixture-name>.<codec-name>`. The [js/make-fixtures.js](./js/make-fixtures.js) program (run with `node js/make-fixtures.js`) is used to generate the fixtures in [fixtures](./fixtures/) for each of the source files.
+The [_fixtures_src](./_fixtures_src/) directory contains the source of each of fixtures contained in the [fixtures](./fixtures/) directory. Each file in [_fixtures_src](./_fixtures_src/) contains an encoded form of a block using one of the supported codecs. The name of the file is `<fixture-name>.<codec-name>`. The [js/make-fixtures.js](./js/make-fixtures.js) program (run with `make build`) is used to generate the fixtures in [fixtures](./fixtures/) for each of the source files.
 
 Fixture generation uses the JavaScript stack for generating data, but this is not a requirement. If you would like to add fixtures and would like to create them manually, or add an alternative mechanism for generating fixtures from source then please do so.
 

go/codecs_test.go

@@ -170,8 +170,7 @@ func testNegativeFixtureEncode(codecName string, fixture negativeFixtureEncode)
 		err = encoder(node, &buf)
 		if err == nil {
 			t.Errorf("should error on encode")
-		}
-		if !strings.EqualFold(err.Error(), fixture.Error) {
+		} else if !strings.EqualFold(err.Error(), fixture.Error) {
 			t.Logf("error mismatch: [%s] ~= [%s]", err.Error(), fixture.Error)
 		}
 	}

go/fixtures.go

@@ -83,7 +83,7 @@
 		}
 		err = decoder(na, bytes.NewReader(byts))
 		if err != nil {
-			return fixtures, err
+			return fixtures, fmt.Errorf("failed to decode using %s: %w", ext, err)
 		}
 		fixtures[ext] = codecFixture{
 			codec: ext,

go/special_cases.go

@@ -4,9 +4,14 @@ type FixtureName = string
 type FixtureBlacklistReason = string
 
 var FixtureBlacklist = map[FixtureName]FixtureBlacklistReason{
-	"int--11959030306112471732":            "integer out of int64 range",
-	"int-11959030306112471731":             "integer out of int64 range",
-	"int-18446744073709551615":             "integer out of int64 range",
+	// negative integer outside of the int64 range, no support for that in Go
+	"int--11959030306112471732": "integer out of int64 range",
+	// dag-json strconv parsing error, out of int64 range
+	"int-11959030306112471731": "integer out of int64 range",
+	// dag-json strconv parsing error, out of int64 range
+	"int-18446744073709551615": "integer out of int64 range",
+	// dag-pb codec not strict on encoding unordered named link lists, should
+	// error but does not for these two:
 	"dag-pb/encode/bad sort":               "pre-sort not required",
 	"dag-pb/encode/bad sort (incl length)": "pre-sort not required",
 }

js/make-car.js

@@ -0,0 +1,22 @@
+import { createWriteStream } from 'fs'
+import { join } from 'path'
+import { pipeline } from 'stream/promises'
+import { CID } from 'multiformats'
+import { CarWriter } from '@ipld/car'
+import { fixtureDirectories, loadFixture } from './util.js'
+
+const outFile = join(process.cwd(), '..', 'fixtures.car')
+const outStream = createWriteStream(outFile)
+const { writer, out } = await CarWriter.create([])
+const pipe = pipeline(out, outStream)
+
+for (const { name, url } of fixtureDirectories()) {
+  const data = await loadFixture(url)
+  for (const { cid, bytes } of Object.values(data)) {
+    await writer.put({ cid: CID.parse(cid), bytes })
+  }
+}
+
+await writer.close()
+await pipe
+console.log(`Wrote fixtures to ${outFile}`)

js/make-fixtures.js

@@ -10,7 +10,8 @@ const fixturesDir = new URL('../fixtures/', import.meta.url)
 const fixturesSrcDir = new URL('../_fixtures_src/', import.meta.url)
 
 async function makeGarbage () {
-  for (let i = 0; i < 25;) {
+  const count = 25
+  for (let i = 0; i < count;) {
     const value = garbage(5000)
     const block = await Block.encode({ value, codec: codecs['dag-cbor'].codec, hasher: sha256 })
     if (block.bytes.length < 1000) {
@@ -19,9 +20,11 @@ async function makeGarbage () {
     await fs.writeFile(new URL(`./garbage-${i.toString().padStart(2, '0')}.dag-cbor`, fixturesSrcDir), block.bytes)
     i++
   }
+  return count
 }
 
 async function makeFixtures () {
+  let count = 0
   await Promise.all((await fs.readdir(fixturesSrcDir)).map(async (file) => {
     const furl = new URL(file, fixturesSrcDir)
     const stat = await fs.stat(furl)
@@ -61,18 +64,16 @@ async function makeFixtures () {
         throw err
       }
       await fs.writeFile(new URL(`./${block.cid.toString()}.${codec.name}`, fdir), block.bytes)
+      count++
     }
   }))
+  return count
 }
 
-if (process.argv.includes('--garbage')) {
-  makeGarbage().catch((err) => {
-    console.error(err)
-    process.exit(1)
-  })
-} else {
-  makeFixtures().catch((err) => {
-    console.error(err)
-    process.exit(1)
-  })
-}
+const p = process.argv.includes('--garbage') ? makeGarbage() : makeFixtures()
+p.then((count) => {
+  console.log(`Wrote ${count} fixtures`)
+}).catch((err) => {
+  console.error(err)
+  process.exit(1)
+})

js/package.json

@@ -7,7 +7,8 @@
   "private": true,
   "scripts": {
     "build": "node make-fixtures.js",
-    "test": "mocha test.js"
+    "test": "mocha test.js",
+    "car": "node make-car.js"
   },
   "repository": {
     "type": "git",
@@ -20,13 +21,14 @@
   },
   "homepage": "https://github.com/ipld/codec-fixtures#readme",
   "dependencies": {
-    "@ipld/dag-cbor": "^9.0.0",
-    "@ipld/dag-json": "^10.0.0",
-    "@ipld/dag-pb": "^4.0.0",
-    "chai": "^4.3.4",
+    "@ipld/car": "^5.2.4",
+    "@ipld/dag-cbor": "^9.0.6",
+    "@ipld/dag-json": "^10.1.5",
+    "@ipld/dag-pb": "^4.0.6",
+    "chai": "^4.3.10",
     "ipld-garbage": "^5.0.0",
-    "mocha": "^10.0.0",
-    "multiformats": "^12.0.0",
-    "testmark.js": "^1.0.0"
+    "mocha": "^10.2.0",
+    "multiformats": "^12.1.3",
+    "testmark.js": "^1.0.7"
   }
 }