-
Notifications
You must be signed in to change notification settings - Fork 4
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
adds incremental verification to CAR files.
- Loading branch information
Showing
12 changed files
with
1,011 additions
and
21 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1 +1 @@ | ||
node_modules/ | ||
node_modules/ |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,4 +1,4 @@ | ||
node_modules/ | ||
test/ | ||
.*/ | ||
*.config.*js | ||
*.config.*js |
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,51 @@ | ||
import { CarBlockIterator } from '@ipld/car/iterator' | ||
import toIterable from 'browser-readablestream-to-it' | ||
|
||
import { verifyBlock } from './car.js' | ||
import { promiseTimeout } from './timers.js' | ||
import { TimeoutError, VerificationError } from './errors.js' | ||
|
||
// Assumptions | ||
// * client and server are both using DFS traversal. | ||
// * Server is sending CARs with duplicate blocks. | ||
export class CarBlockGetter { | ||
constructor (carItr, opts = {}) { | ||
this.carItr = carItr | ||
this.getBlockTimeout = opts.getBlockTimeout ?? 1_000 * 10 | ||
} | ||
|
||
static async fromStream (carStream) { | ||
const iterable = await CarBlockIterator.fromIterable( | ||
asAsyncIterable(carStream) | ||
) | ||
const carItr = iterable[Symbol.asyncIterator]() | ||
return new CarBlockGetter(carItr) | ||
} | ||
|
||
async get (cid, options) { | ||
const { value, done } = await promiseTimeout( | ||
this.carItr.next(), | ||
this.getBlockTimeout, | ||
new TimeoutError(`get block ${cid} timed out`) | ||
) | ||
|
||
if (!value && done) { | ||
throw new VerificationError('CAR file has no more blocks.') | ||
} | ||
|
||
const { cid: blockCid, bytes } = value | ||
await verifyBlock(blockCid, bytes) | ||
|
||
if (!cid.equals(blockCid)) { | ||
throw new VerificationError( | ||
`received block with cid ${blockCid}, expected ${cid}` | ||
) | ||
} | ||
|
||
return bytes | ||
} | ||
} | ||
|
||
function asAsyncIterable (readable) { | ||
return Symbol.asyncIterator in readable ? readable : toIterable(readable) | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,13 @@ | ||
export class VerificationError extends Error { | ||
constructor (message) { | ||
super(message) | ||
this.name = 'VerificationError' | ||
} | ||
} | ||
|
||
export class TimeoutError extends Error { | ||
constructor (message) { | ||
super(message) | ||
this.name = 'TimeoutError' | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,119 @@ | ||
import assert from 'node:assert/strict' | ||
import { createHash } from 'node:crypto' | ||
import fs from 'node:fs' | ||
import { describe, it } from 'node:test' | ||
|
||
import { CarReader, CarWriter } from '@ipld/car' | ||
import { CID } from 'multiformats/cid' | ||
|
||
import { extractVerifiedContent } from '#src/utils/car.js' | ||
|
||
async function getFileHash (itr) { | ||
const hasher = createHash('sha256') | ||
for await (const chunk of itr) { | ||
hasher.update(chunk) | ||
} | ||
return hasher.digest('hex') | ||
} | ||
|
||
describe('CAR Verification', () => { | ||
it('should extract content from a valid CAR', async () => { | ||
const cidPath = | ||
'bafybeiaxzcil63nfqz5z3c34iajzgve5buqxrc7xcoslxlq2ndg5kfvc2u' | ||
const filepath = | ||
'./fixtures/bafybeiaxzcil63nfqz5z3c34iajzgve5buqxrc7xcoslxlq2ndg5kfvc2u.car' | ||
const carStream = fs.createReadStream(filepath) | ||
|
||
const contentItr = await extractVerifiedContent(cidPath, carStream) | ||
const actualHash = await getFileHash(contentItr) | ||
const expectedHash = | ||
'983408ac1d97f5913b6a3abee741d3c0032bc465ae0802023d61ce38136e292b' | ||
|
||
assert.strictEqual(actualHash, expectedHash) | ||
}) | ||
|
||
it('should verify file paths', async () => { | ||
const cidPath = | ||
'bafybeihin6eiifex5e76ama6do2dhmt2da3g5dsrazm5eskwm6raczbjtu/2160.png' | ||
const filepath = | ||
'./fixtures/bafybeihin6eiifex5e76ama6do2dhmt2da3g5dsrazm5eskwm6raczbjtu_2160_png.car' | ||
const carStream = fs.createReadStream(filepath) | ||
|
||
const contentItr = await extractVerifiedContent(cidPath, carStream) | ||
const actualHash = await getFileHash(contentItr) | ||
const expectedHash = | ||
'64e4e4abef8095853c29cabcebb0240b986d4ff1e1498639f59ad1ef1ff2a3a9' | ||
|
||
assert.strictEqual(actualHash, expectedHash) | ||
}) | ||
|
||
it('should error if CAR is missing blocks', async () => { | ||
const cidPath = | ||
'bafybeiaxzcil63nfqz5z3c34iajzgve5buqxrc7xcoslxlq2ndg5kfvc2u' | ||
const filepath = | ||
'./fixtures/bafybeiaxzcil63nfqz5z3c34iajzgve5buqxrc7xcoslxlq2ndg5kfvc2u.car' | ||
const carStream = fs.createReadStream(filepath) | ||
|
||
// Create an invalid CAR that only has 1 block instead of 4 | ||
const outCid = CID.parse(cidPath) | ||
const { writer, out } = await CarWriter.create([outCid]); | ||
(async () => { | ||
// need wrapping IIFE to avoid node exiting early | ||
const reader = await CarReader.fromIterable(carStream) | ||
await writer.put(await reader.get(cidPath)) | ||
await writer.close() | ||
})() | ||
|
||
await assert.rejects( | ||
async () => { | ||
for await (const _ of extractVerifiedContent(cidPath, out)) {} | ||
}, | ||
{ | ||
name: 'VerificationError', | ||
message: 'CAR file has no more blocks.' | ||
} | ||
) | ||
}) | ||
|
||
it('should error if CAR blocks are in the wrong traversal order', async () => { | ||
const cidPath = | ||
'bafybeiaxzcil63nfqz5z3c34iajzgve5buqxrc7xcoslxlq2ndg5kfvc2u' | ||
const filepath = | ||
'./fixtures/bafybeiaxzcil63nfqz5z3c34iajzgve5buqxrc7xcoslxlq2ndg5kfvc2u.car' | ||
const carStream = fs.createReadStream(filepath) | ||
|
||
// Create an invalid CAR that has blocks in the wrong order | ||
const outCid = CID.parse(cidPath) | ||
const { writer, out } = await CarWriter.create([outCid]); | ||
(async () => { | ||
// need wrapping IIFE to avoid node exiting early | ||
const reader = await CarReader.fromIterable(carStream) | ||
|
||
const blocks = [] | ||
for await (const block of reader.blocks()) { | ||
blocks.push(block) | ||
} | ||
|
||
const temp = blocks[0] | ||
blocks[0] = blocks[1] | ||
blocks[1] = temp | ||
|
||
for (const block of blocks) { | ||
await writer.put(block) | ||
} | ||
await writer.close() | ||
})() | ||
|
||
await assert.rejects( | ||
async () => { | ||
for await (const _ of extractVerifiedContent(cidPath, out)) { | ||
} | ||
}, | ||
{ | ||
name: 'VerificationError', | ||
message: | ||
'received block with cid bafkreih64b5ydhx6tz4hpfbhfhkjrlvrin4hvv5jz2bp62fmyrkzs633si, expected bafybeiaxzcil63nfqz5z3c34iajzgve5buqxrc7xcoslxlq2ndg5kfvc2u' | ||
} | ||
) | ||
}) | ||
}) |
Binary file added
BIN
+2.05 MB
test/fixtures/bafybeiaxzcil63nfqz5z3c34iajzgve5buqxrc7xcoslxlq2ndg5kfvc2u.car
Binary file not shown.
Binary file added
BIN
+1.45 MB
test/fixtures/bafybeihin6eiifex5e76ama6do2dhmt2da3g5dsrazm5eskwm6raczbjtu_2160_png.car
Binary file not shown.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters