From d749e05650291ab434a72c628d838f0864856645 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Tue, 2 Apr 2019 15:44:03 +0100 Subject: [PATCH] feat: load files/dirs from hamt shards Use the HAMT support and IPFS overlay build in to the mfs related files commands to enable loading files from HAMT shards. --- src/index.js | 2 +- src/resolver.js | 167 +++++++++++++++--------------------------------- 2 files changed, 53 insertions(+), 116 deletions(-) diff --git a/src/index.js b/src/index.js index a36a941..64377b2 100644 --- a/src/index.js +++ b/src/index.js @@ -28,7 +28,7 @@ const response = (ipfsNode, ipfsPath) => { return new Promise((resolve, reject) => { // switch case with true feels so wrong. switch (true) { - case (errorString === 'Error: This dag node is a directory'): + case (errorString.includes('dag node is a directory')): resolver.directory(node, path, error.cid) .then((content) => { // dir render diff --git a/src/resolver.js b/src/resolver.js index f578f35..93e4f19 100644 --- a/src/resolver.js +++ b/src/resolver.js @@ -2,142 +2,79 @@ const mh = require('multihashes') const promisify = require('promisify-es6') -const reduce = require('async/reduce') const CID = require('cids') -const Unixfs = require('ipfs-unixfs') const debug = require('debug') -const log = debug('ipfs:http:response:resolver') - +const tryEach = require('async/tryEach') +const waterfall = require('async/waterfall') +const log = debug('jsipfs:http:response:resolver') +log.error = debug('jsipfs:http:response:resolver:error') const dirView = require('./dir-view') -const pathUtil = require('./utils/path') -function getIndexFiles (links) { - const INDEX_HTML_FILES = [ - 'index.html', - 'index.htm', - 'index.shtml' - ] - // directory - let indexes = links.filter((link) => INDEX_HTML_FILES.indexOf(link.Name) !== -1) - if (indexes.length) { - return indexes - } - // hamt-sharded-directory uses a 2 char prefix - return links.filter((link) => { - return link.Name.length > 2 && INDEX_HTML_FILES.indexOf(link.Name.substring(2)) !== -1 - }) +const INDEX_HTML_FILES = [ + 'index.html', + 'index.htm', + 'index.shtml' +] + +const findIndexFile = (ipfs, path, callback) => { + return tryEach(INDEX_HTML_FILES.map(file => { + return (cb) => { + waterfall([ + (cb) => ipfs.files.stat(`${path}/${file}`, cb), + (stats, cb) => cb(null, { + name: file, + cid: new CID(stats.hash) + }) + ], cb) + } + }), callback) } const directory = promisify((ipfs, path, cid, callback) => { - cid = new CID(cid) - - ipfs.object.get(cid.buffer, (err, dagNode) => { + // Test if it is a Website + findIndexFile(ipfs, path, (err, res) => { if (err) { - return callback(err) - } + if (err.message.includes('does not exist')) { + // not a website, just show a directory listing + return ipfs.dag.get(cid, (err, result) => { + if (err) { + return callback(err) + } - // Test if it is a Website - const indexFiles = getIndexFiles(dagNode.Links) + return callback(null, dirView.render(path, result.value.Links)) + }) + } - if (indexFiles.length) { - return callback(null, indexFiles) + return callback(err) } - return callback(null, dirView.render(path, dagNode.Links)) + callback(err, [{ + Name: res.name + }]) }) }) const cid = promisify((ipfs, path, callback) => { - const parts = pathUtil.cidArray(path) - let firstCid = parts.shift() - let currentCid - - // TODO: replace below with ipfs.resolve(path, {recursive: true}) - // (requires changes to js-ipfs/js-ipfs-api) - - reduce( - parts, - firstCid, - (memo, item, next) => { - try { - currentCid = new CID(memo) - } catch (err) { - return next(err) - } - - log('memo: ', memo) - log('item: ', item) - - ipfs.dag.get(currentCid, (err, result) => { - if (err) { - return next(err) - } - - const dagNode = result.value - // find multihash/cid of requested named-file in current dagNode's links - let cidOfNextFile - const nextFileName = item - - try { - for (let link of dagNode.Links) { - if (link.Name === nextFileName) { - cidOfNextFile = link.Hash - break - } - } - } catch (err) { - return next(err) - } - - if (!cidOfNextFile) { - const missingLinkErr = new Error(`no link named "${nextFileName}" under ${memo}`) - missingLinkErr.parentDagNode = memo - missingLinkErr.missingLinkName = nextFileName - return next(missingLinkErr) - } - - next(null, cidOfNextFile) - }) - }, (err, cid) => { - if (err) { - return callback(err) - } - - try { - cid = new CID(cid) - } catch (err) { - return callback(err) - } + ipfs.files.stat(path, (err, stats) => { + if (err) { + return callback(err) + } - if (cid.codec === 'raw') { - // no need for additional lookup, its raw data - callback(null, { cid }) - } + const cid = new CID(stats.hash) - ipfs.dag.get(cid, (err, dagResult) => { - if (err) { - return callback(err) - } + if (stats.type.includes('directory')) { + const err = new Error('This dag node is a directory') + err.cid = cid + err.fileName = stats.name + err.dagDirType = stats.type - try { - let dagDataObj = Unixfs.unmarshal(dagResult.value.Data) - // There are at least two types of directories: - // - "directory" - // - "hamt-sharded-directory" (example: QmT5NvUtoM5nWFfrQdVrFtvGfKFmG7AHE8P34isapyhCxX) - if (dagDataObj.type === 'directory' || dagDataObj.type === 'hamt-sharded-directory') { - let isDirErr = new Error('This dag node is a directory') - // store memo of last multihash so it can be used by directory - isDirErr.cid = isDirErr.fileName = cid - isDirErr.dagDirType = dagDataObj.type - return callback(isDirErr) - } - } catch (err) { - return callback(err) - } + return callback(err) + } - callback(null, { cid }) - }) + callback(err, { + cid }) + }) }) const multihash = promisify((ipfs, path, callback) => {