Skip to content
This repository has been archived by the owner on Feb 12, 2024. It is now read-only.

Commit

Permalink
feat: Add --cid-version option to ipfs files add + decodeURIComponent…
Browse files Browse the repository at this point in the history
… for file and directory names
  • Loading branch information
daviddias authored Sep 2, 2017
1 parent b3ad40f commit 7544b7b
Show file tree
Hide file tree
Showing 7 changed files with 138 additions and 17 deletions.
12 changes: 6 additions & 6 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -73,8 +73,8 @@
"expose-loader": "^0.7.3",
"form-data": "^2.3.1",
"gulp": "^3.9.1",
"interface-ipfs-core": "~0.31.13",
"ipfsd-ctl": "~0.21.0",
"interface-ipfs-core": "~0.31.16",
"ipfsd-ctl": "~0.22.0",
"left-pad": "^1.1.3",
"lodash": "^4.17.4",
"mocha": "^3.5.0",
Expand All @@ -99,15 +99,15 @@
"hapi": "^16.5.2",
"hapi-set-header": "^1.0.2",
"hoek": "^4.2.0",
"ipfs-api": "^14.1.2",
"ipfs-api": "^14.2.1",
"ipfs-bitswap": "~0.16.1",
"ipfs-block": "~0.6.0",
"ipfs-block-service": "~0.12.0",
"ipfs-multipart": "~0.1.0",
"ipfs-repo": "~0.17.0",
"ipfs-unixfs": "~0.1.12",
"ipfs-unixfs-engine": "~0.22.0",
"ipld-resolver": "~0.13.0",
"ipld-resolver": "~0.13.1",
"is-ipfs": "^0.3.0",
"is-stream": "^1.1.0",
"joi": "^10.6.0",
Expand All @@ -128,7 +128,7 @@
"mafmt": "^2.1.8",
"mkdirp": "~0.5.1",
"multiaddr": "^2.3.0",
"multihashes": "~0.4.8",
"multihashes": "~0.4.9",
"once": "^1.4.0",
"path-exists": "^3.0.0",
"peer-book": "~0.5.0",
Expand Down Expand Up @@ -208,4 +208,4 @@
"tcme <hi@this-connect.me>",
"ᴠɪᴄᴛᴏʀ ʙᴊᴇʟᴋʜᴏʟᴍ <victorbjelkholm@gmail.com>"
]
}
}
29 changes: 28 additions & 1 deletion src/cli/commands/files/add.js
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,15 @@ module.exports = {
'shard-split-threshold': {
type: 'integer',
default: 1000
},
'raw-leaves': {
type: 'boolean',
default: undefined,
describe: 'Use raw blocks for leaf nodes. (experimental)'
},
'cid-version': {
type: 'integer',
describe: 'Cid version. Non-zero value will change default of \'raw-leaves\' to true. (experimental)'
}
},

Expand All @@ -120,7 +129,25 @@ module.exports = {
const index = inPath.lastIndexOf('/') + 1
const options = {
strategy: argv.trickle ? 'trickle' : 'balanced',
shardSplitThreshold: argv.enableShardingExperiment ? argv.shardSplitThreshold : Infinity
shardSplitThreshold: argv.enableShardingExperiment ? argv.shardSplitThreshold : Infinity,
'cid-version': argv['cid-version'],
'raw-leaves': argv['raw-leaves']
}

// Temporary restriction on raw-leaves:
// When cid-version=1 then raw-leaves MUST be present and false.
//
// This is because raw-leaves is not yet implemented in js-ipfs,
// and go-ipfs changes the value of raw-leaves to true when
// cid-version > 0 unless explicitly set to false.
//
// This retains feature parity without having to implement raw-leaves.
if (argv['cid-version'] > 0 && argv['raw-leaves'] !== false) {
throw new Error('Implied argument raw-leaves must be passed and set to false when cid-version is > 0')
}

if (argv['raw-leaves']) {
throw new Error('Not implemented: raw-leaves')
}

if (argv.enableShardingExperiment && utils.isDaemonOn()) {
Expand Down
22 changes: 15 additions & 7 deletions src/core/components/files.js
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ const unixfsEngine = require('ipfs-unixfs-engine')
const importer = unixfsEngine.importer
const exporter = unixfsEngine.exporter
const promisify = require('promisify-es6')
const multihashes = require('multihashes')
const pull = require('pull-stream')
const sort = require('pull-sort')
const pushable = require('pull-pushable')
Expand All @@ -13,6 +12,7 @@ const toPull = require('stream-to-pull-stream')
const waterfall = require('async/waterfall')
const isStream = require('is-stream')
const Duplex = require('stream').Duplex
const CID = require('cids')

module.exports = function files (self) {
const createAddPullStream = (options) => {
Expand All @@ -24,7 +24,7 @@ module.exports = function files (self) {
pull.map(normalizeContent),
pull.flatten(),
importer(self._ipldResolver, opts),
pull.asyncMap(prepareFile.bind(null, self))
pull.asyncMap(prepareFile.bind(null, self, opts))
)
}

Expand Down Expand Up @@ -68,7 +68,7 @@ module.exports = function files (self) {
pull(
pull.values(normalizeContent(data)),
importer(self._ipldResolver, options),
pull.asyncMap(prepareFile.bind(null, self)),
pull.asyncMap(prepareFile.bind(null, self, options)),
sort((a, b) => {
if (a.path < b.path) return 1
if (a.path > b.path) return -1
Expand Down Expand Up @@ -114,15 +114,23 @@ module.exports = function files (self) {
}
}

function prepareFile (self, file, callback) {
const bs58mh = multihashes.toB58String(file.multihash)
function prepareFile (self, opts, file, callback) {
opts = opts || {}

waterfall([
(cb) => self.object.get(file.multihash, cb),
(node, cb) => {
let cid = new CID(node.multihash)

if (opts['cid-version'] === 1) {
cid = cid.toV1()
}

const b58Hash = cid.toBaseEncodedString()

cb(null, {
path: file.path || bs58mh,
hash: bs58mh,
path: file.path || b58Hash,
hash: b58Hash,
size: node.size
})
}
Expand Down
33 changes: 32 additions & 1 deletion src/http-api/resources/files.js
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ const toPull = require('stream-to-pull-stream')
const pushable = require('pull-pushable')
const EOL = require('os').EOL
const toStream = require('pull-stream-to-stream')
const Joi = require('joi')

exports = module.exports

Expand Down Expand Up @@ -140,6 +141,28 @@ exports.get = {
}

exports.add = {
validate: {
query: Joi.object()
.keys({
'cid-version': Joi.number().integer().min(0).max(1),
// Temporary restriction on raw-leaves:
// When cid-version=1 then raw-leaves MUST be present and false.
//
// This is because raw-leaves is not yet implemented in js-ipfs,
// and go-ipfs changes the value of raw-leaves to true when
// cid-version > 0 unless explicitly set to false.
//
// This retains feature parity without having to implement raw-leaves.
'raw-leaves': Joi.any().when('cid-version', {
is: 1,
then: Joi.boolean().valid(false).required(),
otherwise: Joi.boolean().valid(false)
})
})
// TODO: Necessary until validate "recursive", "stream-channels" etc.
.options({ allowUnknown: true })
},

handler: (request, reply) => {
if (!request.payload) {
return reply({
Expand All @@ -156,6 +179,7 @@ exports.add = {
const fileAdder = pushable()

parser.on('file', (fileName, fileStream) => {
fileName = decodeURIComponent(fileName)
const filePair = {
path: fileName,
content: toPull(fileStream)
Expand All @@ -165,6 +189,8 @@ exports.add = {
})

parser.on('directory', (directory) => {
directory = decodeURIComponent(directory)

fileAdder.push({
path: directory,
content: ''
Expand All @@ -181,9 +207,14 @@ exports.add = {
fileAdder.end()
})

const options = {
'cid-version': request.query['cid-version'],
'raw-leaves': request.query['raw-leaves']
}

pull(
fileAdder,
ipfs.files.createAddPullStream(),
ipfs.files.createAddPullStream(options),
pull.map((file) => {
return {
Name: file.path ? file.path : file.hash,
Expand Down
3 changes: 2 additions & 1 deletion src/http-api/routes/files.js
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,8 @@ module.exports = (server) => {
parse: false,
output: 'stream'
},
handler: resources.files.add.handler
handler: resources.files.add.handler,
validate: resources.files.add.validate
}
})
}
54 changes: 54 additions & 0 deletions test/cli/files.js
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,60 @@ describe('files', () => runOnAndOff((thing) => {
})
})

it('add with cid-version=0', () => {
return ipfs('add src/init-files/init-docs/readme --cid-version=0').then((out) => {
expect(out)
.to.eql('added QmPZ9gcCEpqKTo6aq61g2nXGUhM4iCL3ewB6LDXZCtioEB readme\n')
})
})

// Temporarily expect to fail as raw-leaves not yet implemented.
//
// When cid-version=1 then raw-leaves MUST be present and false.
//
// This is because raw-leaves is not yet implemented in js-ipfs,
// and go-ipfs changes the value of raw-leaves to true when
// cid-version > 0 unless explicitly set to false.
//
// This retains feature parity without having to implement raw-leaves.
it('add with cid-version=1', () => {
return new Promise((resolve, reject) => {
ipfs('add src/init-files/init-docs/readme --cid-version=1')
.then(() => reject(new Error('Raw leaves not expected to be implemented')))
.catch((err) => {
expect(err).to.exist()
resolve()
})
})
})

it('add with cid-version=1 and raw-leaves=false', () => {
return ipfs('add src/init-files/init-docs/readme --cid-version=1 --raw-leaves=false').then((out) => {
expect(out)
.to.eql('added zdj7WWeQ43G6JJvLWQWZpyHuAMq6uYWRjkBXFad11vE2LHhQ7 readme\n')
})
})

// Temporarily expect to fail as raw-leaves not yet implemented
//
// When cid-version=1 then raw-leaves MUST be present and false.
//
// This is because raw-leaves is not yet implemented in js-ipfs,
// and go-ipfs changes the value of raw-leaves to true when
// cid-version > 0 unless explicitly set to false.
//
// This retains feature parity without having to implement raw-leaves.
it('add with cid-version=1 and raw-leaves=true', () => {
return new Promise((resolve, reject) => {
ipfs('add src/init-files/init-docs/readme --cid-version=1 --raw-leaves=true')
.then(() => reject(new Error('Raw leaves not expected to be implemented')))
.catch((err) => {
expect(err).to.exist()
resolve()
})
})
})

it('cat', () => {
return ipfs('files cat QmPZ9gcCEpqKTo6aq61g2nXGUhM4iCL3ewB6LDXZCtioEB')
.then((out) => {
Expand Down
2 changes: 1 addition & 1 deletion test/http-api/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ describe('HTTP API', () => {
.forEach((file) => require('./spec/' + file)(http))
})

describe('## interface tests', () => {
describe.only('## interface tests', () => {
fs.readdirSync(path.join(__dirname, '/interface'))
.forEach((file) => require('./interface/' + file))
})
Expand Down

0 comments on commit 7544b7b

Please sign in to comment.