From a033e8be21ef179179d70bc9bc33879a569b60eb Mon Sep 17 00:00:00 2001
From: Alex Potsides <alex@achingbrain.net>
Date: Mon, 18 Mar 2019 23:24:25 +0000
Subject: [PATCH] feat: add HTTP DAG API (#1930)

---
 src/http/api/resources/dag.js   | 281 +++++++++++++++++++++
 src/http/api/resources/index.js |   1 +
 src/http/api/routes/dag.js      |  43 ++++
 src/http/api/routes/index.js    |   1 +
 test/http-api/inject/dag.js     | 416 ++++++++++++++++++++++++++++++++
 test/http-api/interface.js      |   8 +-
 6 files changed, 749 insertions(+), 1 deletion(-)
 create mode 100644 src/http/api/resources/dag.js
 create mode 100644 src/http/api/routes/dag.js
 create mode 100644 test/http-api/inject/dag.js

diff --git a/src/http/api/resources/dag.js b/src/http/api/resources/dag.js
new file mode 100644
index 0000000000..325588f646
--- /dev/null
+++ b/src/http/api/resources/dag.js
@@ -0,0 +1,281 @@
+'use strict'
+
+const promisify = require('promisify-es6')
+const CID = require('cids')
+const multipart = require('ipfs-multipart')
+const mh = require('multihashes')
+const Joi = require('joi')
+const multibase = require('multibase')
+const Boom = require('boom')
+const debug = require('debug')
+const {
+  cidToString
+} = require('../../../utils/cid')
+const log = debug('ipfs:http-api:dag')
+log.error = debug('ipfs:http-api:dag:error')
+
+// common pre request handler that parses the args and returns `key` which is assigned to `request.pre.args`
+exports.parseKey = (argument = 'Argument', name = 'key', quote = "'") => {
+  return (request) => {
+    if (!request.query.arg) {
+      // for compatibility with go error messages
+      throw Boom.badRequest(`${argument} ${quote}${name}${quote} is required`)
+    }
+
+    let key = request.query.arg.trim()
+    let path
+
+    if (key.startsWith('/ipfs')) {
+      key = key.substring(5)
+    }
+
+    const parts = key.split('/')
+
+    if (parts.length > 1) {
+      key = parts.shift()
+      path = `${parts.join('/')}`
+    }
+
+    if (path && path.endsWith('/')) {
+      path = path.substring(0, path.length - 1)
+    }
+
+    try {
+      return {
+        [name]: new CID(key),
+        path
+      }
+    } catch (err) {
+      log.error(err)
+      throw Boom.badRequest("invalid 'ipfs ref' path")
+    }
+  }
+}
+
+const encodeBufferKeys = (obj, encoding) => {
+  if (!obj) {
+    return obj
+  }
+
+  if (Buffer.isBuffer(obj)) {
+    return obj.toString(encoding)
+  }
+
+  Object.keys(obj).forEach(key => {
+    if (Buffer.isBuffer(obj)) {
+      obj[key] = obj[key].toString(encoding)
+
+      return
+    }
+
+    if (typeof obj[key] === 'object') {
+      obj[key] = encodeBufferKeys(obj[key], encoding)
+    }
+  })
+
+  return obj
+}
+
+exports.get = {
+  validate: {
+    query: Joi.object().keys({
+      'data-encoding': Joi.string().valid(['text', 'base64', 'hex']).default('text'),
+      'cid-base': Joi.string().valid(multibase.names)
+    }).unknown()
+  },
+
+  // uses common parseKey method that returns a `key`
+  parseArgs: exports.parseKey(),
+
+  // main route handler which is called after the above `parseArgs`, but only if the args were valid
+  async handler (request, h) {
+    const {
+      key,
+      path
+    } = request.pre.args
+    const { ipfs } = request.server.app
+
+    let dataEncoding = request.query['data-encoding']
+
+    if (dataEncoding === 'text') {
+      dataEncoding = 'utf8'
+    }
+
+    let result
+
+    try {
+      result = await ipfs.dag.get(key, path)
+    } catch (err) {
+      throw Boom.badRequest(err)
+    }
+
+    try {
+      result.value = encodeBufferKeys(result.value, dataEncoding)
+    } catch (err) {
+      throw Boom.boomify(err)
+    }
+
+    return h.response(result.value)
+  }
+}
+
+exports.put = {
+  validate: {
+    query: Joi.object().keys({
+      format: Joi.string().default('cbor'),
+      'input-enc': Joi.string().default('json'),
+      pin: Joi.boolean(),
+      hash: Joi.string().valid(Object.keys(mh.names)).default('sha2-256'),
+      'cid-base': Joi.string().valid(multibase.names).default('base58btc')
+    }).unknown()
+  },
+
+  // pre request handler that parses the args and returns `node`
+  // which is assigned to `request.pre.args`
+  async parseArgs (request, h) {
+    if (!request.payload) {
+      throw Boom.badRequest("File argument 'object data' is required")
+    }
+
+    const enc = request.query['input-enc']
+
+    if (!request.headers['content-type']) {
+      throw Boom.badRequest("File argument 'object data' is required")
+    }
+
+    const fileStream = await new Promise((resolve, reject) => {
+      multipart.reqParser(request.payload)
+        .on('file', (name, stream) => resolve(stream))
+        .on('end', () => reject(Boom.badRequest("File argument 'object data' is required")))
+    })
+
+    let data = await new Promise((resolve, reject) => {
+      fileStream
+        .on('data', data => resolve(data))
+        .on('end', () => reject(Boom.badRequest("File argument 'object data' is required")))
+    })
+
+    let format = request.query.format
+
+    if (format === 'cbor') {
+      format = 'dag-cbor'
+    }
+
+    let node
+
+    if (format === 'raw') {
+      node = data
+    } else if (enc === 'json') {
+      try {
+        node = JSON.parse(data.toString())
+      } catch (err) {
+        throw Boom.badRequest('Failed to parse the JSON: ' + err)
+      }
+    } else {
+      const { ipfs } = request.server.app
+      const codec = ipfs._ipld.resolvers[format]
+
+      if (!codec) {
+        throw Boom.badRequest(`Missing IPLD format "${request.query.format}"`)
+      }
+
+      const deserialize = promisify(codec.util.deserialize)
+
+      node = await deserialize(data)
+    }
+
+    return {
+      node,
+      format,
+      hashAlg: request.query.hash
+    }
+  },
+
+  // main route handler which is called after the above `parseArgs`, but only if the args were valid
+  async handler (request, h) {
+    const { ipfs } = request.server.app
+    const { node, format, hashAlg } = request.pre.args
+
+    let cid
+
+    try {
+      cid = await ipfs.dag.put(node, {
+        format: format,
+        hashAlg: hashAlg
+      })
+    } catch (err) {
+      throw Boom.boomify(err, { message: 'Failed to put node' })
+    }
+
+    if (request.query.pin) {
+      await ipfs.pin.add(cid)
+    }
+
+    return h.response({
+      Cid: {
+        '/': cidToString(cid, {
+          base: request.query['cid-base']
+        })
+      }
+    })
+  }
+}
+
+exports.resolve = {
+  validate: {
+    query: Joi.object().keys({
+      'cid-base': Joi.string().valid(multibase.names)
+    }).unknown()
+  },
+
+  // uses common parseKey method that returns a `key`
+  parseArgs: exports.parseKey('argument', 'ref', '"'),
+
+  // main route handler which is called after the above `parseArgs`, but only if the args were valid
+  async handler (request, h) {
+    let { ref, path } = request.pre.args
+    const { ipfs } = request.server.app
+
+    // to be consistent with go we need to return the CID to the last node we've traversed
+    // along with the path inside that node as the remainder path
+    try {
+      let lastCid = ref
+      let lastRemainderPath = path
+
+      while (true) {
+        const block = await ipfs.block.get(lastCid)
+        const codec = ipfs._ipld.resolvers[lastCid.codec]
+
+        if (!codec) {
+          throw Boom.badRequest(`Missing IPLD format "${lastCid.codec}"`)
+        }
+
+        const resolve = promisify(codec.resolver.resolve)
+        const res = await resolve(block.data, lastRemainderPath)
+
+        if (!res.remainderPath) {
+          break
+        }
+
+        lastRemainderPath = res.remainderPath
+
+        if (!CID.isCID(res.value)) {
+          break
+        }
+
+        lastCid = res.value
+      }
+
+      return h.response({
+        Cid: {
+          '/': cidToString(lastCid, {
+            base: request.query['cid-base']
+          })
+        },
+        RemPath: lastRemainderPath || ''
+      })
+    } catch (err) {
+      throw Boom.boomify(err)
+    }
+  }
+}
diff --git a/src/http/api/resources/index.js b/src/http/api/resources/index.js
index a54aa0d4f3..3f32cccdf5 100644
--- a/src/http/api/resources/index.js
+++ b/src/http/api/resources/index.js
@@ -15,6 +15,7 @@ exports.bitswap = require('./bitswap')
 exports.file = require('./file')
 exports.filesRegular = require('./files-regular')
 exports.pubsub = require('./pubsub')
+exports.dag = require('./dag')
 exports.dns = require('./dns')
 exports.key = require('./key')
 exports.stats = require('./stats')
diff --git a/src/http/api/routes/dag.js b/src/http/api/routes/dag.js
new file mode 100644
index 0000000000..dd3829b60f
--- /dev/null
+++ b/src/http/api/routes/dag.js
@@ -0,0 +1,43 @@
+'use strict'
+
+const resources = require('../resources')
+
+module.exports = [
+  {
+    method: 'POST',
+    path: '/api/v0/dag/get',
+    options: {
+      pre: [
+        { method: resources.dag.get.parseArgs, assign: 'args' }
+      ],
+      validate: resources.dag.get.validate
+    },
+    handler: resources.dag.get.handler
+  },
+  {
+    method: 'POST',
+    path: '/api/v0/dag/put',
+    options: {
+      payload: {
+        parse: false,
+        output: 'stream'
+      },
+      pre: [
+        { method: resources.dag.put.parseArgs, assign: 'args' }
+      ],
+      validate: resources.dag.put.validate
+    },
+    handler: resources.dag.put.handler
+  },
+  {
+    method: 'POST',
+    path: '/api/v0/dag/resolve',
+    options: {
+      pre: [
+        { method: resources.dag.resolve.parseArgs, assign: 'args' }
+      ],
+      validate: resources.dag.resolve.validate
+    },
+    handler: resources.dag.resolve.handler
+  }
+]
diff --git a/src/http/api/routes/index.js b/src/http/api/routes/index.js
index 0f459aaf8b..48fcfff3f8 100644
--- a/src/http/api/routes/index.js
+++ b/src/http/api/routes/index.js
@@ -19,6 +19,7 @@ module.exports = [
   ...require('./pubsub'),
   require('./debug'),
   ...require('./webui'),
+  ...require('./dag'),
   require('./dns'),
   ...require('./key'),
   ...require('./stats'),
diff --git a/test/http-api/inject/dag.js b/test/http-api/inject/dag.js
new file mode 100644
index 0000000000..62eca02574
--- /dev/null
+++ b/test/http-api/inject/dag.js
@@ -0,0 +1,416 @@
+/* eslint max-nested-callbacks: ["error", 8] */
+/* eslint-env mocha */
+'use strict'
+
+const chai = require('chai')
+const dirtyChai = require('dirty-chai')
+const expect = chai.expect
+chai.use(dirtyChai)
+const promisify = require('promisify-es6')
+const DAGNode = require('ipld-dag-pb').DAGNode
+const createDAGPBNode = promisify(DAGNode.create)
+const Readable = require('stream').Readable
+const FormData = require('form-data')
+const streamToPromise = require('stream-to-promise')
+const CID = require('cids')
+
+const toHeadersAndPayload = async (thing) => {
+  const stream = new Readable()
+  stream.push(thing)
+  stream.push(null)
+
+  const form = new FormData()
+  form.append('file', stream)
+
+  return {
+    headers: form.getHeaders(),
+    payload: await streamToPromise(form)
+  }
+}
+
+module.exports = (http) => {
+  describe('dag endpoint', () => {
+    let api
+
+    before(() => {
+      api = http.api._apiServers[0]
+    })
+
+    describe('/dag/get', () => {
+      it('returns error for request without argument', async () => {
+        const res = await api.inject({
+          method: 'POST',
+          url: '/api/v0/dag/get'
+        })
+
+        expect(res.statusCode).to.equal(400)
+        expect(res.result.Message).to.include("Argument 'key' is required")
+      })
+
+      it('returns error for request with invalid argument', async () => {
+        const res = await api.inject({
+          method: 'POST',
+          url: '/api/v0/dag/get?arg=5'
+        })
+
+        expect(res.statusCode).to.equal(400)
+        expect(res.result.Message).to.include("invalid 'ipfs ref' path")
+      })
+
+      it('returns value', async () => {
+        const node = await createDAGPBNode(Buffer.from([]), [])
+        const cid = await http.api._ipfs.dag.put(node, {
+          format: 'dag-pb',
+          hashAlg: 'sha2-256'
+        })
+        const res = await api.inject({
+          method: 'POST',
+          url: `/api/v0/dag/get?arg=${cid.toBaseEncodedString()}`
+        })
+
+        expect(res.statusCode).to.equal(200)
+        expect(res.result).to.be.ok()
+        expect(res.result.links).to.be.empty()
+        expect(res.result.data).to.be.empty()
+      })
+
+      it('uses text encoding for data by default', async () => {
+        const node = await createDAGPBNode(Buffer.from([0, 1, 2, 3]), [])
+        const cid = await http.api._ipfs.dag.put(node, {
+          format: 'dag-pb',
+          hashAlg: 'sha2-256'
+        })
+
+        const res = await api.inject({
+          method: 'POST',
+          url: `/api/v0/dag/get?arg=${cid.toBaseEncodedString()}`
+        })
+
+        expect(res.statusCode).to.equal(200)
+        expect(res.result).to.be.ok()
+        expect(res.result.links).to.be.empty()
+        expect(res.result.data).to.equal('\u0000\u0001\u0002\u0003')
+      })
+
+      it('overrides data encoding', async () => {
+        const node = await createDAGPBNode(Buffer.from([0, 1, 2, 3]), [])
+        const cid = await http.api._ipfs.dag.put(node, {
+          format: 'dag-pb',
+          hashAlg: 'sha2-256'
+        })
+
+        const res = await api.inject({
+          method: 'POST',
+          url: `/api/v0/dag/get?arg=${cid.toBaseEncodedString()}&data-encoding=base64`
+        })
+
+        expect(res.statusCode).to.equal(200)
+        expect(res.result).to.be.ok()
+        expect(res.result.links).to.be.empty()
+        expect(res.result.data).to.equal('AAECAw==')
+      })
+
+      it('returns value with a path as part of the cid', async () => {
+        const cid = await http.api._ipfs.dag.put({
+          foo: 'bar'
+        }, {
+          format: 'dag-cbor',
+          hashAlg: 'sha2-256'
+        })
+
+        const res = await api.inject({
+          method: 'POST',
+          url: `/api/v0/dag/get?arg=${cid.toBaseEncodedString()}/foo`
+        })
+
+        expect(res.statusCode).to.equal(200)
+        expect(res.result).to.equal('bar')
+      })
+
+      it('returns value with a path as part of the cid for dag-pb nodes', async () => {
+        const node = await createDAGPBNode(Buffer.from([0, 1, 2, 3]), [])
+        const cid = await http.api._ipfs.dag.put(node, {
+          format: 'dag-pb',
+          hashAlg: 'sha2-256'
+        })
+
+        const res = await api.inject({
+          method: 'POST',
+          url: `/api/v0/dag/get?arg=${cid.toBaseEncodedString()}/Data&data-encoding=base64`
+        })
+
+        expect(res.statusCode).to.equal(200)
+        expect(res.result).to.equal('AAECAw==')
+      })
+
+      it('encodes buffers in arbitrary positions', async () => {
+        const cid = await http.api._ipfs.dag.put({
+          foo: 'bar',
+          baz: {
+            qux: Buffer.from([0, 1, 2, 3])
+          }
+        }, {
+          format: 'dag-cbor',
+          hashAlg: 'sha2-256'
+        })
+
+        const res = await api.inject({
+          method: 'POST',
+          url: `/api/v0/dag/get?arg=${cid.toBaseEncodedString()}&data-encoding=base64`
+        })
+
+        expect(res.statusCode).to.equal(200)
+        expect(res.result.baz.qux).to.equal('AAECAw==')
+      })
+
+      it('supports specifying buffer encoding', async () => {
+        const cid = await http.api._ipfs.dag.put({
+          foo: 'bar',
+          baz: Buffer.from([0, 1, 2, 3])
+        }, {
+          format: 'dag-cbor',
+          hashAlg: 'sha2-256'
+        })
+
+        const res = await api.inject({
+          method: 'POST',
+          url: `/api/v0/dag/get?arg=${cid.toBaseEncodedString()}&data-encoding=hex`
+        })
+
+        expect(res.statusCode).to.equal(200)
+        expect(res.result.baz).to.equal('00010203')
+      })
+    })
+
+    describe('/dag/put', () => {
+      it('returns error for request without file argument', async () => {
+        const res = await api.inject({
+          method: 'POST',
+          url: '/api/v0/dag/put'
+        })
+
+        expect(res.statusCode).to.equal(400)
+        expect(res.result.Message).to.include("File argument 'object data' is required")
+      })
+
+      it('adds a dag-cbor node by default', async () => {
+        const node = {
+          foo: 'bar'
+        }
+
+        const res = await api.inject({
+          method: 'POST',
+          url: '/api/v0/dag/put',
+          ...await toHeadersAndPayload(JSON.stringify(node))
+        })
+
+        expect(res.statusCode).to.equal(200)
+
+        const cid = new CID(res.result.Cid['/'])
+
+        expect(cid.codec).to.equal('dag-cbor')
+
+        const added = await http.api._ipfs.dag.get(cid)
+
+        expect(added.value).to.deep.equal(node)
+      })
+
+      it('adds a dag-pb node', async () => {
+        const node = {
+          data: [],
+          links: []
+        }
+
+        const res = await api.inject({
+          method: 'POST',
+          url: '/api/v0/dag/put?format=dag-pb',
+          ...await toHeadersAndPayload(JSON.stringify(node))
+        })
+
+        expect(res.statusCode).to.equal(200)
+
+        const cid = new CID(res.result.Cid['/'])
+
+        expect(cid.codec).to.equal('dag-pb')
+
+        const added = await http.api._ipfs.dag.get(cid)
+
+        expect(added.value.data).to.be.empty()
+        expect(added.value.links).to.be.empty()
+      })
+
+      it('adds a raw node', async () => {
+        const node = Buffer.from([0, 1, 2, 3])
+
+        const res = await api.inject({
+          method: 'POST',
+          url: '/api/v0/dag/put?format=raw',
+          ...await toHeadersAndPayload(node)
+        })
+
+        expect(res.statusCode).to.equal(200)
+
+        const cid = new CID(res.result.Cid['/'])
+
+        expect(cid.codec).to.equal('raw')
+
+        const added = await http.api._ipfs.dag.get(cid)
+
+        expect(added.value).to.deep.equal(node)
+      })
+
+      it('pins a node after adding', async () => {
+        const node = {
+          foo: 'bar',
+          disambiguator: Math.random()
+        }
+
+        const res = await api.inject({
+          method: 'POST',
+          url: '/api/v0/dag/put?pin=true',
+          ...await toHeadersAndPayload(JSON.stringify(node))
+        })
+
+        expect(res.statusCode).to.equal(200)
+
+        const cid = new CID(res.result.Cid['/'])
+        const pinset = await http.api._ipfs.pin.ls()
+
+        expect(pinset.map(pin => pin.hash)).to.contain(cid.toBaseEncodedString('base58btc'))
+      })
+
+      it('does not pin a node after adding', async () => {
+        const node = {
+          foo: 'bar',
+          disambiguator: Math.random()
+        }
+
+        const res = await api.inject({
+          method: 'POST',
+          url: '/api/v0/dag/put?pin=false',
+          ...await toHeadersAndPayload(JSON.stringify(node))
+        })
+
+        expect(res.statusCode).to.equal(200)
+
+        const cid = new CID(res.result.Cid['/'])
+        const pinset = await http.api._ipfs.pin.ls()
+
+        expect(pinset.map(pin => pin.hash)).to.not.contain(cid.toBaseEncodedString('base58btc'))
+      })
+    })
+
+    describe('/dag/resolve', () => {
+      it('returns error for request without argument', async () => {
+        const res = await api.inject({
+          method: 'POST',
+          url: '/api/v0/dag/resolve'
+        })
+
+        expect(res.statusCode).to.equal(400)
+        expect(res.result.Message).to.include('argument "ref" is required')
+      })
+
+      it('resolves a node', async () => {
+        const node = {
+          foo: 'bar'
+        }
+        const cid = await http.api._ipfs.dag.put(node, {
+          format: 'dag-cbor',
+          hashAlg: 'sha2-256'
+        })
+
+        const res = await api.inject({
+          method: 'POST',
+          url: `/api/v0/dag/resolve?arg=${cid.toBaseEncodedString()}`
+        })
+
+        expect(res.statusCode).to.equal(200)
+        expect(res.result).to.be.ok()
+
+        const returnedCid = new CID(res.result.Cid['/'])
+        const returnedRemainerPath = res.result.RemPath
+
+        expect(returnedCid).to.deep.equal(cid)
+        expect(returnedRemainerPath).to.be.empty()
+      })
+
+      it('returns the remainder path from within the resolved node', async () => {
+        const node = {
+          foo: 'bar'
+        }
+        const cid = await http.api._ipfs.dag.put(node, {
+          format: 'dag-cbor',
+          hashAlg: 'sha2-256'
+        })
+
+        const res = await api.inject({
+          method: 'POST',
+          url: `/api/v0/dag/resolve?arg=${cid.toBaseEncodedString()}/foo`
+        })
+
+        expect(res.statusCode).to.equal(200)
+        expect(res.result).to.be.ok()
+
+        const returnedCid = new CID(res.result.Cid['/'])
+        const returnedRemainerPath = res.result.RemPath
+
+        expect(returnedCid).to.deep.equal(cid)
+        expect(returnedRemainerPath).to.equal('foo')
+      })
+
+      it('returns an error when the path is not available', async () => {
+        const node = {
+          foo: 'bar'
+        }
+        const cid = await http.api._ipfs.dag.put(node, {
+          format: 'dag-cbor',
+          hashAlg: 'sha2-256'
+        })
+
+        const res = await api.inject({
+          method: 'POST',
+          url: `/api/v0/dag/resolve?arg=${cid.toBaseEncodedString()}/bar`
+        })
+
+        expect(res.statusCode).to.equal(500)
+        expect(res.result).to.be.ok()
+      })
+
+      it('resolves across multiple nodes, returning the CID of the last node traversed', async () => {
+        const node2 = {
+          bar: 'baz'
+        }
+        const cid2 = await http.api._ipfs.dag.put(node2, {
+          format: 'dag-cbor',
+          hashAlg: 'sha2-256'
+        })
+
+        const node1 = {
+          foo: {
+            '/': cid2
+          }
+        }
+
+        const cid1 = await http.api._ipfs.dag.put(node1, {
+          format: 'dag-cbor',
+          hashAlg: 'sha2-256'
+        })
+
+        const res = await api.inject({
+          method: 'POST',
+          url: `/api/v0/dag/resolve?arg=${cid1.toBaseEncodedString()}/foo/bar`
+        })
+
+        expect(res.statusCode).to.equal(200)
+        expect(res.result).to.be.ok()
+
+        const returnedCid = new CID(res.result.Cid['/'])
+        const returnedRemainerPath = res.result.RemPath
+
+        expect(returnedCid).to.deep.equal(cid2)
+        expect(returnedRemainerPath).to.equal('bar')
+      })
+    })
+  })
+}
diff --git a/test/http-api/interface.js b/test/http-api/interface.js
index 9529fdece2..3ff76c65c8 100644
--- a/test/http-api/interface.js
+++ b/test/http-api/interface.js
@@ -18,7 +18,13 @@ describe('interface-ipfs-core over ipfs-http-client tests', () => {
   tests.config(defaultCommonFactory)
 
   tests.dag(defaultCommonFactory, {
-    skip: { reason: 'TODO: DAG HTTP endpoints not implemented in js-ipfs yet!' }
+    skip: [{
+      name: 'should get only a CID, due to resolving locally only',
+      reason: 'Local resolve option is not implemented yet'
+    }, {
+      name: 'tree',
+      reason: 'dag.tree is not implemented yet'
+    }]
   })
 
   tests.dht(CommonFactory.create({