From 1b15a29f9198eb2366c2bdff3ed47b8b1accdae0 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Tue, 19 Mar 2019 23:59:39 +0100 Subject: [PATCH] feat(brave): self-hosted API and Gateway ports This switches to patched js-ipfs which enables us to initi and start HttpApi in standalone, embedded mode with chrome.sockets (without running jsipfs daemon in CLI) Yes. I know. We are running two HTTP servers in HTTP browser :-)) Quick status: Requests to API port (5002) such as /api/v0/id work. Requests to Gateway (9090) are a mixed bag. Directory listings work, but returning byte streams does not work yet. --- add-on/src/lib/ipfs-client/embedded.js | 125 +++++++++++++++---------- add-on/src/lib/options.js | 2 +- package.json | 5 +- webpack.config.js | 25 +++-- yarn.lock | 33 ++++--- 5 files changed, 110 insertions(+), 80 deletions(-) diff --git a/add-on/src/lib/ipfs-client/embedded.js b/add-on/src/lib/ipfs-client/embedded.js index 28ab574c1..e2697d85d 100644 --- a/add-on/src/lib/ipfs-client/embedded.js +++ b/add-on/src/lib/ipfs-client/embedded.js @@ -3,20 +3,66 @@ // Required by HTTP server process.hrtime = require('browser-process-hrtime') +const uptimeStart = Date.now() +process.uptime = () => Math.floor((Date.now() - uptimeStart) / 1000) const Ipfs = require('ipfs') +const HttpApi = require('ipfs/src/http') + const { optionDefaults } = require('../options') -const http = require('http') // courtesy of chrome-net -const Hapi = require('hapi') // courtesy of js-ipfs +// js-ipfs with embedded hapi HTTP server let node = null +let nodeHttpApi = null + +// additional servers for smoke-tests let httpServer = null let hapiServer = null // Enable some debug output from js-ipfs // (borrowed from https://github.com/ipfs-shipyard/ipfs-companion/pull/557) // to include everything (mplex, libp2p, mss): localStorage.debug = '*' -localStorage.debug = 'jsipfs*,ipfs*,-*:mfs*,-*:ipns*' +localStorage.debug = 'jsipfs*,ipfs*,-*:mfs*,-*:ipns*,-ipfs:preload*' + +// Quick smoke-test to confirm require('http') works for MVP +function startRawHttpServer (port) { + const http = require('http') // courtesy of chrome-net + const httpServer = http.createServer(function (req, res) { + res.writeHead(200, { 'Content-Type': 'text/plain' }) + res.end('Hello from ipfs-companion exposing HTTP via chrome.sockets in Brave :-)\n') + }) + httpServer.listen(port, '127.0.0.1') + console.log(`[ipfs-companion] require('http') HTTP server on http://127.0.0.1:${port}`) + return httpServer +} + +function startRawHapiServer (port) { + let options = { + host: '127.0.0.1', + port, + debug: { + log: ['*'], + request: ['*'] + } + } + const initHapi = async () => { + // hapi v18 (js-ipfs >=v0.35.0-pre.0) + const Hapi = require('hapi') // courtesy of js-ipfs + const hapiServer = new Hapi.Server(options) + await hapiServer.route({ + method: 'GET', + path: '/', + handler: (request, h) => { + console.log('[ipfs-companion] hapiServer processing request', request) + return 'Hello from ipfs-companion+Hapi.js exposing HTTP via chrome.sockets in Brave :-)' + } + }) + await hapiServer.start() + console.log(`[ipfs-companion] require('hapi') HTTP server running at: ${hapiServer.info.uri}`) + } + initHapi() + return hapiServer +} exports.init = function init (opts) { // BRAVE TESTS FIRST @@ -27,62 +73,33 @@ exports.init = function init (opts) { // [x] return response // [ ] start js-ipfs with Gateway exposed by embedded Hapi server // - [x] disabling DHT in libp2p solved `TypeError: this._dht.on is not a function`, - // - [ ] Gateway does not start for some reason, nothing in logs + // - [x] API port starts and returns valid response + // - [ ] Gateway port starts, but returns invalid response for things different than QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn // ======================================= // TEST RAW require('http') SERVER if (!httpServer) { - let port = 9091 - httpServer = http.createServer(function (req, res) { - res.writeHead(200, { 'Content-Type': 'text/plain' }) - res.end('Hello from ipfs-companion exposing HTTP via chrome.sockets in Brave :-)\n') - }) - httpServer.listen(port, '127.0.0.1') - console.log(`[ipfs-companion] require('http') HTTP server on http://127.0.0.1:${port}`) + httpServer = startRawHttpServer(9091) } // ======================================= // TEST require('hapi') HTTP SERVER (same as in js-ipfs) if (!hapiServer) { - let port = 9092 - let options = { - host: '127.0.0.1', - port, - debug: { - log: ['*'], - request: ['*'] - } - } - const initHapi = async () => { - // hapi v18 (js-ipfs >=v0.35.0-pre.0) - hapiServer = new Hapi.Server(options) - await hapiServer.route({ - method: 'GET', - path: '/', - handler: (request, h) => { - console.log('[ipfs-companion] hapiServer processing request', request) - return 'Hello from ipfs-companion+Hapi.js exposing HTTP via chrome.sockets in Brave :-)' - } - }) - // await hapiServer.register({ - // }) - await hapiServer.start() - console.log(`[ipfs-companion] require('hapi') HTTP server running at: ${hapiServer.info.uri}`) - } - initHapi() + hapiServer = startRawHapiServer(9092) } // ======================================= // Resume regular startup console.log('[ipfs-companion] Embedded ipfs init') - node = new Ipfs( - JSON.parse(opts.ipfsNodeConfig || optionDefaults.ipfsNodeConfig) - ) - - if (node.isOnline()) { - return Promise.resolve(node) - } + // TODO: replace with @nodeutils/defaults-deep + const ipfsOpts = Object.assign(JSON.parse(opts.ipfsNodeConfig || optionDefaults.ipfsNodeConfig), { start: false }) + node = new Ipfs(ipfsOpts) return new Promise((resolve, reject) => { - // TODO: replace error listener after a 'ready' event. + node.on('start', async () => { + // HttpApi is off in browser context and needs to be started separately + const httpServers = new HttpApi(node, ipfsOpts) + nodeHttpApi = await httpServers.start() + return resolve(node) + }) node.once('error', (error) => { console.error('[ipfs-companion] Something went terribly wrong during startup of js-ipfs!', error) reject(error) @@ -91,14 +108,13 @@ exports.init = function init (opts) { node.on('error', error => { console.error('[ipfs-companion] Something went terribly wrong in embedded js-ipfs!', error) }) - return resolve(node) + node.start() }) }) } exports.destroy = async function () { console.log('[ipfs-companion] Embedded ipfs destroy') - if (!node) return if (httpServer) { httpServer.close() @@ -116,7 +132,16 @@ exports.destroy = async function () { } hapiServer = null } - - await node.stop() - node = null + if (nodeHttpApi) { + try { + await nodeHttpApi.stop() + } catch (err) { + console.error(`[ipfs-companion] failed to stop HttpApi`, err) + } + nodeHttpApi = null + } + if (node) { + await node.stop() + node = null + } } diff --git a/add-on/src/lib/options.js b/add-on/src/lib/options.js index e25e1464e..fdb3a4e54 100644 --- a/add-on/src/lib/options.js +++ b/add-on/src/lib/options.js @@ -9,7 +9,7 @@ exports.optionDefaults = Object.freeze({ config: { Addresses: { Swarm: [], - // API: '/ip4/127.0.0.1/tcp/5002', + API: '/ip4/127.0.0.1/tcp/5002', Gateway: '/ip4/127.0.0.1/tcp/9090' } }, diff --git a/package.json b/package.json index c673217a3..98c510d68 100644 --- a/package.json +++ b/package.json @@ -64,7 +64,8 @@ "preferGlobal": false, "resolutions": { "stream-http": "3.0.0", - "multiaddr": "6.0.3" + "multiaddr": "6.0.3", + "pino": "https://github.com/pinojs/pino/tarball/ab3dd74843eb4186fbc02b43e4b293840182fd7c/pino.tar.gz" }, "devDependencies": { "@babel/core": "7.3.4", @@ -113,7 +114,7 @@ "filesize": "4.1.2", "http-dns": "3.0.1", "http-node": "1.2.0", - "ipfs": "0.35.0-rc.0", + "ipfs": "https://github.com/ipfs/js-ipfs/tarball/cd2ccb008c1b4f8e5b8982d008546766dcdef6e2/js-ipfs.tar.gz", "ipfs-css": "0.12.0", "ipfs-http-client": "30.1.0", "ipfs-http-response": "0.2.2", diff --git a/webpack.config.js b/webpack.config.js index a659cfd98..34c2ff1df 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -22,10 +22,11 @@ const commonConfig = { // Speed Up cache: true, parallel: true, - // Default minify settings break js-ipfs: - // https://github.com/ipfs-shipyard/ipfs-companion/issues/521 - compress: { unused: false }, - mangle: true + // Custom settings to unbreak js-ipfs + compress: { + unused: false // https://github.com/ipfs-shipyard/ipfs-companion/issues/521 + }, + mangle: true // https://github.com/ipfs-shipyard/ipfs-companion/issues/521 } }) ] @@ -38,7 +39,9 @@ const commonConfig = { new webpack.DefinePlugin({ global: 'window', // https://github.com/webpack/webpack/issues/5627#issuecomment-394309966 'process.env': { - NODE_ENV: '"production"' + NODE_ENV: '"production"', + IPFS_MONITORING: Boolean(false), + DEBUG: true } }) ], @@ -52,20 +55,22 @@ const commonConfig = { ] }, resolve: { + /* mainFields: ['browser', 'main'], */ extensions: ['.js', '.json'], alias: { 'url': 'iso-url', - 'http': 'http-node', - 'dns': 'http-dns', - 'dgram': 'chrome-dgram', - 'net': 'chrome-net' + 'http': 'http-node', // chrome.sockets + 'dns': 'http-dns', // chrome.sockets + 'dgram': 'chrome-dgram', // chrome.sockets + 'net': 'chrome-net' // chrome.sockets } }, node: { global: false, // https://github.com/webpack/webpack/issues/5627#issuecomment-394309966 Buffer: true, fs: 'empty', - tls: 'empty' + tls: 'empty', + cluster: 'empty' // expected by js-ipfs dependency: node_modules/prom-client/lib/cluster.js }, watchOptions: { ignored: ['add-on/dist/**/*', 'node_modules'] diff --git a/yarn.lock b/yarn.lock index 48e9c1797..a07595ce4 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4773,9 +4773,9 @@ fast-levenshtein@~2.0.4: resolved "https://registry.yarnpkg.com/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz#3d8a5c66883a16a30ca8643e851f19baa7797917" integrity sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc= -fast-redact@^1.4.2: +fast-redact@^1.4.4: version "1.4.4" - resolved "https://registry.npmjs.org/fast-redact/-/fast-redact-1.4.4.tgz#d29bd1d0cc3ab808a9d4f9870f6e27e85c750db4" + resolved "https://registry.yarnpkg.com/fast-redact/-/fast-redact-1.4.4.tgz#d29bd1d0cc3ab808a9d4f9870f6e27e85c750db4" integrity sha512-QOQZ8sDDQPZMJ6x6zlm6hLZ2cjPDqfN3R/AYnAbM+yy8VNPvOnVXdUF/E/xbMv7g44c1krhWuzgjH2u0V5Vhsg== fast-safe-stringify@^2.0.6: @@ -6501,10 +6501,9 @@ ipfs-unixfs@~0.1.14, ipfs-unixfs@~0.1.16: dependencies: protons "^1.0.1" -ipfs@0.35.0-rc.0: +"ipfs@https://github.com/ipfs/js-ipfs/tarball/cd2ccb008c1b4f8e5b8982d008546766dcdef6e2/js-ipfs.tar.gz": version "0.35.0-rc.0" - resolved "https://registry.yarnpkg.com/ipfs/-/ipfs-0.35.0-rc.0.tgz#e4c2de16b46eafe3b44c03677340a534b7ca8264" - integrity sha512-esQoEiLWBekKt2us7hwZZegoH/Ew0B351qAGqmYd6k7TdK+p29LnC1ZKDYkSfjUvXDXi+oDdztIzDAgLY8QlmA== + resolved "https://github.com/ipfs/js-ipfs/tarball/cd2ccb008c1b4f8e5b8982d008546766dcdef6e2/js-ipfs.tar.gz#6ef53f81f857a3f32d48a0a39c764b473a9feaea" dependencies: "@nodeutils/defaults-deep" "^1.1.0" async "^2.6.1" @@ -6574,6 +6573,7 @@ ipfs@0.35.0-rc.0: multiaddr "^6.0.0" multiaddr-to-uri "^4.0.1" multibase "~0.6.0" + multicodec "~0.5.0" multihashes "~0.4.14" multihashing-async "~0.5.1" node-fetch "^2.3.0" @@ -10460,17 +10460,16 @@ pino-std-serializers@^2.3.0: resolved "https://registry.yarnpkg.com/pino-std-serializers/-/pino-std-serializers-2.3.0.tgz#34eeaab97c055c28e22c0542ae55978e7e427786" integrity sha512-klfGoOsP6sJH7ON796G4xoUSx2fkpFgKHO4YVVO2zmz31jR+etzc/QzGJILaOIiCD6HTCFgkPx+XN8nk+ruqPw== -pino@5.11.1, pino@^5.10.1, pino@~5.11.0: +pino@5.11.1, pino@^5.10.1, "pino@https://github.com/pinojs/pino/tarball/ab3dd74843eb4186fbc02b43e4b293840182fd7c/pino.tar.gz", pino@~5.11.0: version "5.11.1" - resolved "https://registry.npmjs.org/pino/-/pino-5.11.1.tgz#2d6d8edb7ebc7c354be03bfa04fd436352e1d67b" - integrity sha512-NIua0mGb9Adknq35ONvQmvh93LCUVUjp2+1q1EcvIkJmpJnSd3E5rHVKlKNjzMXFl/z3fI+QA0xXCjPEKNiLvQ== + resolved "https://github.com/pinojs/pino/tarball/ab3dd74843eb4186fbc02b43e4b293840182fd7c/pino.tar.gz#bf13b3983ad9778a2479c688f78dde456d28582b" dependencies: - fast-redact "^1.4.2" + fast-redact "^1.4.4" fast-safe-stringify "^2.0.6" flatstr "^1.0.9" pino-std-serializers "^2.3.0" - quick-format-unescaped "^3.0.0" - sonic-boom "^0.7.1" + quick-format-unescaped "^3.0.2" + sonic-boom "^0.7.3" pkg-conf@^2.0.0: version "2.1.0" @@ -11070,10 +11069,10 @@ querystringify@^2.0.0: resolved "https://registry.yarnpkg.com/querystringify/-/querystringify-2.1.0.tgz#7ded8dfbf7879dcc60d0a644ac6754b283ad17ef" integrity sha512-sluvZZ1YiTLD5jsqZcDmFyV2EwToyXZBfpoVOmktMmW+VEnhgakFHnasVph65fOjGPTWN0Nw3+XQaSeMayr0kg== -quick-format-unescaped@^3.0.0: - version "3.0.1" - resolved "https://registry.yarnpkg.com/quick-format-unescaped/-/quick-format-unescaped-3.0.1.tgz#e1e8526f6aa56dc5452926111461e85755f79acf" - integrity sha512-Tnk4iJQ8x3V8ml3x9sLIf4tSDaVB9OJY/5gOrnxgK63CXKphhn8oYOPI4tqnXPQcZ3tCv7GFjeoYY5h6UAvuzg== +quick-format-unescaped@^3.0.2: + version "3.0.2" + resolved "https://registry.yarnpkg.com/quick-format-unescaped/-/quick-format-unescaped-3.0.2.tgz#0137e94d8fb37ffeb70040535111c378e75396fb" + integrity sha512-FXTaCkwvpIlkdKeGDNgcq07SXWS383noQUuZjvdE1QcTt+eLuqof6/BDiEPqB59FWLie/l91+HtlJSw7iCViSA== quick-lru@^1.0.0: version "1.1.0" @@ -12275,9 +12274,9 @@ somever@2.x.x: bounce "1.x.x" hoek "6.x.x" -sonic-boom@^0.7.1: +sonic-boom@^0.7.3: version "0.7.3" - resolved "https://registry.npmjs.org/sonic-boom/-/sonic-boom-0.7.3.tgz#cbfc18e87c2b8078b00e38ad9475c05fce5ea696" + resolved "https://registry.yarnpkg.com/sonic-boom/-/sonic-boom-0.7.3.tgz#cbfc18e87c2b8078b00e38ad9475c05fce5ea696" integrity sha512-A9EyoIeLD+g9vMLYQKjNCatJtAKdBQMW03+L8ZWWX/A6hq+srRCwdqHrBD1R8oSMLXov3oHN13dljtZf12q2Ow== dependencies: flatstr "^1.0.9"