From 362c97effcec4a864dcfc1a544a2d1f8aa3c6347 Mon Sep 17 00:00:00 2001 From: Amean Asad Date: Fri, 3 Nov 2023 11:11:36 -0400 Subject: [PATCH] feat: add customer origin url fallback (#37) * feat: add customer origin url fallback * remove unused code * simplify code * feat: fallback from flat file origins * use format to determine file format * rename origin url * DRY up fetch options * raw fallback url * address comments * fix * fixes * remove unused code * fix again --- src/client.js | 167 ++++++++++++++++++++------------------- src/types.js | 14 ++++ src/utils/errors.js | 3 +- test/fallback.spec.js | 46 ++++++++++- test/fixtures/hello.json | 3 + test/index.spec.js | 8 +- test/test-utils.js | 34 ++++++-- 7 files changed, 181 insertions(+), 94 deletions(-) create mode 100644 test/fixtures/hello.json diff --git a/src/client.js b/src/client.js index ae43f41..b90b4da 100644 --- a/src/client.js +++ b/src/client.js @@ -14,6 +14,7 @@ import { isErrorUnavoidable } from './utils/errors.js' const MAX_NODE_WEIGHT = 100 /** * @typedef {import('./types.js').Node} Node + * @typedef {import('./types.js').FetchOptions} FetchOptions */ export class Saturn { @@ -21,19 +22,20 @@ export class Saturn { static defaultRaceCount = 3 /** * - * @param {object} [opts={}] - * @param {string} [opts.clientKey] - * @param {string} [opts.clientId=randomUUID()] - * @param {string} [opts.cdnURL=saturn.ms] - * @param {number} [opts.connectTimeout=5000] - * @param {number} [opts.downloadTimeout=0] - * @param {string} [opts.orchURL] - * @param {number} [opts.fallbackLimit] - * @param {boolean} [opts.experimental] - * @param {import('./storage/index.js').Storage} [opts.storage] + * @param {object} [config={}] + * @param {string} [config.clientKey] + * @param {string} [config.clientId=randomUUID()] + * @param {string} [config.cdnURL=saturn.ms] + * @param {number} [config.connectTimeout=5000] + * @param {number} [config.downloadTimeout=0] + * @param {string} [config.orchURL] + * @param {string} [config.customerFallbackURL] + * @param {number} [config.fallbackLimit] + * @param {boolean} [config.experimental] + * @param {import('./storage/index.js').Storage} [config.storage] */ - constructor (opts = {}) { - this.opts = Object.assign({}, { + constructor (config = {}) { + this.config = Object.assign({}, { clientId: randomUUID(), cdnURL: 'l1s.saturn.ms', logURL: 'https://twb3qukm2i654i3tnvx36char40aymqq.lambda-url.us-west-2.on.aws/', @@ -42,9 +44,9 @@ export class Saturn { fallbackLimit: 5, connectTimeout: 5_000, downloadTimeout: 0 - }, opts) + }, config) - if (!this.opts.clientKey) { + if (!this.config.clientKey) { throw new Error('clientKey is required') } @@ -55,28 +57,24 @@ export class Saturn { if (this.reportingLogs && this.hasPerformanceAPI) { this._monitorPerformanceBuffer() } - this.storage = this.opts.storage || memoryStorage() - this.loadNodesPromise = this.opts.experimental ? this._loadNodes(this.opts) : null + this.storage = this.config.storage || memoryStorage() + this.loadNodesPromise = this.config.experimental ? this._loadNodes(this.config) : null } /** * * @param {string} cidPath - * @param {object} [opts={}] - * @param {Node[]} [opts.nodes] - * @param {Node} [opts.node] - * @param {('car'|'raw')} [opts.format] - * @param {number} [opts.connectTimeout=5000] - * @param {number} [opts.downloadTimeout=0] + * @param {FetchOptions} [opts={}] * @returns {Promise} */ async fetchCIDWithRace (cidPath, opts = {}) { - const [cid] = (cidPath ?? '').split('/') - CID.parse(cid) - - const jwt = await getJWT(this.opts, this.storage) - - const options = Object.assign({}, this.opts, { format: 'car', jwt }, opts) + const options = Object.assign({}, this.config, { format: 'car' }, opts) + if (!opts.originFallback) { + const [cid] = (cidPath ?? '').split('/') + CID.parse(cid) + const jwt = await getJWT(options, this.storage) + options.jwt = jwt + } if (!isBrowserContext) { options.headers = { @@ -87,7 +85,7 @@ export class Saturn { let nodes = options.nodes if (!nodes || nodes.length === 0) { - const replacementNode = options.node ?? { url: this.opts.cdnURL } + const replacementNode = { url: options.cdnURL } nodes = [replacementNode] } const controllers = [] @@ -157,22 +155,20 @@ export class Saturn { /** * * @param {string} cidPath - * @param {object} [opts={}] - * @param {('car'|'raw')} [opts.format] - * @param {Node} [opts.node] - * @param {number} [opts.connectTimeout=5000] - * @param {number} [opts.downloadTimeout=0] + * @param {FetchOptions} [opts={}] * @returns {Promise} */ async fetchCID (cidPath, opts = {}) { - const [cid] = (cidPath ?? '').split('/') - CID.parse(cid) - - const jwt = await getJWT(this.opts, this.storage) + const options = Object.assign({}, this.config, { format: 'car' }, opts) + if (!opts.originFallback) { + const [cid] = (cidPath ?? '').split('/') + CID.parse(cid) + const jwt = await getJWT(this.config, this.storage) + options.jwt = jwt + } - const options = Object.assign({}, this.opts, { format: 'car', jwt }, opts) - const node = options.node - const origin = node?.url ?? this.opts.cdnURL + const node = options.nodes && options.nodes[0] + const origin = node?.url ?? this.config.cdnURL const url = this.createRequestURL(cidPath, { ...options, url: origin }) let log = { @@ -242,20 +238,15 @@ export class Saturn { /** * * @param {string} cidPath - * @param {object} [opts={}] - * @param {('car'|'raw')} [opts.format] - * @param {boolean} [opts.raceNodes] - * @param {string} [opts.url] - * @param {number} [opts.connectTimeout=5000] - * @param {number} [opts.downloadTimeout=0] - * @param {AbortController} [opts.controller] + * @param {FetchOptions} [opts={}] * @returns {Promise>} */ async * fetchContentWithFallback (cidPath, opts = {}) { - const upstreamController = opts.controller; - delete opts.controller; + const upstreamController = opts.controller + delete opts.controller let lastError = null + let skipNodes = false // we use this to checkpoint at which chunk a request failed. // this is temporary until range requests are supported. let byteCountCheckpoint = 0 @@ -264,16 +255,17 @@ export class Saturn { throw new Error(`All attempts to fetch content have failed. Last error: ${lastError.message}`) } - const fetchContent = async function * () { - const controller = new AbortController(); - opts.controller = controller; + const fetchContent = async function * (options) { + const controller = new AbortController() + opts.controller = controller if (upstreamController) { upstreamController.signal.addEventListener('abort', () => { - controller.abort(); - }); + controller.abort() + }) } let byteCount = 0 - const byteChunks = await this.fetchContent(cidPath, opts) + const fetchOptions = Object.assign(opts, { format: 'car' }, options) + const byteChunks = await this.fetchContent(cidPath, fetchOptions) for await (const chunk of byteChunks) { // avoid sending duplicate chunks if (byteCount < byteCountCheckpoint) { @@ -291,33 +283,34 @@ export class Saturn { } }.bind(this) + // Use CDN origin if node list is not loaded if (this.nodes.length === 0) { // fetch from origin in the case that no nodes are loaded - opts.url = this.opts.cdnURL + opts.nodes = Array({ url: this.config.cdnURL }) try { yield * fetchContent() return } catch (err) { lastError = err if (err.res?.status === 410 || isErrorUnavoidable(err)) { - throwError() + skipNodes = true + } else { + await this.loadNodesPromise } - await this.loadNodesPromise } } let fallbackCount = 0 const nodes = this.nodes for (let i = 0; i < nodes.length; i++) { - if (fallbackCount > this.opts.fallbackLimit || upstreamController?.signal.aborted) { - return + if (fallbackCount > this.config.fallbackLimit || skipNodes || upstreamController?.signal.aborted) { + break } if (opts.raceNodes) { opts.nodes = nodes.slice(i, i + Saturn.defaultRaceCount) } else { - opts.node = nodes[i] + opts.nodes = Array(nodes[i]) } - try { yield * fetchContent() return @@ -331,6 +324,17 @@ export class Saturn { } if (lastError) { + const originUrl = opts.customerFallbackURL ?? this.config.customerFallbackURL + // Use customer origin if cid is not retrievable by lassie. + if (originUrl) { + opts.nodes = Array({ url: originUrl }) + try { + yield * fetchContent({ format: null, originFallback: true }) + return + } catch (err) { + lastError = err + } + } throwError() } } @@ -338,11 +342,7 @@ export class Saturn { /** * * @param {string} cidPath - * @param {object} [opts={}] - * @param {('car'|'raw')} [opts.format] - * @param {boolean} [opts.raceNodes] - * @param {number} [opts.connectTimeout=5000] - * @param {number} [opts.downloadTimeout=0] + * @param {FetchOptions} [opts={}] * @returns {Promise>} */ async * fetchContent (cidPath, opts = {}) { @@ -365,7 +365,11 @@ export class Saturn { try { const itr = metricsIterable(asAsyncIterable(res.body)) - yield * extractVerifiedContent(cidPath, itr) + if (opts.format === 'car') { + yield * extractVerifiedContent(cidPath, itr) + } else { + yield * itr + } } catch (err) { log.error = err.message controller.abort() @@ -379,11 +383,7 @@ export class Saturn { /** * * @param {string} cidPath - * @param {object} [opts={}] - * @param {('car'|'raw')} [opts.format] - * @param {boolean} [opts.raceNodes] - * @param {number} [opts.connectTimeout=5000] - * @param {number} [opts.downloadTimeout=0] + * @param {FetchOptions} [opts={}] * @returns {Promise} */ async fetchContentBuffer (cidPath, opts = {}) { @@ -395,14 +395,21 @@ export class Saturn { * @param {string} cidPath * @param {object} [opts={}] * @param {string} [opts.url] + * @param {string} [opts.format] + * @param {string} [opts.originFallback] + * @param {object} [opts.jwt] * @returns {URL} */ - createRequestURL (cidPath, opts) { - let origin = opts.url ?? this.opts.cdnURL + createRequestURL (cidPath, opts = {}) { + let origin = opts.url ?? this.config.cdnURL origin = addHttpPrefix(origin) + if (opts.originFallback) { + return new URL(origin) + } const url = new URL(`${origin}/ipfs/${cidPath}`) - url.searchParams.set('format', opts.format) + if (opts.format) url.searchParams.set('format', opts.format) + if (opts.format === 'car') { url.searchParams.set('dag-scope', 'entity') } @@ -444,10 +451,10 @@ export class Saturn { : this.logs await fetch( - this.opts.logURL, + this.config.logURL, { method: 'POST', - body: JSON.stringify({ bandwidthLogs, logSender: this.opts.logSender }) + body: JSON.stringify({ bandwidthLogs, logSender: this.config.logSender }) } ) @@ -569,7 +576,7 @@ export class Saturn { const url = new URL(origin) const controller = new AbortController() - const options = Object.assign({}, { method: 'GET' }, this.opts) + const options = Object.assign({}, { method: 'GET' }, this.config) const connectTimeout = setTimeout(() => { controller.abort() diff --git a/src/types.js b/src/types.js index 844a074..26d3a78 100644 --- a/src/types.js +++ b/src/types.js @@ -12,4 +12,18 @@ * @property {string} url */ +/** + * Common options for fetch functions. + * + * @typedef {object} FetchOptions + * @property {Node[]} [nodes] - An array of nodes. + * @property {('car'|'raw')} [format] - The format of the fetched content. + * @property {boolean} [originFallback] - Is this a fallback to the customer origin + * @property {boolean} [raceNodes] - Does the fetch race multiple nodes on requests. + * @property {string} [customerFallbackURL] - Customer Origin that is a fallback. + * @property {number} [connectTimeout=5000] - Connection timeout in milliseconds. + * @property {number} [downloadTimeout=0] - Download timeout in milliseconds. + * @property {AbortController} [controller] + */ + export {} diff --git a/src/utils/errors.js b/src/utils/errors.js index f80566b..5833367 100644 --- a/src/utils/errors.js +++ b/src/utils/errors.js @@ -19,7 +19,8 @@ export function isErrorUnavoidable (error) { /file does not exist/, /Cannot read properties of undefined \(reading '([^']+)'\)/, /([a-zA-Z_.]+) is undefined/, - /undefined is not an object \(evaluating '([^']+)'\)/ + /undefined is not an object \(evaluating '([^']+)'\)/, + /all retrievals failed/ ] for (const pattern of errorPatterns) { diff --git a/test/fallback.spec.js b/test/fallback.spec.js index 425393d..fa285c5 100644 --- a/test/fallback.spec.js +++ b/test/fallback.spec.js @@ -3,12 +3,13 @@ import assert from 'node:assert/strict' import { describe, mock, test } from 'node:test' import { Saturn } from '#src/index.js' -import { concatChunks, generateNodes, getMockServer, HTTP_STATUS_GONE, mockJWT, mockNodesHandlers, mockOrchHandler, mockSaturnOriginHandler, MSW_SERVER_OPTS } from './test-utils.js' +import { concatChunks, generateNodes, getMockServer, HTTP_STATUS_GONE, HTTP_STATUS_TIMEOUT, mockFlatFileOriginHandler, mockJWT, mockNodesHandlers, mockOrchHandler, mockOriginHandler, MSW_SERVER_OPTS } from './test-utils.js' const TEST_DEFAULT_ORCH = 'https://orchestrator.strn.pl.test/nodes' const TEST_NODES_LIST_KEY = 'saturn-nodes' const TEST_AUTH = 'https://auth.test/' const TEST_ORIGIN_DOMAIN = 'l1s.saturn.test' +const TEST_CUSTOMER_ORIGIN = 'customer.test/ipfs/bafkreifjjcie6lypi6ny7amxnfftagclbuxndqonfipmb64f2km2devei4' const CLIENT_KEY = 'key' const options = { @@ -120,7 +121,7 @@ describe('Client Fallback', () => { const handlers = [ mockOrchHandler(2, TEST_DEFAULT_ORCH, TEST_ORIGIN_DOMAIN), mockJWT(TEST_AUTH), - mockSaturnOriginHandler(TEST_ORIGIN_DOMAIN, 0, true), + mockOriginHandler(TEST_ORIGIN_DOMAIN, 0, true), ...mockNodesHandlers(2, TEST_ORIGIN_DOMAIN) ] const server = getMockServer(handlers) @@ -153,7 +154,7 @@ describe('Client Fallback', () => { const handlers = [ mockOrchHandler(5, TEST_DEFAULT_ORCH, TEST_ORIGIN_DOMAIN), mockJWT(TEST_AUTH), - mockSaturnOriginHandler(TEST_ORIGIN_DOMAIN, 0, true), + mockOriginHandler(TEST_ORIGIN_DOMAIN, 0, true), ...mockNodesHandlers(5, TEST_ORIGIN_DOMAIN) ] const server = getMockServer(handlers) @@ -186,7 +187,7 @@ describe('Client Fallback', () => { const handlers = [ mockOrchHandler(5, TEST_DEFAULT_ORCH, TEST_ORIGIN_DOMAIN), mockJWT(TEST_AUTH), - mockSaturnOriginHandler(TEST_ORIGIN_DOMAIN, 0, true), + mockOriginHandler(TEST_ORIGIN_DOMAIN, 0, true), ...mockNodesHandlers(5, TEST_ORIGIN_DOMAIN, 2) ] const server = getMockServer(handlers) @@ -273,6 +274,43 @@ describe('Client Fallback', () => { server.close() }) + test('should hit origin if failed to fetch', async (t) => { + const numNodes = 3 + const handlers = [ + mockOrchHandler(numNodes, TEST_DEFAULT_ORCH, TEST_ORIGIN_DOMAIN), + mockJWT(TEST_AUTH), + mockOriginHandler(TEST_ORIGIN_DOMAIN, 0, true), + mockFlatFileOriginHandler(TEST_CUSTOMER_ORIGIN, 0, false), + ...mockNodesHandlers(numNodes, TEST_ORIGIN_DOMAIN, numNodes, HTTP_STATUS_TIMEOUT) + ] + + const server = getMockServer(handlers) + server.listen(MSW_SERVER_OPTS) + + const expectedNodes = generateNodes(5, TEST_ORIGIN_DOMAIN) + + // Mocking storage object + const mockStorage = { + get: async (key) => expectedNodes, + set: async (key, value) => { return null } + } + t.mock.method(mockStorage, 'get') + t.mock.method(mockStorage, 'set') + + const saturn = new Saturn({ storage: mockStorage, customerFallbackURL: TEST_CUSTOMER_ORIGIN, ...options }) + + const cid = saturn.fetchContentWithFallback(TEST_CUSTOMER_ORIGIN, { raceNodes: true }) + + const buffer = await concatChunks(cid) + const actualContent = String.fromCharCode(...buffer) + const jsonContent = JSON.parse(actualContent) + const expectedContent = JSON.parse('{ "hello": "world" }') + + assert.deepEqual(jsonContent, expectedContent) + mock.reset() + server.close() + }) + test('Should abort fallback on 410s', async () => { const numNodes = 3 const handlers = [ diff --git a/test/fixtures/hello.json b/test/fixtures/hello.json new file mode 100644 index 0000000..d02fab0 --- /dev/null +++ b/test/fixtures/hello.json @@ -0,0 +1,3 @@ +{ + "hello": "world" +} \ No newline at end of file diff --git a/test/index.spec.js b/test/index.spec.js index df1c26f..96015b9 100644 --- a/test/index.spec.js +++ b/test/index.spec.js @@ -17,23 +17,23 @@ describe('Saturn client', () => { it('should work with custom client ID', () => { const clientId = randomUUID() const saturn = new Saturn({ clientId, clientKey }) - assert.strictEqual(saturn.opts.clientId, clientId) + assert.strictEqual(saturn.config.clientId, clientId) }) it('should work with custom CDN URL', () => { const cdnURL = 'custom.com' const saturn = new Saturn({ cdnURL, clientKey }) - assert.strictEqual(saturn.opts.cdnURL, cdnURL) + assert.strictEqual(saturn.config.cdnURL, cdnURL) }) it('should work with custom connect timeout', () => { const saturn = new Saturn({ connectTimeout: 1234, clientKey }) - assert.strictEqual(saturn.opts.connectTimeout, 1234) + assert.strictEqual(saturn.config.connectTimeout, 1234) }) it('should work with custom download timeout', () => { const saturn = new Saturn({ downloadTimeout: 3456, clientKey }) - assert.strictEqual(saturn.opts.downloadTimeout, 3456) + assert.strictEqual(saturn.config.downloadTimeout, 3456) }) }) diff --git a/test/test-utils.js b/test/test-utils.js index 7f2b33e..aa07c7c 100644 --- a/test/test-utils.js +++ b/test/test-utils.js @@ -46,15 +46,15 @@ export function generateNodes (count, originDomain) { /** * Generates a mock handler to mimick Saturn's orchestrator /nodes endpoint. * - * @param {string} cdnURL - orchestratorUrl + * @param {string} originUrl - originUrl * @param {number} delay - request delay in ms * @param {boolean} error * @returns {RestHandler} */ -export function mockSaturnOriginHandler (cdnURL, delay = 0, error = false) { - cdnURL = addHttpPrefix(cdnURL) - cdnURL = `${cdnURL}/ipfs/:cid` - return rest.get(cdnURL, (req, res, ctx) => { +export function mockOriginHandler (originUrl, delay = 0, error = false) { + originUrl = addHttpPrefix(originUrl) + originUrl = `${originUrl}/ipfs/:cid` + return rest.get(originUrl, (req, res, ctx) => { if (error) { throw Error('Simulated Error') } @@ -68,6 +68,30 @@ export function mockSaturnOriginHandler (cdnURL, delay = 0, error = false) { }) } +/** + * Generates a mock handler to mimick an origin that serves flat files. + * + * @param {string} originUrl - originUrl + * @param {number} delay - request delay in ms + * @param {boolean} error + * @returns {RestHandler} + */ +export function mockFlatFileOriginHandler (originUrl, delay = 0, error = false) { + originUrl = addHttpPrefix(originUrl) + return rest.get(originUrl, (req, res, ctx) => { + if (error) { + throw Error('Simulated Error') + } + const filepath = getFixturePath('hello.json') + const fileContents = fs.readFileSync(filepath) + return res( + ctx.delay(delay), + ctx.status(HTTP_STATUS_OK), + ctx.body(fileContents) + ) + }) +} + /** * Generates a mock handler to mimick Saturn's orchestrator /nodes endpoint. *