feat: ✨ Asset discovery headers (#384)

* feat: ✨ Add request-headers option for asset-discovery

* ✅ Add tests for auth header and protected resources

Remove http-server in favor of using express which is already a dep. Also add
express-basic-auth as a dev-dep for testing.

package.json

@@ -61,7 +61,6 @@
     "@types/deepmerge": "^2.2.0",
     "@types/express": "^4.16.0",
     "@types/generic-pool": "^3.1.9",
-    "@types/http-server": "^0.10.0",
     "@types/js-yaml": "^3.11.2",
     "@types/mocha": "^5.2.5",
     "@types/nock": "^11.1.0",
@@ -76,7 +75,7 @@
     "chai-http": "^4.0.0",
     "cheerio": "^1.0.0-rc.3",
     "eslint-config-oclif": "^3.0.0",
-    "http-server": "^0.11.1",
+    "express-basic-auth": "^1.2.0",
     "husky": "^3.0.0",
     "interactor.js": "^1.3.1",
     "karma": "^4.1.0",

src/configuration/asset-discovery-configuration.ts

@@ -1,4 +1,5 @@
 export interface AssetDiscoveryConfiguration {
+  'request-headers': any,
   'allowed-hostnames': string[],
   'network-idle-timeout': number,
   'page-pool-size-min': number,

src/configuration/configuration.ts

@@ -22,6 +22,7 @@ export const DEFAULT_CONFIGURATION: Configuration = {
   'agent': {
     'port': DEFAULT_PORT,
     'asset-discovery': {
+      'request-headers': {},
       'allowed-hostnames': [],
       'network-idle-timeout': 50, // ms
       'page-pool-size-min': 1, // pages

src/percy-agent-client/snapshot-options.ts

@@ -1,4 +1,5 @@
 export interface SnapshotOptions {
+  requestHeaders?: string,
   enableJavaScript?: boolean,
   enableJavascript?: boolean, // deprecated. Use enableJavaScript
   widths?: number[],

src/services/agent-service.ts

@@ -100,6 +100,7 @@ export class AgentService {
     const hasWidths = !!request.body.widths && request.body.widths.length
     const snapshotOptions: SnapshotOptions = {
       percyCSS: percySpecificCSS,
+      requestHeaders: request.body.requestHeaders,
       widths: hasWidths ? request.body.widths : this.snapshotConfig.widths,
       enableJavaScript: request.body.enableJavaScript != null
         ? request.body.enableJavaScript

src/services/asset-discovery-service.ts

@@ -1,3 +1,4 @@
+import * as merge from 'deepmerge'
 import * as pool from 'generic-pool'
 import * as puppeteer from 'puppeteer'
 import { AssetDiscoveryConfiguration } from '../configuration/asset-discovery-configuration'
@@ -102,23 +103,27 @@ export class AssetDiscoveryService extends PercyClientService {
 
     logger.debug(`discovering assets for URL: ${rootResourceUrl}`)
 
-    const enableJavaScript = options.enableJavaScript || false
-    const widths = options.widths || DEFAULT_CONFIGURATION.snapshot.widths
+    const {
+      enableJavaScript = false,
+      widths = DEFAULT_CONFIGURATION.snapshot.widths,
+      requestHeaders,
+    } = options
 
     // Do asset discovery for each requested width in parallel. We don't keep track of which page
     // is doing work, and instead rely on the fact that we always have fewer widths to work on than
     // the number of pages in our pool. If we wanted to do something smarter here, we should consider
     // switching to use puppeteer-cluster instead.
-    profile('--> assetDiscoveryService.discoverForWidths', {url: rootResourceUrl})
+    profile('--> assetDiscoveryService.discoverForWidths', { url: rootResourceUrl })
 
     let resources: any[] = [].concat(...(await Promise.all(
-      widths.map((width) => this.resourcesForWidth(
+      widths.map((width: number) => this.resourcesForWidth(
         // @ts-ignore - for some reason, ts thinks we're assigning null here
         this.pagePool,
         width,
         domSnapshot,
         rootResourceUrl,
         enableJavaScript,
+        requestHeaders,
         logger,
       )),
     )) as any[])
@@ -136,7 +141,7 @@ export class AssetDiscoveryService extends PercyClientService {
       return false
     })
 
-    profile('-> assetDiscoveryService.discoverResources', {resourcesDiscovered: resources.length})
+    profile('-> assetDiscoveryService.discoverResources', { resourcesDiscovered: resources.length })
 
     return resources
   }
@@ -168,15 +173,20 @@ export class AssetDiscoveryService extends PercyClientService {
     domSnapshot: string,
     rootResourceUrl: string,
     enableJavaScript: boolean,
+    requestHeaders: any = {},
     logger: any,
   ): Promise<any[]> {
     logger.debug(`discovering assets for width: ${width}`)
 
-    profile('--> assetDiscoveryService.pool.acquire', {url: rootResourceUrl})
+    profile('--> assetDiscoveryService.pool.acquire', { url: rootResourceUrl })
     const page = await pool.acquire()
     profile('--> assetDiscoveryService.pool.acquire')
     await page.setJavaScriptEnabled(enableJavaScript)
-    await page.setViewport(Object.assign(page.viewport(), {width}))
+    await page.setViewport(Object.assign(page.viewport(), { width }))
+    await page.setExtraHTTPHeaders(merge.all([
+      this.configuration['request-headers'],
+      requestHeaders,
+    ]) as {})
 
     page.on('request', async (request) => {
       try {
@@ -228,7 +238,7 @@ export class AssetDiscoveryService extends PercyClientService {
     })
 
     try {
-      profile('--> assetDiscoveryService.page.goto', {url: rootResourceUrl})
+      profile('--> assetDiscoveryService.page.goto', { url: rootResourceUrl })
       await page.goto(rootResourceUrl)
       profile('--> assetDiscoveryService.page.goto')
 
@@ -240,7 +250,7 @@ export class AssetDiscoveryService extends PercyClientService {
       const maybeResources: any[] = await Promise.all(maybeResourcePromises)
       profile('--> assetDiscoveryServer.waitForResourceProcessing')
 
-      profile('--> assetDiscoveryService.pool.release', {url: rootResourceUrl})
+      profile('--> assetDiscoveryService.pool.release', { url: rootResourceUrl })
       await page.removeAllListeners('request')
       await page.removeAllListeners('requestfinished')
       await page.removeAllListeners('requestfailed')

src/services/response-service.ts

@@ -87,7 +87,7 @@ export default class ResponseService extends PercyClientService {
       }
 
       const redirectedURL = `${rootUrl}${response.headers().location}` as string
-      return this.handleRedirectResouce(url, redirectedURL, width, logger)
+      return this.handleRedirectResouce(url, redirectedURL, request.headers(), width, logger)
     }
 
     return this.handlePuppeteerResource(url, response, width, logger)
@@ -99,11 +99,21 @@ export default class ResponseService extends PercyClientService {
    * requesting url. This works since axios follows the redirect chain
    * automatically.
    */
-  async handleRedirectResouce(originalURL: string, redirectedURL: string, width: number, logger: any) {
+  async handleRedirectResouce(
+    originalURL: string,
+    redirectedURL: string,
+    requestHeaders: any,
+    width: number,
+    logger: any,
+  ) {
     logger.debug(`Making local copy of redirected response: ${originalURL}`)
 
     try {
-      const { data, headers } = await Axios(originalURL, { responseType: 'arraybuffer' }) as any
+      const { data, headers } = await Axios(originalURL, {
+        responseType: 'arraybuffer',
+        headers: requestHeaders,
+      }) as any
+
       const buffer = Buffer.from(data)
       const sha = crypto.createHash('sha256').update(buffer).digest('hex')
       const localCopy = path.join(os.tmpdir(), sha)

test/integration/agent-integration.test.ts

@@ -1,6 +1,7 @@
 import * as cheerio from 'cheerio'
+import * as express from 'express'
+import * as basicAuth from 'express-basic-auth'
 import { Server } from 'http'
-import * as httpServer from 'http-server'
 import { describe } from 'mocha'
 import * as puppeteer from 'puppeteer'
 import { agentJsFilename, postSnapshot } from '../../src/utils/sdk-utils'
@@ -108,11 +109,14 @@ describe('Integration test', () => {
   describe('on local test cases', () => {
     const testCaseDir = `${__dirname}/testcases`
     const PORT = 8000
+
+    let app: express.Application
     let server: Server
 
     before(() => {
-      server = httpServer.createServer({root: testCaseDir}) as Server
-      server.listen(PORT)
+      app = express()
+      app.use(express.static(testCaseDir))
+      server = app.listen(PORT)
     })
 
     after(() => {
@@ -130,6 +134,39 @@ describe('Integration test', () => {
       })
     })
 
+    describe('protected resources', () => {
+      const username = 'test'
+      const password = 'test'
+
+      before(async () => {
+        app.get('/auth/redirected.png', (_, res) => {
+          res.redirect(301, '/auth/fairy-emojione.png')
+        })
+
+        app.use(
+          '/auth',
+          basicAuth({ users: { [username]: password }, challenge: true }),
+          express.static(testCaseDir),
+        )
+
+        await page.authenticate({ username, password })
+        await page.goto(`http://localhost:${PORT}/auth/protected-with-basic-auth.html`)
+      })
+
+      it('does not capture protected resources without the correct headers', async () => {
+        // the snapshot should show missing resources
+        await snapshot(page, 'Protected assets')
+      })
+
+      it('captures protected resources with the correct headers', async () => {
+        await snapshot(page, 'Captured protected assets', {
+          requestHeaders: {
+            Authorization: `Basic ${Buffer.from(`${username}:${password}`).toString('base64')}`,
+          },
+        })
+      })
+    })
+
     describe('large resources', () => {
       it('snapshots large DOM', async () => {
         await page.goto(`http://localhost:${PORT}/exceeds-dom-snapshot-size-limit.html`)

test/integration/testcases/protected-with-basic-auth.html

@@ -0,0 +1,12 @@
+<!doctype html>
+<html>
+  <head>
+    <title>Protected resources integration test case</title>
+    <meta charset="utf-8"/>
+  </head>
+  <body>
+    <p>All requests on this page are protected</p>
+    <img src="redirected.png" />
+    <img src="taxi-emojione.png" />
+  </body>
+</html>