feat: Support capturing assets from allowed hostnames (#320)

* WIP * Add an integration tests for allowed hostnames * Update flag description * Add test for flag
percy · Aug 7, 2019 · c3fb8ac · c3fb8ac
1 parent d3662e5
commit c3fb8ac
Show file tree

Hide file tree

Showing 14 changed files with 85 additions and 8 deletions.
diff --git a/package.json b/package.json
@@ -142,7 +142,7 @@
     "preversion": "npm run clean",
     "test": "npm run build-client && PERCY_TOKEN=abc mocha --forbid-only \"test/**/*.test.ts\" --exclude \"test/percy-agent-client/**/*.test.ts\" --exclude \"test/integration/**/*\"",
     "test-client": "karma start ./test/percy-agent-client/karma.conf.js",
-    "test-integration": "npm run build-client && node ./bin/run exec -- mocha test/integration/**/*.test.ts",
+    "test-integration": "npm run build-client && node ./bin/run exec -h localtest.me -- mocha test/integration/**/*.test.ts",
     "test-snapshot-command": "./bin/run snapshot test/integration/test-static-site -b /dummy-base-url -i '(red-keep)' -c '\\.(html)$'",
     "version": "oclif-dev readme && git add README.md",
     "watch": "npm-watch"

diff --git a/src/commands/exec.ts b/src/commands/exec.ts
@@ -15,6 +15,11 @@ export default class Exec extends PercyCommand {
   ]
 
   static flags = {
+    'allowed-hostname': flags.string({
+      char: 'h',
+      description: 'Allowable hostname(s) to capture assets from',
+      multiple: true,
+    }),
     'network-idle-timeout': flags.integer({
       char: 't',
       default: DEFAULT_CONFIGURATION.agent['asset-discovery']['network-idle-timeout'],

diff --git a/src/commands/snapshot.ts b/src/commands/snapshot.ts
@@ -38,6 +38,11 @@ export default class Snapshot extends PercyCommand {
       'of the webserver\'s root path, set that subdirectory with this flag.',
       default: DEFAULT_CONFIGURATION['static-snapshots']['base-url'],
     }),
+    'allowed-hostname': flags.string({
+      char: 'h',
+      description: 'Allowable hostname(s) to capture assets from',
+      multiple: true,
+    }),
     // from exec command. needed to start the agent service.
     'network-idle-timeout': flags.integer({
       char: 't',

diff --git a/src/commands/start.ts b/src/commands/start.ts
@@ -19,6 +19,11 @@ export default class Start extends PercyCommand {
       char: 'd',
       description: 'start as a detached process',
     }),
+    'allowed-hostname': flags.string({
+      char: 'h',
+      description: 'Allowable hostname(s) to capture assets from',
+      multiple: true,
+    }),
     'network-idle-timeout': flags.integer({
       char: 't',
       default: DEFAULT_CONFIGURATION.agent['asset-discovery']['network-idle-timeout'],

diff --git a/src/configuration/asset-discovery-configuration.ts b/src/configuration/asset-discovery-configuration.ts
@@ -1,4 +1,5 @@
 export interface AssetDiscoveryConfiguration {
+  'allowed-hostnames': string[],
   'network-idle-timeout': number,
   'page-pool-size-min': number,
   'page-pool-size-max': number

diff --git a/src/configuration/configuration.ts b/src/configuration/configuration.ts
@@ -19,6 +19,7 @@ export const DEFAULT_CONFIGURATION: Configuration = {
   'agent': {
     'port': DEFAULT_PORT,
     'asset-discovery': {
+      'allowed-hostnames': [],
       'network-idle-timeout': 50, // ms
       'page-pool-size-min': 1, // pages
       'page-pool-size-max': 5, // pages

diff --git a/src/services/asset-discovery-service.ts b/src/services/asset-discovery-service.ts
@@ -19,10 +19,10 @@ export class AssetDiscoveryService extends PercyClientService {
 
   constructor(buildId: number, configuration?: AssetDiscoveryConfiguration) {
     super()
-    this.responseService = new ResponseService(buildId)
     this.browser = null
     this.pagePool = null
     this.configuration = configuration || DEFAULT_CONFIGURATION.agent['asset-discovery']
+    this.responseService = new ResponseService(buildId, this.configuration['allowed-hostnames'])
   }
 
   async setup() {

diff --git a/src/services/configuration-service.ts b/src/services/configuration-service.ts
@@ -39,6 +39,10 @@ export default class ConfigurationService {
       this.configuration.agent.port = flags.port
     }
 
+    if (flags['allowed-hostname']) {
+      this.configuration.agent['asset-discovery']['allowed-hostnames'] = flags['allowed-hostname']
+    }
+
     if (flags['network-idle-timeout']) {
       this.configuration.agent['asset-discovery']['network-idle-timeout'] = flags['network-idle-timeout']
     }

diff --git a/src/services/response-service.ts b/src/services/response-service.ts
@@ -14,10 +14,28 @@ export default class ResponseService extends PercyClientService {
 
   readonly ALLOWED_RESPONSE_STATUSES = [200, 201, 304]
   responsesProcessed: Map<string, string> = new Map()
+  allowedHostnames: string[]
 
-  constructor(buildId: number) {
+  constructor(buildId: number, allowedHostnames: string[]) {
     super()
     this.resourceService = new ResourceService(buildId)
+    this.allowedHostnames = allowedHostnames
+  }
+
+  shouldCaptureResource(rootUrl: string, resourceUrl: string): boolean {
+    // Capture if the resourceUrl is the same as the rootUrL
+    if (resourceUrl.startsWith(rootUrl)) {
+      return true
+    }
+
+    // Capture if the resourceUrl has a hostname in the allowedHostnames
+    const parsedResourceUrl = new URL(resourceUrl)
+    if (this.allowedHostnames.some((hostname) => parsedResourceUrl.hostname === hostname)) {
+      return true
+    }
+
+    // Resource is not allowed
+    return false
   }
 
   async processResponse(rootResourceUrl: string, response: puppeteer.Response, width: number): Promise<any | null> {
@@ -44,13 +62,13 @@ export default class ResponseService extends PercyClientService {
       return
     }
 
-    if (!request.url().startsWith(rootUrl)) {
+    if (!this.shouldCaptureResource(rootUrl, request.url())) {
       // Disallow remote resource requests.
       logger.debug(`Skipping [is_remote_resource] [${width} px]: ${request.url()}`)
       return
     }
 
-    if (!response.url().startsWith(rootUrl)) {
+    if (!this.shouldCaptureResource(rootUrl, response.url())) {
       // Disallow remote redirects.
       logger.debug(`Skipping [is_remote_redirect] [${width} px]: ${response.url()}`)
       return

diff --git a/test/integration/agent-integration.test.ts b/test/integration/agent-integration.test.ts
@@ -1,5 +1,4 @@
 import * as cheerio from 'cheerio'
-import * as fs from 'fs'
 import { Server } from 'http'
 import * as httpServer from 'http-server'
 import { describe } from 'mocha'
@@ -117,14 +116,22 @@ describe('Integration test', () => {
       })
     })
 
-    describe('responsive  assets', () => {
-      it('properly  captures all assets', async () => {
+    describe('responsive assets', () => {
+      it('properly captures all assets', async () => {
         await page.goto(`http://localhost:${PORT}/responsive-assets.html`)
 
         await snapshot(page, 'Responsive assets')
       })
     })
 
+    describe('alternate hostnames', () => {
+      it('properly captures assets from alternate hostnames', async () => {
+        await page.goto(`http://localhost:${PORT}/alternate-hostname.html`)
+
+        await snapshot(page, 'Alternate hostname')
+      })
+    })
+
     describe('stabilizes DOM', () => {
       before(async () => {
         await page.goto(`http://localhost:${PORT}/stabilize-dom.html`)

diff --git a/test/integration/testcases/alternate-hostname.html b/test/integration/testcases/alternate-hostname.html
@@ -0,0 +1,18 @@
+<!DOCTYPE html>
+<html><head>
+    <title>Alternate hostname testing</title>
+</head>
+<body>
+  <div>
+    <p>
+      If alternate hostname support is working, there should be an svg image of the vue logo below.
+    </p>
+    <p>
+      It works because localhost.me resolves to 127.0.0.1, allowing us to serve the vue logo over
+      it, and we start the integration tests by specifying localhost.me as an allowed-hostname
+      with the -h flag.
+    </p>
+    <img src="http://localtest.me:8000/vue.svg">
+  </div>
+</body>
+</html>
diff --git a/test/integration/testcases/vue.svg b/test/integration/testcases/vue.svg
diff --git a/test/services/configuration-service.test.ts b/test/services/configuration-service.test.ts
@@ -23,6 +23,7 @@ describe('ConfigurationService', () => {
       expect(subject['static-snapshots']['snapshot-files']).to.eql('**/*.html')
       expect(subject['static-snapshots']['ignore-files']).to.eql('**/*.htm')
       expect(subject.agent.port).to.eql(1111)
+      expect(subject.agent['asset-discovery']['allowed-hostnames']).to.eql(['localassets.dev'])
       expect(subject.agent['asset-discovery']['network-idle-timeout']).to.eql(50)
       expect(subject.agent['asset-discovery']['page-pool-size-min']).to.eql(5)
       expect(subject.agent['asset-discovery']['page-pool-size-max']).to.eql(20)
@@ -41,13 +42,17 @@ describe('ConfigurationService', () => {
         'base-url': '/flag/',
         'snapshot-files': 'flags/*.html',
         'ignore-files': 'ignore-flags/*.html',
+        'allowed-hostname': ['additional-hostname.local'],
       }
       const subject = new ConfigurationService('test/support/.percy.yml').applyFlags(flags)
 
       expect(subject['static-snapshots']['base-url']).to.eql('/flag/')
       expect(subject['static-snapshots']['snapshot-files']).to.eql('flags/*.html')
       expect(subject['static-snapshots']['ignore-files']).to.eql('ignore-flags/*.html')
       expect(subject.agent['asset-discovery']['network-idle-timeout']).to.eql(51)
+      expect(subject.agent['asset-discovery']['allowed-hostnames']).to.eql(
+        ['additional-hostname.local'],
+      )
     })
   })
 

diff --git a/test/support/.percy.yml b/test/support/.percy.yml
@@ -11,6 +11,8 @@ static-snapshots:
 agent:
   port: 1111
   asset-discovery:
+    allowed-hostnames:
+      - localassets.dev
     network-idle-timeout: 50 # ms
     page-pool-size-min: 5 # pages
     page-pool-size-max: 20 # pages