feat: Improved large resource handling (#243)

* skip early resources that exceed our max file size

* skip snapshots that have dom snapshots exceeding our max file size

* truncate large dom snapshots from the debug logs

* avoid mutating incoming dom snapshot

* add test cases for each of the large resource paths

* skip resources based on inspecting the local copy filesize not reading the buffer

* minor log improvements

src/services/agent-service.ts

@@ -69,9 +69,20 @@ export default class AgentService {
   private async handleSnapshot(request: express.Request, response: express.Response) {
     profile('agentService.handleSnapshot')
 
+    // truncate domSnapshot for the logs if it's very large
+    let domSnapshotLog = request.body.domSnapshot
+    if (domSnapshotLog.length > Constants.MAX_LOG_LENGTH) {
+      domSnapshotLog = domSnapshotLog.substring(0, Constants.MAX_LOG_LENGTH)
+      domSnapshotLog += `[truncated at ${Constants.MAX_LOG_LENGTH}]`
+    }
+
     logger.debug('handling snapshot:')
     logger.debug(`-> headers: ${JSON.stringify(request.headers)}`)
-    logger.debug(`-> body: ${JSON.stringify(request.body)}`)
+    logger.debug(`-> name: ${request.body.name}`)
+    logger.debug(`-> url: ${request.body.url}`)
+    logger.debug(`-> clientInfo: ${request.body.clientInfo}`)
+    logger.debug(`-> environmentInfo: ${request.body.environmentInfo}`)
+    logger.debug(`-> domSnapshot: ${domSnapshotLog}`)
 
     if (!this.snapshotService) { return response.json({success: false}) }
 
@@ -82,9 +93,16 @@ export default class AgentService {
       minHeight: request.body.minHeight || snapshotConfiguration['min-height'],
     }
 
+    const domSnapshot = request.body.domSnapshot
+
+    if (domSnapshot.length > Constants.MAX_FILE_SIZE_BYTES) {
+      logger.info(`snapshot skipped[max_file_size_exceeded]: '${request.body.name}'`)
+      return response.json({success: true})
+    }
+
     const resources = await this.snapshotService.buildResources(
       request.body.url,
-      request.body.domSnapshot,
+      domSnapshot,
       snapshotOptions,
     )
 

src/services/constants.ts

@@ -6,4 +6,7 @@ export default class Constants {
   static readonly SNAPSHOT_PATH = '/percy/snapshot'
   static readonly STOP_PATH = '/percy/stop'
   static readonly HEALTHCHECK_PATH = '/percy/healthcheck'
+
+  static readonly MAX_FILE_SIZE_BYTES = 15728640 // 15MB
+  static readonly MAX_LOG_LENGTH = 1024
 }

src/services/response-service.ts

@@ -5,6 +5,7 @@ import * as path from 'path'
 import * as puppeteer from 'puppeteer'
 import {URL} from 'url'
 import logger from '../utils/logger'
+import Constants from './constants'
 import PercyClientService from './percy-client-service'
 import ResourceService from './resource-service'
 
@@ -56,16 +57,23 @@ export default class ResponseService extends PercyClientService {
     }
 
     const localCopy = await this.makeLocalCopy(response)
-    const contentType = response.headers()['content-type']
 
+    const responseBodySize = fs.statSync(localCopy).size
+    if (responseBodySize > Constants.MAX_FILE_SIZE_BYTES) {
+      // Skip large resources
+      logger.debug(`Skipping [max_file_size_exceeded_${responseBodySize}] [${width} px]: ${response.url()}`)
+      return
+    }
+
+    const contentType = response.headers()['content-type']
     const resource = this.resourceService.createResourceFromFile(url, localCopy, contentType)
     this.responsesProcessed.set(url, resource)
 
     return resource
   }
 
   async makeLocalCopy(response: puppeteer.Response): Promise<string> {
-    logger.debug(`making local copy of response: ${response.url()}`)
+    logger.debug(`Making local copy of response: ${response.url()}`)
 
     const buffer = await response.buffer()
     const sha = crypto.createHash('sha256').update(buffer).digest('hex')
@@ -74,7 +82,7 @@ export default class ResponseService extends PercyClientService {
     if (!fs.existsSync(filename)) {
       fs.writeFileSync(filename, buffer)
     } else {
-      logger.debug(`Skipping file copy (already copied): ${response.url()}`)
+      logger.debug(`Skipping file copy [already_copied]: ${response.url()}`)
     }
 
     return filename