core(driver): attach to worker targets

cli/test/smokehouse/test-definitions/oopif-scripts.js

@@ -61,8 +61,13 @@ const expectations = {
               {url: 'http://localhost:10503/simple-script.js', resourceType: 'Fetch'},
               {url: 'http://localhost:10200/simple-worker.js'},
               {url: 'http://localhost:10503/simple-worker.js'},
-              // Several requests are emitted in workers but Lighthouse doesn't capture them.
-              // https://github.com/GoogleChrome/lighthouse/issues/14211
+              // Requests from worker targets
+              {url: 'http://localhost:10200/simple-worker.mjs'},
+              {url: 'http://localhost:10503/simple-worker.mjs'},
+              {url: 'http://localhost:10200/simple-script.js?esm', resourceType: 'Script'},
+              {url: 'http://localhost:10503/simple-script.js?esm', resourceType: 'Script'},
+              {url: 'http://localhost:10200/simple-script.js?importScripts', resourceType: 'Other'},
+              {url: 'http://localhost:10503/simple-script.js?importScripts', resourceType: 'Other'},
             ],
             // Ensure the above is exhaustive (except for favicon, which won't be fetched in devtools/LR).
             _excludes: [

core/computed/page-dependency-graph.js

@@ -78,6 +78,7 @@ class PageDependencyGraph {
 
     networkRecords.forEach(record => {
       if (IGNORED_MIME_TYPES_REGEX.test(record.mimeType)) return;
+      if (record.sessionTargetType === 'worker') return;
 
       // Network record requestIds can be duplicated for an unknown reason
       // Suffix all subsequent records with `:duplicate` until it's unique

core/gather/driver/target-manager.js

@@ -94,6 +94,16 @@ class TargetManager extends ProtocolEventEmitter {
     throw new Error(`session ${sessionId} not found`);
   }
 
+  /**
+   * @param {string} targetType
+   * @return {targetType is LH.Protocol.TargetType}
+   */
+  _isAcceptedTargetType(targetType) {
+    return targetType === 'page' ||
+      targetType === 'iframe' ||
+      targetType === 'worker';
+  }
+
   /**
    * Returns the root session.
    * @return {LH.Gatherer.ProtocolSession}
@@ -116,19 +126,19 @@ class TargetManager extends ProtocolEventEmitter {
     const newSession = new ProtocolSession(cdpSession);
 
     try {
-      const target = await newSession.sendCommand('Target.getTargetInfo').catch(() => null);
-      const targetType = target?.targetInfo?.type;
-      const hasValidTargetType = targetType === 'page' || targetType === 'iframe';
+      const {targetInfo} = await newSession.sendCommand('Target.getTargetInfo');
+      const targetType = targetInfo.type;
+
       // TODO: should detach from target in this case?
       // See pptr: https://github.com/puppeteer/puppeteer/blob/733cbecf487c71483bee8350e37030edb24bc021/src/common/Page.ts#L495-L526
-      if (!target || !hasValidTargetType) return;
+      if (!this._isAcceptedTargetType(targetType)) return;
 
       // No need to continue if target has already been seen.
-      const targetId = target.targetInfo.targetId;
+      const targetId = targetInfo.targetId;
       if (this._targetIdToTargets.has(targetId)) return;
 
-      newSession.setTargetInfo(target.targetInfo);
-      const targetName = target.targetInfo.url || target.targetInfo.targetId;
+      newSession.setTargetInfo(targetInfo);
+      const targetName = targetInfo.url || targetInfo.targetId;
       log.verbose('target-manager', `target ${targetName} attached`);
 
       const trueProtocolListener = this._getProtocolEventListener(targetType, newSession.id());
@@ -139,7 +149,7 @@ class TargetManager extends ProtocolEventEmitter {
       cdpSession.on('sessionattached', this._onSessionAttached);
 
       const targetWithSession = {
-        target: target.targetInfo,
+        target: targetInfo,
         cdpSession,
         session: newSession,
         protocolListener,

core/gather/gatherers/dobetterweb/optimized-images.js

@@ -54,8 +54,8 @@ class OptimizedImages extends BaseGatherer {
     /** @type {Set<string>} */
     const seenUrls = new Set();
     return networkRecords.reduce((prev, record) => {
-      // Skip records that we've seen before, never finished, or came from OOPIFs.
-      if (seenUrls.has(record.url) || !record.finished || record.isOutOfProcessIframe) {
+      // Skip records that we've seen before, never finished, or came from OOPIFs/web workers.
+      if (seenUrls.has(record.url) || !record.finished || record.sessionTargetType !== 'page') {
         return prev;
       }
 

core/gather/gatherers/dobetterweb/response-compression.js

@@ -56,7 +56,7 @@ class ResponseCompression extends BaseGatherer {
     const unoptimizedResponses = [];
 
     networkRecords.forEach(record => {
-      if (record.isOutOfProcessIframe) return;
+      if (record.sessionTargetType !== 'page') return;
 
       const mimeType = record.mimeType;
       const resourceType = record.resourceType || NetworkRequest.TYPES.Other;

core/gather/gatherers/script-elements.js

@@ -65,7 +65,7 @@
 
     const scriptRecords = networkRecords
       .filter(record => record.resourceType === NetworkRequest.TYPES.Script)
-      .filter(record => !record.isOutOfProcessIframe);
+      .filter(record => record.sessionTargetType === 'page');
 
     for (let i = 0; i < scriptRecords.length; i++) {
       const record = scriptRecords[i];

core/test/computed/page-dependency-graph-test.js

@@ -19,10 +19,19 @@ function createRequest(
   url,
   networkRequestTime = 0,
   initiator = null,
-  resourceType = NetworkRequest.TYPES.Document
+  resourceType = NetworkRequest.TYPES.Document,
+  sessionTargetType = 'page'
 ) {
   const networkEndTime = networkRequestTime + 50;
-  return {requestId, url, networkRequestTime, networkEndTime, initiator, resourceType};
+  return {
+    requestId,
+    url,
+    networkRequestTime,
+    networkEndTime,
+    initiator,
+    resourceType,
+    sessionTargetType,
+  };
 }
 
 const TOPLEVEL_TASK_NAME = 'TaskQueueManager::ProcessTaskFromWorkQueue';
@@ -105,6 +114,19 @@ describe('PageDependencyGraph computed artifact:', () => {
       }
     });
 
+    it('should ignore worker requests', () => {
+      const workerRequest = createRequest(4, 'https://example.com/worker.js', 0, null, 'Script', 'worker');
+      const recordsWithWorker = [
+        ...networkRecords,
+        workerRequest,
+      ];
+
+      const networkNodeOutput = PageDependencyGraph.getNetworkNodeOutput(recordsWithWorker);
+
+      expect(networkNodeOutput.nodes).toHaveLength(3);
+      expect(networkNodeOutput.nodes.map(node => node.record)).not.toContain(workerRequest);
+    });
+
     it('should index nodes by ID', () => {
       const networkNodeOutput = PageDependencyGraph.getNetworkNodeOutput(networkRecords);
       const indexedById = networkNodeOutput.idToNodeMap;

core/test/gather/driver/target-manager-test.js

@@ -108,8 +108,8 @@ describe('TargetManager', () => {
       expect(sendMock.findAllInvocations('Runtime.runIfWaitingForDebugger')).toHaveLength(4);
     });
 
-    it('should ignore non-frame targets', async () => {
-      targetInfo.type = 'worker';
+    it('should ignore targets that are not frames or web workers', async () => {
+      targetInfo.type = 'service_worker';
       sendMock
         .mockResponse('Target.getTargetInfo', {targetInfo})
         .mockResponse('Target.setAutoAttach');
@@ -179,7 +179,8 @@ describe('TargetManager', () => {
     });
 
     it('should resume the target when finished', async () => {
-      sendMock.mockResponse('Target.getTargetInfo', {});
+      targetInfo.type = 'service_worker';
+      sendMock.mockResponse('Target.getTargetInfo', {targetInfo});
       await targetManager.enable();
 
       const invocations = sendMock.findAllInvocations('Runtime.runIfWaitingForDebugger');

core/test/gather/gatherers/dobetterweb/optimized-images-test.js

@@ -112,6 +112,16 @@ describe('Optimized images', () => {
         transferSize: 20000,
         finished: true,
       },
+      {
+        requestId: '11',
+        url: 'http://gmail.com/image-worker.jpg',
+        mimeType: 'image/jpeg',
+        resourceSize: 15000,
+        transferSize: 20000,
+        resourceType: 'Image',
+        finished: true,
+        sessionTargetType: 'worker', // ignore for being a worker
+      },
     ]);
   });
 

core/test/gather/gatherers/dobetterweb/response-compression-test.js

@@ -30,6 +30,7 @@ const networkRecords = [
     }],
     content: 'aaabbbccc',
     finished: true,
+    sessionTargetType: 'page',
   },
   {
     url: 'http://google.com/index.css',
@@ -42,6 +43,7 @@ const networkRecords = [
     responseHeaders: [],
     content: 'abcabc',
     finished: true,
+    sessionTargetType: 'page',
   },
   {
     url: 'http://google.com/index.json',
@@ -54,6 +56,7 @@ const networkRecords = [
     responseHeaders: [],
     content: '1234567',
     finished: true,
+    sessionTargetType: 'page',
   },
   {
     url: 'http://google.com/index-oopif.json',
@@ -79,6 +82,7 @@ const networkRecords = [
     responseHeaders: [],
     content: '1234567',
     finished: true,
+    sessionTargetType: 'page',
   },
   {
     url: 'http://google.com/other.json',
@@ -91,6 +95,7 @@ const networkRecords = [
     responseHeaders: [],
     content: '1234567',
     finished: false, // ignore for not finishing
+    sessionTargetType: 'page',
   },
   {
     url: 'http://google.com/index.jpg',
@@ -103,6 +108,7 @@ const networkRecords = [
     responseHeaders: [],
     content: 'aaaaaaaaaa',
     finished: true,
+    sessionTargetType: 'page',
   },
   {
     url: 'http://google.com/helloworld.mp4',
@@ -115,6 +121,20 @@ const networkRecords = [
     responseHeaders: [],
     content: 'bbbbbbbb',
     finished: true,
+    sessionTargetType: 'page',
+  },
+  {
+    url: 'http://google.com/index-worker.json',
+    statusCode: 200,
+    mimeType: 'application/json',
+    requestId: 28,
+    resourceSize: 7,
+    transferSize: 8,
+    resourceType: 'XHR',
+    responseHeaders: [],
+    content: '1234567',
+    finished: true,
+    sessionTargetType: 'worker', // ignore for being from a worker
   },
 ].map((record) => Object.assign(new NetworkRequest(), record));
 
@@ -172,6 +192,7 @@ describe('Optimized responses', () => {
         responseHeaders: [],
         content: 'aaaaaaaaaa',
         finished: true,
+        sessionTargetType: 'page',
       },
       {
         url: 'http://google.com/chrome-extension.css',
@@ -183,6 +204,7 @@ describe('Optimized responses', () => {
         responseHeaders: [],
         content: 'aaaaaaaaaa',
         finished: true,
+        sessionTargetType: 'page',
       },
     ];
 

core/test/gather/gatherers/script-elements-test.js

@@ -21,6 +21,7 @@ const {NetworkRequest} = await import('../../../lib/network-request.js');
 function mockRecord(partial) {
   const request = new NetworkRequest();
   request.resourceType = NetworkRequest.TYPES.Script;
+  request.sessionTargetType = 'page';
   return Object.assign(request, partial);
 }
 
@@ -80,11 +81,12 @@ describe('_getArtifact', () => {
     ]);
   });
 
-  it('ignore OOPIF records', async () => {
+  it('ignore OOPIF and worker records', async () => {
     networkRecords = [
       mainDocument,
       mockRecord({url: 'https://example.com/script.js', requestId: '1'}),
       mockRecord({url: 'https://oopif.com/script.js', requestId: '2', sessionTargetType: 'iframe'}),
+      mockRecord({url: 'https://oopif.com/worker.js', requestId: '2', sessionTargetType: 'worker'}),
     ];
     // OOPIF would not produce script element
     scriptElements = [

core/test/scenarios/api-test-pptr.js

@@ -180,27 +180,52 @@ describe('Individual modes API', function() {
         .map((r) => ({url: r.url, sessionTargetType: r.sessionTargetType}))
         // @ts-expect-error
         .sort((a, b) => a.url.localeCompare(b.url));
-      expect(networkRequests).toHaveLength(4);
+      expect(networkRequests).toHaveLength(10);
       expect(networkRequests.filter(r => r.sessionTargetType === 'page')).toHaveLength(2);
       expect(networkRequests.filter(r => r.sessionTargetType === 'iframe')).toHaveLength(2);
+      expect(networkRequests.filter(r => r.sessionTargetType === 'worker')).toHaveLength(6);
       expect(networkRequests).toMatchInlineSnapshot(`
 Array [
   Object {
     "sessionTargetType": "page",
     "url": "http://localhost:10200/simple-script.js",
   },
+  Object {
+    "sessionTargetType": "worker",
+    "url": "http://localhost:10200/simple-script.js?esm",
+  },
+  Object {
+    "sessionTargetType": "worker",
+    "url": "http://localhost:10200/simple-script.js?importScripts",
+  },
   Object {
     "sessionTargetType": "page",
     "url": "http://localhost:10200/simple-worker.js",
   },
+  Object {
+    "sessionTargetType": "worker",
+    "url": "http://localhost:10200/simple-worker.mjs",
+  },
   Object {
     "sessionTargetType": "iframe",
     "url": "http://localhost:10503/simple-script.js",
   },
+  Object {
+    "sessionTargetType": "worker",
+    "url": "http://localhost:10503/simple-script.js?esm",
+  },
+  Object {
+    "sessionTargetType": "worker",
+    "url": "http://localhost:10503/simple-script.js?importScripts",
+  },
   Object {
     "sessionTargetType": "iframe",
     "url": "http://localhost:10503/simple-worker.js",
   },
+  Object {
+    "sessionTargetType": "worker",
+    "url": "http://localhost:10503/simple-worker.mjs",
+  },
 ]
 `);