test_runner: add support for coverage via run()

lib/internal/test_runner/runner.js

@@ -54,6 +54,7 @@ const {
   validateObject,
   validateOneOf,
   validateInteger,
+  validateStringArray,
 } = require('internal/validators');
 const { getInspectPort, isUsingInspector, isInspectorMessage } = require('internal/util/inspector');
 const { isRegExp } = require('internal/util/types');
@@ -510,7 +511,16 @@ function watchFiles(testFiles, opts) {
 function run(options = kEmptyObject) {
   validateObject(options, 'options');
 
-  let { testNamePatterns, testSkipPatterns, shard } = options;
+  let {
+    testNamePatterns,
+    testSkipPatterns,
+    shard,
+    coverageExcludeGlobs,
+    coverageIncludeGlobs,
+    lineCoverage,
+    branchCoverage,
+    functionCoverage,
+  } = options;
   const {
     concurrency,
     timeout,
@@ -523,6 +533,7 @@ function run(options = kEmptyObject) {
     setup,
     only,
     globPatterns,
+    coverage,
   } = options;
 
   if (files != null) {
@@ -601,6 +612,65 @@ function run(options = kEmptyObject) {
     });
   }
   validateOneOf(isolation, 'options.isolation', ['process', 'none']);
+  if (coverage != null) {
+    validateBoolean(coverage, 'options.coverage');
+  }
+  if (coverageExcludeGlobs != null) {
+    if (!coverage) {
+      throw new ERR_INVALID_ARG_VALUE(
+        'options.coverageExcludeGlobs',
+        coverageExcludeGlobs,
+        'is only supported when coverage is enabled',
+      );
+    }
+    if (!ArrayIsArray(coverageExcludeGlobs)) {
+      coverageExcludeGlobs = [coverageExcludeGlobs];
+    }
+    validateStringArray(coverageExcludeGlobs, 'options.coverageExcludeGlobs');
+  }
+  if (coverageIncludeGlobs != null) {
+    if (!coverage) {
+      throw new ERR_INVALID_ARG_VALUE(
+        'options.coverageIncludeGlobs',
+        coverageIncludeGlobs,
+        'is only supported when coverage is enabled',
+      );
+    }
+    if (!ArrayIsArray(coverageIncludeGlobs)) {
+      coverageIncludeGlobs = [coverageIncludeGlobs];
+    }
+    validateStringArray(coverageIncludeGlobs, 'options.coverageIncludeGlobs');
+  }
+  if (lineCoverage != null) {
+    if (!coverage) {
+      throw new ERR_INVALID_ARG_VALUE(
+        'options.lineCoverage',
+        lineCoverage,
+        'is only supported when coverage is enabled',
+      );
+    }
+    validateInteger(lineCoverage, 'options.lineCoverage', 0, 100);
+  }
+  if (branchCoverage != null) {
+    if (!coverage) {
+      throw new ERR_INVALID_ARG_VALUE(
+        'options.branchCoverage',
+        branchCoverage,
+        'is only supported when coverage is enabled',
+      );
+    }
+    validateInteger(branchCoverage, 'options.branchCoverage', 0, 100);
+  }
+  if (functionCoverage != null) {
+    if (!coverage) {
+      throw new ERR_INVALID_ARG_VALUE(
+        'options.functionCoverage',
+        functionCoverage,
+        'is only supported when coverage is enabled',
+      );
+    }
+    validateInteger(functionCoverage, 'options.functionCoverage', 0, 100);
+  }
 
   const rootTestOptions = { __proto__: null, concurrency, timeout, signal };
   const globalOptions = {
@@ -609,6 +679,12 @@ function run(options = kEmptyObject) {
     // behavior has relied on it, so removing it must be done in a semver major.
     ...parseCommandLine(),
     setup,  // This line can be removed when parseCommandLine() is removed here.
+    coverage,
+    coverageExcludeGlobs,
+    coverageIncludeGlobs,
+    lineCoverage,
+    branchCoverage,
+    functionCoverage,
   };
   const root = createTestTree(rootTestOptions, globalOptions);
   let testFiles = files ?? createTestFileList(globPatterns);

test/parallel/test-runner-run-coverage.mjs

@@ -0,0 +1,159 @@
+import * as common from '../common/index.mjs';
+import * as fixtures from '../common/fixtures.mjs';
+import { describe, it, run } from 'node:test';
+import assert from 'node:assert';
+
+const files = [fixtures.path('test-runner', 'coverage.js')];
+const skipIfNoInspector = {
+  skip: !process.features.inspector ? 'inspector disabled' : false
+};
+
+describe('require(\'node:test\').run Coverage settings', { concurrency: true }, () => {
+  describe('validation', () => {
+    it('should only allow boolean in options.coverage', async () => {
+      [Symbol(), {}, () => {}, 0, 1, 0n, 1n, '', '1', Promise.resolve(true), []]
+        .forEach((coverage) => assert.throws(() => run({ coverage }), {
+          code: 'ERR_INVALID_ARG_TYPE'
+        }));
+    });
+
+    it('should only allow coverage options when coverage is true', async () => {
+      assert.throws(
+        () => run({ coverage: false, coverageIncludeGlobs: [] }),
+        { code: 'ERR_INVALID_ARG_VALUE' },
+      );
+      assert.throws(
+        () => run({ coverage: false, coverageExcludeGlobs: [] }),
+        { code: 'ERR_INVALID_ARG_VALUE' },
+      );
+      assert.throws(
+        () => run({ coverage: false, lineCoverage: 0 }),
+        { code: 'ERR_INVALID_ARG_VALUE' },
+      );
+      assert.throws(
+        () => run({ coverage: false, branchCoverage: 0 }),
+        { code: 'ERR_INVALID_ARG_VALUE' },
+      );
+      assert.throws(
+        () => run({ coverage: false, functionCoverage: 0 }),
+        { code: 'ERR_INVALID_ARG_VALUE' },
+      );
+    });
+
+    it('should only allow string|string[] in options.coverageExcludeGlobs', async () => {
+      [Symbol(), {}, () => {}, 0, 1, 0n, 1n, Promise.resolve([]), true, false]
+        .forEach((coverageExcludeGlobs) => {
+          assert.throws(() => run({ coverage: true, coverageExcludeGlobs }), {
+            code: 'ERR_INVALID_ARG_TYPE'
+          });
+          assert.throws(() => run({ coverage: true, coverageExcludeGlobs: [coverageExcludeGlobs] }), {
+            code: 'ERR_INVALID_ARG_TYPE'
+          });
+        });
+      run({ files: [], signal: AbortSignal.abort(), coverage: true, coverageExcludeGlobs: [''] });
+      run({ files: [], signal: AbortSignal.abort(), coverage: true, coverageExcludeGlobs: '' });
+    });
+
+    it('should only allow string|string[] in options.coverageIncludeGlobs', async () => {
+      [Symbol(), {}, () => {}, 0, 1, 0n, 1n, Promise.resolve([]), true, false]
+        .forEach((coverageIncludeGlobs) => {
+          assert.throws(() => run({ coverage: true, coverageIncludeGlobs }), {
+            code: 'ERR_INVALID_ARG_TYPE'
+          });
+          assert.throws(() => run({ coverage: true, coverageIncludeGlobs: [coverageIncludeGlobs] }), {
+            code: 'ERR_INVALID_ARG_TYPE'
+          });
+        });
+
+      run({ files: [], signal: AbortSignal.abort(), coverage: true, coverageIncludeGlobs: [''] });
+      run({ files: [], signal: AbortSignal.abort(), coverage: true, coverageIncludeGlobs: '' });
+    });
+
+    it('should only allow an int in options.lineCoverage', async () => {
+      [Symbol(), {}, () => {}, [], 0n, 1n, Promise.resolve([]), true, false]
+        .forEach((lineCoverage) => {
+          assert.throws(() => run({ coverage: true, lineCoverage }), {
+            code: 'ERR_INVALID_ARG_TYPE'
+          });
+          assert.throws(() => run({ coverage: true, lineCoverage: [lineCoverage] }), {
+            code: 'ERR_INVALID_ARG_TYPE'
+          });
+        });
+
+      run({ files: [], signal: AbortSignal.abort(), coverage: true, lineCoverage: 0 });
+    });
+
+    it('should only allow an int in options.branchCoverage', async () => {
+      [Symbol(), {}, () => {}, [], 0n, 1n, Promise.resolve([]), true, false]
+        .forEach((branchCoverage) => {
+          assert.throws(() => run({ coverage: true, branchCoverage }), {
+            code: 'ERR_INVALID_ARG_TYPE'
+          });
+          assert.throws(() => run({ coverage: true, branchCoverage: [branchCoverage] }), {
+            code: 'ERR_INVALID_ARG_TYPE'
+          });
+        });
+
+      run({ files: [], signal: AbortSignal.abort(), coverage: true, branchCoverage: 0 });
+    });
+
+    it('should only allow an int in options.functionCoverage', async () => {
+      [Symbol(), {}, () => {}, [], 0n, 1n, Promise.resolve([]), true, false]
+        .forEach((functionCoverage) => {
+          assert.throws(() => run({ coverage: true, functionCoverage }), {
+            code: 'ERR_INVALID_ARG_TYPE'
+          });
+          assert.throws(() => run({ coverage: true, functionCoverage: [functionCoverage] }), {
+            code: 'ERR_INVALID_ARG_TYPE'
+          });
+        });
+
+      run({ files: [], signal: AbortSignal.abort(), coverage: true, functionCoverage: 0 });
+    });
+  });
+
+  describe('run with coverage', skipIfNoInspector, () => {
+    it('should run with coverage', async () => {
+      const stream = run({ files, coverage: true });
+      stream.on('test:fail', common.mustNotCall());
+      stream.on('test:pass', common.mustCall());
+      stream.on('test:coverage', common.mustCall());
+      // eslint-disable-next-line no-unused-vars
+      for await (const _ of stream);
+    });
+
+    it('should run with coverage and exclude by glob', async () => {
+      const stream = run({ files, coverage: true, coverageExcludePatterns: ['test/*/test-runner/invalid-tap.js'] });
+      stream.on('test:fail', common.mustNotCall());
+      stream.on('test:pass', common.mustCall(1));
+      stream.on('test:coverage', common.mustCall(({ summary: { files } }) => {
+        const filesPaths = files.map(({ path }) => path);
+        assert.strictEqual(filesPaths.some((path) => path.includes('test-runner/invalid-tap.js')), false);
+      }));
+      // eslint-disable-next-line no-unused-vars
+      for await (const _ of stream);
+    });
+
+    it('should run with coverage and include by glob', async () => {
+      const stream = run({ files, coverage: true, coverageIncludePatterns: ['test/*/test-runner/invalid-tap.js'] });
+      stream.on('test:fail', common.mustNotCall());
+      stream.on('test:pass', common.mustCall(1));
+      stream.on('test:coverage', common.mustCall(({ summary: { files } }) => {
+        const filesPaths = files.map(({ path }) => path);
+        assert.strictEqual(filesPaths.some((path) => path.includes('test-runner/invalid-tap.js')), true);
+      }));
+      // eslint-disable-next-line no-unused-vars
+      for await (const _ of stream);
+    });
+  });
+});
+
+
+// exitHandler doesn't run until after the tests / after hooks finish.
+process.on('exit', () => {
+  assert.strictEqual(process.listeners('uncaughtException').length, 0);
+  assert.strictEqual(process.listeners('unhandledRejection').length, 0);
+  assert.strictEqual(process.listeners('beforeExit').length, 0);
+  assert.strictEqual(process.listeners('SIGINT').length, 0);
+  assert.strictEqual(process.listeners('SIGTERM').length, 0);
+});