From 01b951557ef18a10471db64689f51310a234643e Mon Sep 17 00:00:00 2001 From: Moshe Atlow Date: Sat, 18 Feb 2023 18:19:24 +0200 Subject: [PATCH] test_runner: flatten TAP output when running using `--test` PR-URL: https://github.com/nodejs/node/pull/46440 Fixes: https://github.com/nodejs/node/issues/45833 Refs: https://github.com/nodejs/node/issues/45833 Reviewed-By: Colin Ihrig Reviewed-By: Benjamin Gruenbaum --- lib/internal/test_runner/runner.js | 89 +- lib/internal/test_runner/test.js | 64 +- test/message/test_runner_abort.out | 10 + test/message/test_runner_abort_suite.out | 2 + test/message/test_runner_output_cli.out | 1124 ++++++++--------- test/parallel/test-runner-cli.js | 102 +- .../test-runner-extraneous-async-activity.js | 10 +- test/parallel/test-runner-reporters.js | 16 +- test/parallel/test-runner-run.mjs | 4 +- 9 files changed, 724 insertions(+), 697 deletions(-) diff --git a/lib/internal/test_runner/runner.js b/lib/internal/test_runner/runner.js index d2b5335706e6e5..d73b14868be32a 100644 --- a/lib/internal/test_runner/runner.js +++ b/lib/internal/test_runner/runner.js @@ -8,9 +8,11 @@ const { ArrayPrototypeSlice, ArrayPrototypeSome, ArrayPrototypeSort, + FunctionPrototypeCall, + Number, ObjectAssign, + ObjectKeys, PromisePrototypeThen, - SafePromiseAll, SafePromiseAllReturnVoid, SafePromiseAllSettledReturnVoid, SafeMap, @@ -35,7 +37,14 @@ const { validateArray, validateBoolean } = require('internal/validators'); const { getInspectPort, isUsingInspector, isInspectorMessage } = require('internal/util/inspector'); const { kEmptyObject } = require('internal/util'); const { createTestTree } = require('internal/test_runner/harness'); -const { kSubtestsFailed, Test } = require('internal/test_runner/test'); +const { + kAborted, + kCancelledByParent, + kSubtestsFailed, + kTestCodeFailure, + kTestTimeoutFailure, + Test, +} = require('internal/test_runner/test'); const { TapParser } = require('internal/test_runner/tap_parser'); const { YAMLToJs } = require('internal/test_runner/yaml_to_js'); const { TokenKind } = require('internal/test_runner/tap_lexer'); @@ -55,6 +64,9 @@ const kFilterArgs = ['--test', '--experimental-test-coverage', '--watch']; const kFilterArgValues = ['--test-reporter', '--test-reporter-destination']; const kDiagnosticsFilterArgs = ['tests', 'pass', 'fail', 'cancelled', 'skipped', 'todo', 'duration_ms']; +const kCanceledTests = new SafeSet() + .add(kCancelledByParent).add(kAborted).add(kTestTimeoutFailure); + // TODO(cjihrig): Replace this with recursive readdir once it lands. function processPath(path, testFiles, options) { const stats = statSync(path); @@ -133,6 +145,11 @@ function getRunArgs({ path, inspectPort }) { class FileTest extends Test { #buffer = []; + #counters = { __proto__: null, all: 0, failed: 0, passed: 0, cancelled: 0, skipped: 0, todo: 0, totalFailed: 0 }; + failedSubtests = false; + #skipReporting() { + return this.#counters.all > 0 && (!this.error || this.error.failureType === kSubtestsFailed); + } #checkNestedComment({ comment }) { const firstSpaceIndex = StringPrototypeIndexOf(comment, ' '); if (firstSpaceIndex === -1) return false; @@ -141,8 +158,6 @@ class FileTest extends Test { ArrayPrototypeIncludes(kDiagnosticsFilterArgs, StringPrototypeSlice(comment, 0, firstSpaceIndex)); } #handleReportItem({ kind, node, comments, nesting = 0 }) { - nesting += 1; - if (comments) { ArrayPrototypeForEach(comments, (comment) => this.reporter.diagnostic(nesting, this.name, comment)); } @@ -153,6 +168,9 @@ class FileTest extends Test { break; case TokenKind.TAP_PLAN: + if (nesting === 0 && this.#skipReporting()) { + break; + } this.reporter.plan(nesting, this.name, node.end - node.start + 1); break; @@ -160,10 +178,10 @@ class FileTest extends Test { this.reporter.start(nesting, this.name, node.name); break; - case TokenKind.TAP_TEST_POINT: - // eslint-disable-next-line no-case-declarations + case TokenKind.TAP_TEST_POINT: { + const { todo, skip, pass } = node.status; - // eslint-disable-next-line no-case-declarations + let directive; if (skip) { @@ -174,29 +192,22 @@ class FileTest extends Test { directive = kEmptyObject; } - if (pass) { - this.reporter.ok( - nesting, - this.name, - node.id, - node.description, - YAMLToJs(node.diagnostics), - directive, - ); - } else { - this.reporter.fail( - nesting, - this.name, - node.id, - node.description, - YAMLToJs(node.diagnostics), - directive, - ); + const diagnostics = YAMLToJs(node.diagnostics); + const cancelled = kCanceledTests.has(diagnostics.error?.failureType); + const testNumber = nesting === 0 ? (Number(node.id) + this.testNumber - 1) : node.id; + const method = pass ? 'ok' : 'fail'; + this.reporter[method](nesting, this.name, testNumber, node.description, diagnostics, directive); + if (nesting === 0) { + FunctionPrototypeCall(super.countSubtest, + { finished: true, skipped: skip, isTodo: todo, passed: pass, cancelled }, + this.#counters); + this.failedSubtests ||= !pass; } break; + } case TokenKind.COMMENT: - if (nesting === 1 && this.#checkNestedComment(node)) { + if (nesting === 0 && this.#checkNestedComment(node)) { // Ignore file top level diagnostics break; } @@ -216,10 +227,24 @@ class FileTest extends Test { this.reportStarted(); this.#handleReportItem(ast); } + countSubtest(counters) { + if (this.#counters.all === 0) { + return super.countSubtest(counters); + } + ArrayPrototypeForEach(ObjectKeys(counters), (key) => { + counters[key] += this.#counters[key]; + }); + } + reportStarted() {} report() { - this.reportStarted(); + const skipReporting = this.#skipReporting(); + if (!skipReporting) { + super.reportStarted(); + } ArrayPrototypeForEach(this.#buffer, (ast) => this.#handleReportItem(ast)); - super.report(); + if (!skipReporting) { + super.report(); + } } } @@ -274,16 +299,14 @@ function runTestFile(path, root, inspectPort, filesWatcher) { subtest.addToReport(ast); }); - const { 0: { 0: code, 1: signal } } = await SafePromiseAll([ - once(child, 'exit', { signal: t.signal }), - child.stdout.toArray({ signal: t.signal }), - ]); + const { 0: code, 1: signal } = await once(child, 'exit', { signal: t.signal }); runningProcesses.delete(path); runningSubtests.delete(path); if (code !== 0 || signal !== null) { if (!err) { - err = ObjectAssign(new ERR_TEST_FAILURE('test failed', kSubtestsFailed), { + const failureType = subtest.failedSubtests ? kSubtestsFailed : kTestCodeFailure; + err = ObjectAssign(new ERR_TEST_FAILURE('test failed', failureType), { __proto__: null, exitCode: code, signal: signal, diff --git a/lib/internal/test_runner/test.js b/lib/internal/test_runner/test.js index 5bbbf4b19e9e8c..9fb918f96b828e 100644 --- a/lib/internal/test_runner/test.js +++ b/lib/internal/test_runner/test.js @@ -57,6 +57,7 @@ const { availableParallelism } = require('os'); const { bigint: hrtime } = process.hrtime; const kCallbackAndPromisePresent = 'callbackAndPromisePresent'; const kCancelledByParent = 'cancelledByParent'; +const kAborted = 'testAborted'; const kParentAlreadyFinished = 'parentAlreadyFinished'; const kSubtestsFailed = 'subtestsFailed'; const kTestCodeFailure = 'testCodeFailure'; @@ -390,10 +391,12 @@ class Test extends AsyncResource { } #abortHandler = () => { - this.cancel(this.#outerSignal?.reason || new AbortError('The test was aborted')); + const error = this.#outerSignal?.reason || new AbortError('The test was aborted'); + error.failureType = kAborted; + this.#cancel(error); }; - cancel(error) { + #cancel(error) { if (this.endTime !== null) { return; } @@ -470,7 +473,7 @@ class Test extends AsyncResource { return true; } if (this.#outerSignal?.aborted) { - this.cancel(this.#outerSignal.reason || new AbortError('The test was aborted')); + this.#abortHandler(); return true; } } @@ -563,7 +566,7 @@ class Test extends AsyncResource { try { await afterEach(); } catch { /* test is already failing, let's the error */ } if (isTestFailureError(err)) { if (err.failureType === kTestTimeoutFailure) { - this.cancel(err); + this.#cancel(err); } else { this.fail(err); } @@ -577,9 +580,31 @@ class Test extends AsyncResource { this.postRun(); } - postRun(pendingSubtestsError) { - const counters = { __proto__: null, failed: 0, passed: 0, cancelled: 0, skipped: 0, todo: 0, totalFailed: 0 }; + countSubtest(counters) { + // Check SKIP and TODO tests first, as those should not be counted as + // failures. + if (this.skipped) { + counters.skipped++; + } else if (this.isTodo) { + counters.todo++; + } else if (this.cancelled) { + counters.cancelled++; + } else if (!this.passed) { + counters.failed++; + } else { + counters.passed++; + } + + if (!this.passed) { + counters.totalFailed++; + } + counters.all++; + } + postRun(pendingSubtestsError) { + const counters = { + __proto__: null, all: 0, failed: 0, passed: 0, cancelled: 0, skipped: 0, todo: 0, totalFailed: 0, + }; // If the test was failed before it even started, then the end time will // be earlier than the start time. Correct that here. if (this.endTime < this.startTime) { @@ -594,27 +619,10 @@ class Test extends AsyncResource { const subtest = this.subtests[i]; if (!subtest.finished) { - subtest.cancel(pendingSubtestsError); + subtest.#cancel(pendingSubtestsError); subtest.postRun(pendingSubtestsError); } - - // Check SKIP and TODO tests first, as those should not be counted as - // failures. - if (subtest.skipped) { - counters.skipped++; - } else if (subtest.isTodo) { - counters.todo++; - } else if (subtest.cancelled) { - counters.cancelled++; - } else if (!subtest.passed) { - counters.failed++; - } else { - counters.passed++; - } - - if (!subtest.passed) { - counters.totalFailed++; - } + subtest.countSubtest(counters); } if ((this.passed || this.parent === null) && counters.totalFailed > 0) { @@ -634,13 +642,13 @@ class Test extends AsyncResource { this.parent.processPendingSubtests(); } else if (!this.reported) { this.reported = true; - this.reporter.plan(this.nesting, kFilename, this.subtests.length); + this.reporter.plan(this.nesting, kFilename, counters.all); for (let i = 0; i < this.diagnostics.length; i++) { this.reporter.diagnostic(this.nesting, kFilename, this.diagnostics[i]); } - this.reporter.diagnostic(this.nesting, kFilename, `tests ${this.subtests.length}`); + this.reporter.diagnostic(this.nesting, kFilename, `tests ${counters.all}`); this.reporter.diagnostic(this.nesting, kFilename, `pass ${counters.passed}`); this.reporter.diagnostic(this.nesting, kFilename, `fail ${counters.failed}`); this.reporter.diagnostic(this.nesting, kFilename, `cancelled ${counters.cancelled}`); @@ -825,6 +833,8 @@ module.exports = { kCancelledByParent, kSubtestsFailed, kTestCodeFailure, + kTestTimeoutFailure, + kAborted, kUnwrapErrors, Suite, Test, diff --git a/test/message/test_runner_abort.out b/test/message/test_runner_abort.out index 26f89a29dbb8f7..3f1a3c2b7703f1 100644 --- a/test/message/test_runner_abort.out +++ b/test/message/test_runner_abort.out @@ -40,6 +40,7 @@ TAP version 13 not ok 7 - not ok 3 --- duration_ms: * + failureType: 'testAborted' error: 'This operation was aborted' code: 20 stack: |- @@ -58,6 +59,7 @@ TAP version 13 not ok 8 - not ok 4 --- duration_ms: * + failureType: 'testAborted' error: 'This operation was aborted' code: 20 stack: |- @@ -76,6 +78,7 @@ TAP version 13 not ok 9 - not ok 5 --- duration_ms: * + failureType: 'testAborted' error: 'This operation was aborted' code: 20 stack: |- @@ -94,6 +97,7 @@ TAP version 13 not ok 1 - promise timeout signal --- duration_ms: * + failureType: 'testAborted' error: 'The operation was aborted due to timeout' code: 23 stack: |- @@ -106,6 +110,7 @@ not ok 1 - promise timeout signal not ok 2 - promise abort signal --- duration_ms: * + failureType: 'testAborted' error: 'This operation was aborted' code: 20 stack: |- @@ -160,6 +165,7 @@ not ok 2 - promise abort signal not ok 7 - not ok 3 --- duration_ms: * + failureType: 'testAborted' error: 'This operation was aborted' code: 20 stack: |- @@ -178,6 +184,7 @@ not ok 2 - promise abort signal not ok 8 - not ok 4 --- duration_ms: * + failureType: 'testAborted' error: 'This operation was aborted' code: 20 stack: |- @@ -196,6 +203,7 @@ not ok 2 - promise abort signal not ok 9 - not ok 5 --- duration_ms: * + failureType: 'testAborted' error: 'This operation was aborted' code: 20 stack: |- @@ -214,6 +222,7 @@ not ok 2 - promise abort signal not ok 3 - callback timeout signal --- duration_ms: * + failureType: 'testAborted' error: 'The operation was aborted due to timeout' code: 23 stack: |- @@ -226,6 +235,7 @@ not ok 3 - callback timeout signal not ok 4 - callback abort signal --- duration_ms: * + failureType: 'testAborted' error: 'This operation was aborted' code: 20 stack: |- diff --git a/test/message/test_runner_abort_suite.out b/test/message/test_runner_abort_suite.out index 38669978406b4f..4dc71da99a766f 100644 --- a/test/message/test_runner_abort_suite.out +++ b/test/message/test_runner_abort_suite.out @@ -64,6 +64,7 @@ TAP version 13 not ok 1 - describe timeout signal --- duration_ms: * + failureType: 'testAborted' error: 'The operation was aborted due to timeout' code: 23 stack: |- @@ -76,6 +77,7 @@ not ok 1 - describe timeout signal not ok 2 - describe abort signal --- duration_ms: * + failureType: 'testAborted' error: 'This operation was aborted' code: 20 stack: |- diff --git a/test/message/test_runner_output_cli.out b/test/message/test_runner_output_cli.out index cc9c07f4c5c409..a76b37a77ce187 100644 --- a/test/message/test_runner_output_cli.out +++ b/test/message/test_runner_output_cli.out @@ -1,37 +1,196 @@ TAP version 13 -# Subtest: *test_runner_output.js - # Subtest: sync pass todo - ok 1 - sync pass todo # TODO - --- - duration_ms: * - ... - # Subtest: sync pass todo with message - ok 2 - sync pass todo with message # TODO this is a passing todo - --- - duration_ms: * - ... - # Subtest: sync fail todo - not ok 3 - sync fail todo # TODO - --- - duration_ms: * - failureType: 'testCodeFailure' - error: 'thrown from sync fail todo' - code: 'ERR_TEST_FAILURE' - stack: |- - * - * - * - * - * - * - * - ... - # Subtest: sync fail todo with message - not ok 4 - sync fail todo with message # TODO this is a failing todo +# Subtest: sync pass todo +ok 1 - sync pass todo # TODO + --- + duration_ms: * + ... +# Subtest: sync pass todo with message +ok 2 - sync pass todo with message # TODO this is a passing todo + --- + duration_ms: * + ... +# Subtest: sync fail todo +not ok 3 - sync fail todo # TODO + --- + duration_ms: * + failureType: 'testCodeFailure' + error: 'thrown from sync fail todo' + code: 'ERR_TEST_FAILURE' + stack: |- + * + * + * + * + * + * + * + ... +# Subtest: sync fail todo with message +not ok 4 - sync fail todo with message # TODO this is a failing todo + --- + duration_ms: * + failureType: 'testCodeFailure' + error: 'thrown from sync fail todo with message' + code: 'ERR_TEST_FAILURE' + stack: |- + * + * + * + * + * + * + * + ... +# Subtest: sync skip pass +ok 5 - sync skip pass # SKIP + --- + duration_ms: * + ... +# Subtest: sync skip pass with message +ok 6 - sync skip pass with message # SKIP this is skipped + --- + duration_ms: * + ... +# Subtest: sync pass +ok 7 - sync pass + --- + duration_ms: * + ... +# this test should pass +# Subtest: sync throw fail +not ok 8 - sync throw fail + --- + duration_ms: * + failureType: 'testCodeFailure' + error: 'thrown from sync throw fail' + code: 'ERR_TEST_FAILURE' + stack: |- + * + * + * + * + * + * + * + ... +# Subtest: async skip pass +ok 9 - async skip pass # SKIP + --- + duration_ms: * + ... +# Subtest: async pass +ok 10 - async pass + --- + duration_ms: * + ... +# Subtest: async throw fail +not ok 11 - async throw fail + --- + duration_ms: * + failureType: 'testCodeFailure' + error: 'thrown from async throw fail' + code: 'ERR_TEST_FAILURE' + stack: |- + * + * + * + * + * + * + * + ... +# Subtest: async skip fail +not ok 12 - async skip fail # SKIP + --- + duration_ms: * + failureType: 'testCodeFailure' + error: 'thrown from async throw fail' + code: 'ERR_TEST_FAILURE' + stack: |- + * + * + * + * + * + * + * + ... +# Subtest: async assertion fail +not ok 13 - async assertion fail + --- + duration_ms: * + failureType: 'testCodeFailure' + error: |- + Expected values to be strictly equal: + + true !== false + + code: 'ERR_ASSERTION' + expected: false + actual: true + operator: 'strictEqual' + stack: |- + * + * + * + * + * + * + * + ... +# Subtest: resolve pass +ok 14 - resolve pass + --- + duration_ms: * + ... +# Subtest: reject fail +not ok 15 - reject fail + --- + duration_ms: * + failureType: 'testCodeFailure' + error: 'rejected from reject fail' + code: 'ERR_TEST_FAILURE' + stack: |- + * + * + * + * + * + * + * + ... +# Subtest: unhandled rejection - passes but warns +ok 16 - unhandled rejection - passes but warns + --- + duration_ms: * + ... +# Subtest: async unhandled rejection - passes but warns +ok 17 - async unhandled rejection - passes but warns + --- + duration_ms: * + ... +# Subtest: immediate throw - passes but warns +ok 18 - immediate throw - passes but warns + --- + duration_ms: * + ... +# Subtest: immediate reject - passes but warns +ok 19 - immediate reject - passes but warns + --- + duration_ms: * + ... +# Subtest: immediate resolve pass +ok 20 - immediate resolve pass + --- + duration_ms: * + ... +# Subtest: subtest sync throw fail + # Subtest: +sync throw fail + not ok 1 - +sync throw fail --- duration_ms: * failureType: 'testCodeFailure' - error: 'thrown from sync fail todo with message' + error: 'thrown from subtest sync throw fail' code: 'ERR_TEST_FAILURE' stack: |- * @@ -41,616 +200,447 @@ TAP version 13 * * * - ... - # Subtest: sync skip pass - ok 5 - sync skip pass # SKIP - --- - duration_ms: * - ... - # Subtest: sync skip pass with message - ok 6 - sync skip pass with message # SKIP this is skipped - --- - duration_ms: * - ... - # Subtest: sync pass - ok 7 - sync pass - --- - duration_ms: * - ... - # this test should pass - # Subtest: sync throw fail - not ok 8 - sync throw fail - --- - duration_ms: * - failureType: 'testCodeFailure' - error: 'thrown from sync throw fail' - code: 'ERR_TEST_FAILURE' - stack: |- - * - * * * * - * - * - ... - # Subtest: async skip pass - ok 9 - async skip pass # SKIP - --- - duration_ms: * - ... - # Subtest: async pass - ok 10 - async pass - --- - duration_ms: * ... - # Subtest: async throw fail - not ok 11 - async throw fail + # this subtest should make its parent test fail + 1..1 +not ok 21 - subtest sync throw fail + --- + duration_ms: * + failureType: 'subtestsFailed' + error: '1 subtest failed' + code: 'ERR_TEST_FAILURE' + ... +# Subtest: sync throw non-error fail +not ok 22 - sync throw non-error fail + --- + duration_ms: * + failureType: 'testCodeFailure' + error: 'Symbol(thrown symbol from sync throw non-error fail)' + code: 'ERR_TEST_FAILURE' + ... +# Subtest: level 0a + # Subtest: level 1a + ok 1 - level 1a --- duration_ms: * - failureType: 'testCodeFailure' - error: 'thrown from async throw fail' - code: 'ERR_TEST_FAILURE' - stack: |- - * - * - * - * - * - * - * ... - # Subtest: async skip fail - not ok 12 - async skip fail # SKIP + # Subtest: level 1b + ok 2 - level 1b --- duration_ms: * - failureType: 'testCodeFailure' - error: 'thrown from async throw fail' - code: 'ERR_TEST_FAILURE' - stack: |- - * - * - * - * - * - * - * ... - # Subtest: async assertion fail - not ok 13 - async assertion fail + # Subtest: level 1c + ok 3 - level 1c --- duration_ms: * - failureType: 'testCodeFailure' - error: |- - Expected values to be strictly equal: - - true !== false - - code: 'ERR_ASSERTION' - expected: false - actual: true - operator: 'strictEqual' - stack: |- - * - * - * - * - * - * - * ... - # Subtest: resolve pass - ok 14 - resolve pass + # Subtest: level 1d + ok 4 - level 1d --- duration_ms: * ... - # Subtest: reject fail - not ok 15 - reject fail + 1..4 +ok 23 - level 0a + --- + duration_ms: * + ... +# Subtest: top level + # Subtest: +long running + not ok 1 - +long running --- duration_ms: * - failureType: 'testCodeFailure' - error: 'rejected from reject fail' + failureType: 'cancelledByParent' + error: 'test did not finish before its parent and was cancelled' code: 'ERR_TEST_FAILURE' - stack: |- - * - * - * - * - * - * - * - ... - # Subtest: unhandled rejection - passes but warns - ok 16 - unhandled rejection - passes but warns - --- - duration_ms: * - ... - # Subtest: async unhandled rejection - passes but warns - ok 17 - async unhandled rejection - passes but warns - --- - duration_ms: * - ... - # Subtest: immediate throw - passes but warns - ok 18 - immediate throw - passes but warns - --- - duration_ms: * - ... - # Subtest: immediate reject - passes but warns - ok 19 - immediate reject - passes but warns - --- - duration_ms: * - ... - # Subtest: immediate resolve pass - ok 20 - immediate resolve pass - --- - duration_ms: * ... - # Subtest: subtest sync throw fail - # Subtest: +sync throw fail - not ok 1 - +sync throw fail + # Subtest: +short running + # Subtest: ++short running + ok 1 - ++short running --- duration_ms: * - failureType: 'testCodeFailure' - error: 'thrown from subtest sync throw fail' - code: 'ERR_TEST_FAILURE' - stack: |- - * - * - * - * - * - * - * - * - * - * ... - # this subtest should make its parent test fail 1..1 - not ok 21 - subtest sync throw fail - --- - duration_ms: * - failureType: 'subtestsFailed' - error: '1 subtest failed' - code: 'ERR_TEST_FAILURE' - ... - # Subtest: sync throw non-error fail - not ok 22 - sync throw non-error fail - --- - duration_ms: * - failureType: 'testCodeFailure' - error: 'Symbol(thrown symbol from sync throw non-error fail)' - code: 'ERR_TEST_FAILURE' - ... - # Subtest: level 0a - # Subtest: level 1a - ok 1 - level 1a - --- - duration_ms: * - ... - # Subtest: level 1b - ok 2 - level 1b - --- - duration_ms: * - ... - # Subtest: level 1c - ok 3 - level 1c - --- - duration_ms: * - ... - # Subtest: level 1d - ok 4 - level 1d - --- - duration_ms: * - ... - 1..4 - ok 23 - level 0a - --- - duration_ms: * - ... - # Subtest: top level - # Subtest: +long running - not ok 1 - +long running - --- - duration_ms: * - failureType: 'cancelledByParent' - error: 'test did not finish before its parent and was cancelled' - code: 'ERR_TEST_FAILURE' - ... - # Subtest: +short running - # Subtest: ++short running - ok 1 - ++short running - --- - duration_ms: * - ... - 1..1 - ok 2 - +short running - --- - duration_ms: * - ... - 1..2 - not ok 24 - top level - --- - duration_ms: * - failureType: 'subtestsFailed' - error: '1 subtest failed' - code: 'ERR_TEST_FAILURE' - ... - # Subtest: invalid subtest - pass but subtest fails - ok 25 - invalid subtest - pass but subtest fails - --- - duration_ms: * - ... - # Subtest: sync skip option - ok 26 - sync skip option # SKIP - --- - duration_ms: * - ... - # Subtest: sync skip option with message - ok 27 - sync skip option with message # SKIP this is skipped - --- - duration_ms: * - ... - # Subtest: sync skip option is false fail - not ok 28 - sync skip option is false fail - --- - duration_ms: * - failureType: 'testCodeFailure' - error: 'this should be executed' - code: 'ERR_TEST_FAILURE' - stack: |- - * - * - * - * - * - * - * - ... - # Subtest: - ok 29 - - --- - duration_ms: * - ... - # Subtest: functionOnly - ok 30 - functionOnly - --- - duration_ms: * - ... - # Subtest: - ok 31 - - --- - duration_ms: * - ... - # Subtest: test with only a name provided - ok 32 - test with only a name provided - --- - duration_ms: * - ... - # Subtest: - ok 33 - - --- - duration_ms: * - ... - # Subtest: - ok 34 - # SKIP - --- - duration_ms: * - ... - # Subtest: test with a name and options provided - ok 35 - test with a name and options provided # SKIP - --- - duration_ms: * - ... - # Subtest: functionAndOptions - ok 36 - functionAndOptions # SKIP - --- - duration_ms: * - ... - # Subtest: escaped description \\ \# \\\#\\ \\n \\t \\f \\v \\b \\r - ok 37 - escaped description \\ \# \\\#\\ \\n \\t \\f \\v \\b \\r + ok 2 - +short running --- duration_ms: * ... - # Subtest: escaped skip message - ok 38 - escaped skip message # SKIP \#skip + 1..2 +not ok 24 - top level + --- + duration_ms: * + failureType: 'subtestsFailed' + error: '1 subtest failed' + code: 'ERR_TEST_FAILURE' + ... +# Subtest: invalid subtest - pass but subtest fails +ok 25 - invalid subtest - pass but subtest fails + --- + duration_ms: * + ... +# Subtest: sync skip option +ok 26 - sync skip option # SKIP + --- + duration_ms: * + ... +# Subtest: sync skip option with message +ok 27 - sync skip option with message # SKIP this is skipped + --- + duration_ms: * + ... +# Subtest: sync skip option is false fail +not ok 28 - sync skip option is false fail + --- + duration_ms: * + failureType: 'testCodeFailure' + error: 'this should be executed' + code: 'ERR_TEST_FAILURE' + stack: |- + * + * + * + * + * + * + * + ... +# Subtest: +ok 29 - + --- + duration_ms: * + ... +# Subtest: functionOnly +ok 30 - functionOnly + --- + duration_ms: * + ... +# Subtest: +ok 31 - + --- + duration_ms: * + ... +# Subtest: test with only a name provided +ok 32 - test with only a name provided + --- + duration_ms: * + ... +# Subtest: +ok 33 - + --- + duration_ms: * + ... +# Subtest: +ok 34 - # SKIP + --- + duration_ms: * + ... +# Subtest: test with a name and options provided +ok 35 - test with a name and options provided # SKIP + --- + duration_ms: * + ... +# Subtest: functionAndOptions +ok 36 - functionAndOptions # SKIP + --- + duration_ms: * + ... +# Subtest: escaped description \\ \# \\\#\\ \\n \\t \\f \\v \\b \\r +ok 37 - escaped description \\ \# \\\#\\ \\n \\t \\f \\v \\b \\r + --- + duration_ms: * + ... +# Subtest: escaped skip message +ok 38 - escaped skip message # SKIP \#skip + --- + duration_ms: * + ... +# Subtest: escaped todo message +ok 39 - escaped todo message # TODO \#todo + --- + duration_ms: * + ... +# Subtest: escaped diagnostic +ok 40 - escaped diagnostic + --- + duration_ms: * + ... +# \#diagnostic +# Subtest: callback pass +ok 41 - callback pass + --- + duration_ms: * + ... +# Subtest: callback fail +not ok 42 - callback fail + --- + duration_ms: * + failureType: 'testCodeFailure' + error: 'callback failure' + code: 'ERR_TEST_FAILURE' + stack: |- + * + * + ... +# Subtest: sync t is this in test +ok 43 - sync t is this in test + --- + duration_ms: * + ... +# Subtest: async t is this in test +ok 44 - async t is this in test + --- + duration_ms: * + ... +# Subtest: callback t is this in test +ok 45 - callback t is this in test + --- + duration_ms: * + ... +# Subtest: callback also returns a Promise +not ok 46 - callback also returns a Promise + --- + duration_ms: * + failureType: 'callbackAndPromisePresent' + error: 'passed a callback but also returned a Promise' + code: 'ERR_TEST_FAILURE' + ... +# Subtest: callback throw +not ok 47 - callback throw + --- + duration_ms: * + failureType: 'testCodeFailure' + error: 'thrown from callback throw' + code: 'ERR_TEST_FAILURE' + stack: |- + * + * + * + * + * + * + * + ... +# Subtest: callback called twice +not ok 48 - callback called twice + --- + duration_ms: * + failureType: 'multipleCallbackInvocations' + error: 'callback invoked multiple times' + code: 'ERR_TEST_FAILURE' + stack: |- + * + * + ... +# Subtest: callback called twice in different ticks +ok 49 - callback called twice in different ticks + --- + duration_ms: * + ... +# Subtest: callback called twice in future tick +not ok 50 - callback called twice in future tick + --- + duration_ms: * + failureType: 'uncaughtException' + error: 'callback invoked multiple times' + code: 'ERR_TEST_FAILURE' + stack: |- + * + ... +# Subtest: callback async throw +not ok 51 - callback async throw + --- + duration_ms: * + failureType: 'uncaughtException' + error: 'thrown from callback async throw' + code: 'ERR_TEST_FAILURE' + stack: |- + * + * + ... +# Subtest: callback async throw after done +ok 52 - callback async throw after done + --- + duration_ms: * + ... +# Subtest: only is set but not in only mode + # Subtest: running subtest 1 + ok 1 - running subtest 1 --- duration_ms: * ... - # Subtest: escaped todo message - ok 39 - escaped todo message # TODO \#todo + # Subtest: running subtest 2 + ok 2 - running subtest 2 --- duration_ms: * ... - # Subtest: escaped diagnostic - ok 40 - escaped diagnostic + # Subtest: running subtest 3 + ok 3 - running subtest 3 --- duration_ms: * ... - # \#diagnostic - # Subtest: callback pass - ok 41 - callback pass + # Subtest: running subtest 4 + ok 4 - running subtest 4 --- duration_ms: * ... - # Subtest: callback fail - not ok 42 - callback fail + 1..4 +ok 53 - only is set but not in only mode + --- + duration_ms: * + ... +# Subtest: custom inspect symbol fail +not ok 54 - custom inspect symbol fail + --- + duration_ms: * + failureType: 'testCodeFailure' + error: 'customized' + code: 'ERR_TEST_FAILURE' + ... +# Subtest: custom inspect symbol that throws fail +not ok 55 - custom inspect symbol that throws fail + --- + duration_ms: * + failureType: 'testCodeFailure' + error: |- + { + foo: 1, + [Symbol(nodejs.util.inspect.custom)]: [Function: [nodejs.util.inspect.custom]] + } + code: 'ERR_TEST_FAILURE' + ... +# Subtest: subtest sync throw fails + # Subtest: sync throw fails at first + not ok 1 - sync throw fails at first --- duration_ms: * failureType: 'testCodeFailure' - error: 'callback failure' + error: 'thrown from subtest sync throw fails at first' code: 'ERR_TEST_FAILURE' stack: |- * * - ... - # Subtest: sync t is this in test - ok 43 - sync t is this in test - --- - duration_ms: * - ... - # Subtest: async t is this in test - ok 44 - async t is this in test - --- - duration_ms: * - ... - # Subtest: callback t is this in test - ok 45 - callback t is this in test - --- - duration_ms: * - ... - # Subtest: callback also returns a Promise - not ok 46 - callback also returns a Promise - --- - duration_ms: * - failureType: 'callbackAndPromisePresent' - error: 'passed a callback but also returned a Promise' - code: 'ERR_TEST_FAILURE' - ... - # Subtest: callback throw - not ok 47 - callback throw - --- - duration_ms: * - failureType: 'testCodeFailure' - error: 'thrown from callback throw' - code: 'ERR_TEST_FAILURE' - stack: |- - * * * * * * * - ... - # Subtest: callback called twice - not ok 48 - callback called twice - --- - duration_ms: * - failureType: 'multipleCallbackInvocations' - error: 'callback invoked multiple times' - code: 'ERR_TEST_FAILURE' - stack: |- * * ... - # Subtest: callback called twice in different ticks - ok 49 - callback called twice in different ticks - --- - duration_ms: * - ... - # Subtest: callback called twice in future tick - not ok 50 - callback called twice in future tick + # Subtest: sync throw fails at second + not ok 2 - sync throw fails at second --- duration_ms: * - failureType: 'uncaughtException' - error: 'callback invoked multiple times' + failureType: 'testCodeFailure' + error: 'thrown from subtest sync throw fails at second' code: 'ERR_TEST_FAILURE' stack: |- * - ... - # Subtest: callback async throw - not ok 51 - callback async throw - --- - duration_ms: * - failureType: 'uncaughtException' - error: 'thrown from callback async throw' - code: 'ERR_TEST_FAILURE' - stack: |- * * - ... - # Subtest: callback async throw after done - ok 52 - callback async throw after done - --- - duration_ms: * - ... - # Subtest: only is set but not in only mode - # Subtest: running subtest 1 - ok 1 - running subtest 1 - --- - duration_ms: * - ... - # Subtest: running subtest 2 - ok 2 - running subtest 2 - --- - duration_ms: * - ... - # Subtest: running subtest 3 - ok 3 - running subtest 3 - --- - duration_ms: * - ... - # Subtest: running subtest 4 - ok 4 - running subtest 4 - --- - duration_ms: * - ... - 1..4 - ok 53 - only is set but not in only mode - --- - duration_ms: * - ... - # Subtest: custom inspect symbol fail - not ok 54 - custom inspect symbol fail - --- - duration_ms: * - failureType: 'testCodeFailure' - error: 'customized' - code: 'ERR_TEST_FAILURE' - ... - # Subtest: custom inspect symbol that throws fail - not ok 55 - custom inspect symbol that throws fail - --- - duration_ms: * - failureType: 'testCodeFailure' - error: |- - { - foo: 1, - [Symbol(nodejs.util.inspect.custom)]: [Function: [nodejs.util.inspect.custom]] - } - code: 'ERR_TEST_FAILURE' - ... - # Subtest: subtest sync throw fails - # Subtest: sync throw fails at first - not ok 1 - sync throw fails at first - --- - duration_ms: * - failureType: 'testCodeFailure' - error: 'thrown from subtest sync throw fails at first' - code: 'ERR_TEST_FAILURE' - stack: |- - * - * - * - * - * - * - * - * - * - * - ... - # Subtest: sync throw fails at second - not ok 2 - sync throw fails at second - --- - duration_ms: * - failureType: 'testCodeFailure' - error: 'thrown from subtest sync throw fails at second' - code: 'ERR_TEST_FAILURE' - stack: |- - * - * - * - * - * - * - * - * - * - * - ... - 1..2 - not ok 56 - subtest sync throw fails - --- - duration_ms: * - failureType: 'subtestsFailed' - error: '2 subtests failed' - code: 'ERR_TEST_FAILURE' - ... - # Subtest: timed out async test - not ok 57 - timed out async test - --- - duration_ms: * - failureType: 'testTimeoutFailure' - error: 'test timed out after 5ms' - code: 'ERR_TEST_FAILURE' - ... - # Subtest: timed out callback test - not ok 58 - timed out callback test - --- - duration_ms: * - failureType: 'testTimeoutFailure' - error: 'test timed out after 5ms' - code: 'ERR_TEST_FAILURE' - ... - # Subtest: large timeout async test is ok - ok 59 - large timeout async test is ok - --- - duration_ms: * - ... - # Subtest: large timeout callback test is ok - ok 60 - large timeout callback test is ok - --- - duration_ms: * - ... - # Subtest: successful thenable - ok 61 - successful thenable - --- - duration_ms: * - ... - # Subtest: rejected thenable - not ok 62 - rejected thenable - --- - duration_ms: * - failureType: 'testCodeFailure' - error: 'custom error' - code: 'ERR_TEST_FAILURE' - ... - # Subtest: unfinished test with uncaughtException - not ok 63 - unfinished test with uncaughtException - --- - duration_ms: * - failureType: 'uncaughtException' - error: 'foo' - code: 'ERR_TEST_FAILURE' - stack: |- * * * - ... - # Subtest: unfinished test with unhandledRejection - not ok 64 - unfinished test with unhandledRejection - --- - duration_ms: * - failureType: 'unhandledRejection' - error: 'bar' - code: 'ERR_TEST_FAILURE' - stack: |- * * * - ... - # Subtest: invalid subtest fail - not ok 65 - invalid subtest fail - --- - duration_ms: * - failureType: 'parentAlreadyFinished' - error: 'test could not be started because its parent finished' - code: 'ERR_TEST_FAILURE' - stack: |- * ... - 1..65 - # Warning: Test "unhandled rejection - passes but warns" generated asynchronous activity after the test ended. This activity created the error "Error: rejected from unhandled rejection fail" and would have caused the test to fail, but instead triggered an unhandledRejection event. - # Warning: Test "async unhandled rejection - passes but warns" generated asynchronous activity after the test ended. This activity created the error "Error: rejected from async unhandled rejection fail" and would have caused the test to fail, but instead triggered an unhandledRejection event. - # Warning: Test "immediate throw - passes but warns" generated asynchronous activity after the test ended. This activity created the error "Error: thrown from immediate throw fail" and would have caused the test to fail, but instead triggered an uncaughtException event. - # Warning: Test "immediate reject - passes but warns" generated asynchronous activity after the test ended. This activity created the error "Error: rejected from immediate reject fail" and would have caused the test to fail, but instead triggered an unhandledRejection event. - # Warning: Test "callback called twice in different ticks" generated asynchronous activity after the test ended. This activity created the error "Error [ERR_TEST_FAILURE]: callback invoked multiple times" and would have caused the test to fail, but instead triggered an uncaughtException event. - # Warning: Test "callback async throw after done" generated asynchronous activity after the test ended. This activity created the error "Error: thrown from callback async throw after done" and would have caused the test to fail, but instead triggered an uncaughtException event. -not ok 1 - *test_runner_output.js + 1..2 +not ok 56 - subtest sync throw fails --- duration_ms: * failureType: 'subtestsFailed' - exitCode: 1 - error: 'test failed' + error: '2 subtests failed' + code: 'ERR_TEST_FAILURE' + ... +# Subtest: timed out async test +not ok 57 - timed out async test + --- + duration_ms: * + failureType: 'testTimeoutFailure' + error: 'test timed out after 5ms' + code: 'ERR_TEST_FAILURE' + ... +# Subtest: timed out callback test +not ok 58 - timed out callback test + --- + duration_ms: * + failureType: 'testTimeoutFailure' + error: 'test timed out after 5ms' + code: 'ERR_TEST_FAILURE' + ... +# Subtest: large timeout async test is ok +ok 59 - large timeout async test is ok + --- + duration_ms: * + ... +# Subtest: large timeout callback test is ok +ok 60 - large timeout callback test is ok + --- + duration_ms: * + ... +# Subtest: successful thenable +ok 61 - successful thenable + --- + duration_ms: * + ... +# Subtest: rejected thenable +not ok 62 - rejected thenable + --- + duration_ms: * + failureType: 'testCodeFailure' + error: 'custom error' + code: 'ERR_TEST_FAILURE' + ... +# Subtest: unfinished test with uncaughtException +not ok 63 - unfinished test with uncaughtException + --- + duration_ms: * + failureType: 'uncaughtException' + error: 'foo' + code: 'ERR_TEST_FAILURE' + stack: |- + * + * + * + ... +# Subtest: unfinished test with unhandledRejection +not ok 64 - unfinished test with unhandledRejection + --- + duration_ms: * + failureType: 'unhandledRejection' + error: 'bar' + code: 'ERR_TEST_FAILURE' + stack: |- + * + * + * + ... +# Subtest: invalid subtest fail +not ok 65 - invalid subtest fail + --- + duration_ms: * + failureType: 'parentAlreadyFinished' + error: 'test could not be started because its parent finished' code: 'ERR_TEST_FAILURE' + stack: |- + * ... -1..1 -# tests 1 -# pass 0 -# fail 1 -# cancelled 0 -# skipped 0 -# todo 0 +# Warning: Test "unhandled rejection - passes but warns" generated asynchronous activity after the test ended. This activity created the error "Error: rejected from unhandled rejection fail" and would have caused the test to fail, but instead triggered an unhandledRejection event. +# Warning: Test "async unhandled rejection - passes but warns" generated asynchronous activity after the test ended. This activity created the error "Error: rejected from async unhandled rejection fail" and would have caused the test to fail, but instead triggered an unhandledRejection event. +# Warning: Test "immediate throw - passes but warns" generated asynchronous activity after the test ended. This activity created the error "Error: thrown from immediate throw fail" and would have caused the test to fail, but instead triggered an uncaughtException event. +# Warning: Test "immediate reject - passes but warns" generated asynchronous activity after the test ended. This activity created the error "Error: rejected from immediate reject fail" and would have caused the test to fail, but instead triggered an unhandledRejection event. +# Warning: Test "callback called twice in different ticks" generated asynchronous activity after the test ended. This activity created the error "Error [ERR_TEST_FAILURE]: callback invoked multiple times" and would have caused the test to fail, but instead triggered an uncaughtException event. +# Warning: Test "callback async throw after done" generated asynchronous activity after the test ended. This activity created the error "Error: thrown from callback async throw after done" and would have caused the test to fail, but instead triggered an uncaughtException event. +1..65 +# tests 65 +# pass 27 +# fail 21 +# cancelled 2 +# skipped 10 +# todo 5 # duration_ms * diff --git a/test/parallel/test-runner-cli.js b/test/parallel/test-runner-cli.js index 7407e03c875d10..8cfceedfe6a53a 100644 --- a/test/parallel/test-runner-cli.js +++ b/test/parallel/test-runner-cli.js @@ -28,11 +28,10 @@ const testFixtures = fixtures.path('test-runner'); assert.strictEqual(child.signal, null); assert.strictEqual(child.stderr.toString(), ''); const stdout = child.stdout.toString(); - assert.match(stdout, /ok 1 - .+index\.test\.js/); - assert.match(stdout, /not ok 2 - .+random\.test\.mjs/); - assert.match(stdout, /not ok 1 - this should fail/); + assert.match(stdout, /ok 1 - this should pass/); + assert.match(stdout, /not ok 2 - this should fail/); assert.match(stdout, /ok 3 - .+subdir.+subdir_test\.js/); - assert.match(stdout, /ok 4 - .+random\.cjs/); + assert.match(stdout, /ok 4 - this should pass/); } { @@ -41,11 +40,10 @@ const testFixtures = fixtures.path('test-runner'); const child = spawnSync(process.execPath, args); const stdout = child.stdout.toString(); - assert.match(stdout, /ok 1 - .+index\.test\.js/); - assert.match(stdout, /not ok 2 - .+random\.test\.mjs/); - assert.match(stdout, /not ok 1 - this should fail/); + assert.match(stdout, /ok 1 - this should pass/); + assert.match(stdout, /not ok 2 - this should fail/); assert.match(stdout, /ok 3 - .+subdir.+subdir_test\.js/); - assert.match(stdout, /ok 4 - .+random\.cjs/); + assert.match(stdout, /ok 4 - this should pass/); assert.strictEqual(child.status, 1); assert.strictEqual(child.signal, null); assert.strictEqual(child.stderr.toString(), ''); @@ -61,10 +59,10 @@ const testFixtures = fixtures.path('test-runner'); assert.strictEqual(child.stderr.toString(), ''); const stdout = child.stdout.toString(); assert.match(stdout, /not ok 1 - .+index\.js/); - assert.match(stdout, /ok 2 - .+index\.test\.js/); - assert.match(stdout, /not ok 3 - .+random\.test\.mjs/); - assert.match(stdout, /not ok 1 - this should fail/); + assert.match(stdout, /ok 2 - this should pass/); + assert.match(stdout, /not ok 3 - this should fail/); assert.match(stdout, /ok 4 - .+subdir.+subdir_test\.js/); + assert.match(stdout, /ok 5 - this should pass/); } { @@ -89,11 +87,10 @@ const testFixtures = fixtures.path('test-runner'); assert.strictEqual(child.signal, null); assert.strictEqual(child.stderr.toString(), ''); const stdout = child.stdout.toString(); - assert.match(stdout, /ok 1 - .+index\.test\.js/); - assert.match(stdout, /not ok 2 - .+random\.test\.mjs/); - assert.match(stdout, /not ok 1 - this should fail/); + assert.match(stdout, /ok 1 - this should pass/); + assert.match(stdout, /not ok 2 - this should fail/); assert.match(stdout, /ok 3 - .+subdir.+subdir_test\.js/); - assert.match(stdout, /ok 4 - .+random\.cjs/); + assert.match(stdout, /ok 4 - this should pass/); } { @@ -131,42 +128,36 @@ const testFixtures = fixtures.path('test-runner'); assert.strictEqual(child.signal, null); assert.strictEqual(child.stderr.toString(), ''); const stdout = child.stdout.toString(); - assert.match(stdout, /# Subtest: .+index\.test\.js/); - assert.match(stdout, / {4}# Subtest: this should pass/); - assert.match(stdout, / {4}ok 1 - this should pass/); - assert.match(stdout, / {6}---/); - assert.match(stdout, / {6}duration_ms: .*/); - assert.match(stdout, / {6}\.\.\./); - assert.match(stdout, / {4}1\.\.1/); - - assert.match(stdout, /ok 1 - .+index\.test\.js/); + assert.match(stdout, /# Subtest: this should pass/); + assert.match(stdout, /ok 1 - this should pass/); + assert.match(stdout, / {2}---/); + assert.match(stdout, / {2}duration_ms: .*/); + assert.match(stdout, / {2}\.\.\./); assert.match(stdout, /# Subtest: .+invalid-tap\.js/); - assert.match(stdout, / {4}# invalid tap output/); + assert.match(stdout, /# invalid tap output/); assert.match(stdout, /ok 2 - .+invalid-tap\.js/); - assert.match(stdout, /# Subtest: .+nested\.js/); - assert.match(stdout, / {4}# Subtest: level 0a/); - assert.match(stdout, / {8}# Subtest: level 1a/); - assert.match(stdout, / {8}ok 1 - level 1a/); - assert.match(stdout, / {8}# Subtest: level 1b/); - assert.match(stdout, / {8}not ok 2 - level 1b/); - assert.match(stdout, / {10}code: 'ERR_TEST_FAILURE'/); - assert.match(stdout, / {10}stack: |-'/); - assert.match(stdout, / {12}TestContext\. .*/); - assert.match(stdout, / {8}# Subtest: level 1c/); - assert.match(stdout, / {8}ok 3 - level 1c # SKIP aaa/); - assert.match(stdout, / {8}# Subtest: level 1d/); - assert.match(stdout, / {8}ok 4 - level 1d/); - assert.match(stdout, / {4}not ok 1 - level 0a/); - assert.match(stdout, / {6}error: '1 subtest failed'/); - assert.match(stdout, / {4}# Subtest: level 0b/); - assert.match(stdout, / {4}not ok 2 - level 0b/); - assert.match(stdout, / {6}error: 'level 0b error'/); - assert.match(stdout, /not ok 3 - .+nested\.js/); - assert.match(stdout, /# tests 3/); + assert.match(stdout, /# Subtest: level 0a/); + assert.match(stdout, / {4}# Subtest: level 1a/); + assert.match(stdout, / {4}ok 1 - level 1a/); + assert.match(stdout, / {4}# Subtest: level 1b/); + assert.match(stdout, / {4}not ok 2 - level 1b/); + assert.match(stdout, / {6}code: 'ERR_TEST_FAILURE'/); + assert.match(stdout, / {6}stack: |-'/); + assert.match(stdout, / {8}TestContext\. .*/); + assert.match(stdout, / {4}# Subtest: level 1c/); + assert.match(stdout, / {4}ok 3 - level 1c # SKIP aaa/); + assert.match(stdout, / {4}# Subtest: level 1d/); + assert.match(stdout, / {4}ok 4 - level 1d/); + assert.match(stdout, /not ok 3 - level 0a/); + assert.match(stdout, / {2}error: '1 subtest failed'/); + assert.match(stdout, /# Subtest: level 0b/); + assert.match(stdout, /not ok 4 - level 0b/); + assert.match(stdout, / {2}error: 'level 0b error'/); + assert.match(stdout, /# tests 4/); assert.match(stdout, /# pass 2/); - assert.match(stdout, /# fail 1/); + assert.match(stdout, /# fail 2/); } { @@ -181,16 +172,15 @@ const testFixtures = fixtures.path('test-runner'); assert.strictEqual(child.signal, null); assert.strictEqual(child.stderr.toString(), ''); const stdout = child.stdout.toString(); - assert.match(stdout, /# Subtest: .+user-logs\.js/); - assert.match(stdout, / {4}# stderr 1/); - assert.match(stdout, / {4}# stderr 2/); - assert.match(stdout, / {4}# stdout 3/); - assert.match(stdout, / {4}# stderr 6/); - assert.match(stdout, / {4}# not ok 1 - fake test/); - assert.match(stdout, / {4}# stderr 5/); - assert.match(stdout, / {4}# stdout 4/); - assert.match(stdout, / {4}# Subtest: a test/); - assert.match(stdout, / {4}ok 1 - a test/); + assert.match(stdout, /# stderr 1/); + assert.match(stdout, /# stderr 2/); + assert.match(stdout, /# stdout 3/); + assert.match(stdout, /# stderr 6/); + assert.match(stdout, /# not ok 1 - fake test/); + assert.match(stdout, /# stderr 5/); + assert.match(stdout, /# stdout 4/); + assert.match(stdout, /# Subtest: a test/); + assert.match(stdout, /ok 1 - a test/); assert.match(stdout, /# tests 1/); assert.match(stdout, /# pass 1/); } diff --git a/test/parallel/test-runner-extraneous-async-activity.js b/test/parallel/test-runner-extraneous-async-activity.js index d0fbf5fd2b487c..bc4be25d5f974b 100644 --- a/test/parallel/test-runner-extraneous-async-activity.js +++ b/test/parallel/test-runner-extraneous-async-activity.js @@ -10,8 +10,9 @@ const { spawnSync } = require('child_process'); fixtures.path('test-runner', 'extraneous_set_immediate_async.mjs'), ]); const stdout = child.stdout.toString(); - assert.match(stdout, /^# pass 0$/m); - assert.match(stdout, /^# fail 1$/m); + assert.match(stdout, /^# Warning: Test "extraneous async activity test" generated asynchronous activity after the test ended/m); + assert.match(stdout, /^# pass 1/m); + assert.match(stdout, /^# fail 0$/m); assert.match(stdout, /^# cancelled 0$/m); assert.strictEqual(child.status, 1); assert.strictEqual(child.signal, null); @@ -23,8 +24,9 @@ const { spawnSync } = require('child_process'); fixtures.path('test-runner', 'extraneous_set_timeout_async.mjs'), ]); const stdout = child.stdout.toString(); - assert.match(stdout, /^# pass 0$/m); - assert.match(stdout, /^# fail 1$/m); + assert.match(stdout, /^# Warning: Test "extraneous async activity test" generated asynchronous activity after the test ended/m); + assert.match(stdout, /^# pass 1$/m); + assert.match(stdout, /^# fail 0$/m); assert.match(stdout, /^# cancelled 0$/m); assert.strictEqual(child.status, 1); assert.strictEqual(child.signal, null); diff --git a/test/parallel/test-runner-reporters.js b/test/parallel/test-runner-reporters.js index 671b6ac4432167..da4f7102b8c2ae 100644 --- a/test/parallel/test-runner-reporters.js +++ b/test/parallel/test-runner-reporters.js @@ -26,7 +26,7 @@ describe('node:test reporters', { concurrency: true }, () => { it('should default destination to stdout when passing a single reporter', async () => { const child = spawnSync(process.execPath, ['--test', '--test-reporter', 'dot', testFile]); assert.strictEqual(child.stderr.toString(), ''); - assert.strictEqual(child.stdout.toString(), '.XX.X\n'); + assert.strictEqual(child.stdout.toString(), '.XX.\n'); }); it('should throw when passing reporters without a destination', async () => { @@ -45,13 +45,13 @@ describe('node:test reporters', { concurrency: true }, () => { const child = spawnSync(process.execPath, ['--test', '--test-reporter', 'dot', '--test-reporter-destination', 'stdout', testFile]); assert.strictEqual(child.stderr.toString(), ''); - assert.strictEqual(child.stdout.toString(), '.XX.X\n'); + assert.strictEqual(child.stdout.toString(), '.XX.\n'); }); it('should support stderr as a destination', async () => { const child = spawnSync(process.execPath, ['--test', '--test-reporter', 'dot', '--test-reporter-destination', 'stderr', testFile]); - assert.strictEqual(child.stderr.toString(), '.XX.X\n'); + assert.strictEqual(child.stderr.toString(), '.XX.\n'); assert.strictEqual(child.stdout.toString(), ''); }); @@ -61,7 +61,7 @@ describe('node:test reporters', { concurrency: true }, () => { ['--test', '--test-reporter', 'dot', '--test-reporter-destination', file, testFile]); assert.strictEqual(child.stderr.toString(), ''); assert.strictEqual(child.stdout.toString(), ''); - assert.strictEqual(fs.readFileSync(file, 'utf8'), '.XX.X\n'); + assert.strictEqual(fs.readFileSync(file, 'utf8'), '.XX.\n'); }); it('should support multiple reporters', async () => { @@ -75,7 +75,7 @@ describe('node:test reporters', { concurrency: true }, () => { testFile]); assert.match(child.stdout.toString(), /TAP version 13/); assert.match(child.stdout.toString(), /# duration_ms/); - assert.strictEqual(fs.readFileSync(file, 'utf8'), '.XX.X\n'); + assert.strictEqual(fs.readFileSync(file, 'utf8'), '.XX.\n'); const file2Contents = fs.readFileSync(file2, 'utf8'); assert.match(file2Contents, /▶ nested/); assert.match(file2Contents, /✔ ok/); @@ -89,7 +89,7 @@ describe('node:test reporters', { concurrency: true }, () => { ['--test', '--test-reporter', fixtures.fileURL('test-runner/custom_reporters/', filename), testFile]); assert.strictEqual(child.stderr.toString(), ''); - assert.strictEqual(child.stdout.toString(), `${filename} {"test:start":5,"test:pass":2,"test:fail":3,"test:plan":3,"test:diagnostic":7}`); + assert.strictEqual(child.stdout.toString(), `${filename} {"test:start":4,"test:pass":2,"test:fail":2,"test:plan":2,"test:diagnostic":7}`); }); }); @@ -100,7 +100,7 @@ describe('node:test reporters', { concurrency: true }, () => { assert.strictEqual(child.stderr.toString(), ''); assert.match( child.stdout.toString(), - /^package: reporter-cjs{"test:start":5,"test:pass":2,"test:fail":3,"test:plan":3,"test:diagnostic":\d+}$/, + /^package: reporter-cjs{"test:start":4,"test:pass":2,"test:fail":2,"test:plan":2,"test:diagnostic":\d+}$/, ); }); @@ -111,7 +111,7 @@ describe('node:test reporters', { concurrency: true }, () => { assert.strictEqual(child.stderr.toString(), ''); assert.match( child.stdout.toString(), - /^package: reporter-esm{"test:start":5,"test:pass":2,"test:fail":3,"test:plan":3,"test:diagnostic":\d+}$/, + /^package: reporter-esm{"test:start":4,"test:pass":2,"test:fail":2,"test:plan":2,"test:diagnostic":\d+}$/, ); }); }); diff --git a/test/parallel/test-runner-run.mjs b/test/parallel/test-runner-run.mjs index b6e0eee3b259ad..6ac007bfb5dd6c 100644 --- a/test/parallel/test-runner-run.mjs +++ b/test/parallel/test-runner-run.mjs @@ -27,7 +27,7 @@ describe('require(\'node:test\').run', { concurrency: true }, () => { it('should succeed with a file', async () => { const stream = run({ files: [join(testFixtures, 'test/random.cjs')] }); stream.on('test:fail', common.mustNotCall()); - stream.on('test:pass', common.mustCall(2)); + stream.on('test:pass', common.mustCall(1)); // eslint-disable-next-line no-unused-vars for await (const _ of stream); }); @@ -35,7 +35,7 @@ describe('require(\'node:test\').run', { concurrency: true }, () => { it('should run same file twice', async () => { const stream = run({ files: [join(testFixtures, 'test/random.cjs'), join(testFixtures, 'test/random.cjs')] }); stream.on('test:fail', common.mustNotCall()); - stream.on('test:pass', common.mustCall(4)); + stream.on('test:pass', common.mustCall(2)); // eslint-disable-next-line no-unused-vars for await (const _ of stream); });