diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0c447b4..89639e2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,10 +19,10 @@ jobs: node-version: [18.x, 20.x, 21.x] os: [ubuntu-latest, windows-latest] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Use Node.js - uses: actions/setup-node@v2 + uses: actions/setup-node@v4 with: node-version: ${{ matrix.node-version }} @@ -30,6 +30,16 @@ jobs: run: | npm install + - name: Lint + run: | + npm run lint + - name: Run tests run: | - npm run test + npm run unit -- --reporter spec --reporter md:report.md --reporter gh + + - name: Upload report + shell: bash + if: success() || failure() + run: | + cat report.md >> "$GITHUB_STEP_SUMMARY" diff --git a/README.md b/README.md index 3f66214..5990b01 100644 --- a/README.md +++ b/README.md @@ -93,6 +93,70 @@ Note the use of `incremental: true`, which speed up compilation massively. * `--ignore` or `-i`, ignore a glob pattern, and not look for tests there * `--expose-gc`, exposes the gc() function to tests * `--pattern` or `-p`, run tests matching the given glob pattern +* `--reporter` or `-r`, set up a reporter, use a colon to set a file destination. Default: `spec`. + +## Reporters + +Here are the available reporters: + +* `md`: creates a markdown table, useful for setting up a Summary in your GitHub Action +* `gh`: emits `::error` workflow commands for GitHub Actions to show inlined error. Enabled by default when running on GHA. +* `tap`: outputs the test results in the TAP format. +* `spec`: outputs the test results in a human-readable format. +* `dot`: outputs the test results in a compact format, where each passing test is represented by a ., and each failing test is represented by a X. +* `junit`: outputs test results in a jUnit XML format + +## GitHub Action Summary + +The following will automatically show the summary of the test run in the summary page of GitHub Actions. + +```yaml +name: ci + +on: + push: + paths-ignore: + - 'docs/**' + - '*.md' + pull_request: + paths-ignore: + - 'docs/**' + - '*.md' + +jobs: + test: + runs-on: ${{matrix.os}} + + strategy: + matrix: + node-version: [18.x, 20.x, 21.x] + os: [ubuntu-latest, windows-latest] + steps: + - uses: actions/checkout@v4 + + - name: Use Node.js + uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node-version }} + + - name: Install + run: | + npm install + + - name: Lint + run: | + npm run lint + + - name: Run tests + run: | + npm run unit -- --reporter spec --reporter md:report.md + + - name: Upload report + shell: bash + if: success() || failure() + run: | + cat report.md >> "$GITHUB_STEP_SUMMARY" +``` ## License diff --git a/borp.js b/borp.js index fe4b504..65cd868 100755 --- a/borp.js +++ b/borp.js @@ -1,24 +1,23 @@ #! /usr/bin/env node import { parseArgs } from 'node:util' -import { tap, spec } from 'node:test/reporters' +import Reporters from 'node:test/reporters' import { mkdtemp, rm, readFile } from 'node:fs/promises' +import { createWriteStream } from 'node:fs' import { finished } from 'node:stream/promises' import { join, relative } from 'node:path' import posix from 'node:path/posix' import runWithTypeScript from './lib/run.js' +import { MarkdownReporter, GithubWorkflowFailuresReporter } from './lib/reporters.js' import { Report } from 'c8' import os from 'node:os' import { execa } from 'execa' -let reporter /* c8 ignore next 4 */ -if (process.stdout.isTTY) { - /* eslint new-cap: "off" */ - reporter = new spec() -} else { - reporter = tap -} +process.on('unhandledRejection', (err) => { + console.error(err) + process.exit(1) +}) const args = parseArgs({ args: process.argv.slice(2), @@ -32,7 +31,13 @@ const args = parseArgs({ 'coverage-exclude': { type: 'string', short: 'X', multiple: true }, ignore: { type: 'string', short: 'i', multiple: true }, 'expose-gc': { type: 'boolean' }, - help: { type: 'boolean', short: 'h' } + help: { type: 'boolean', short: 'h' }, + reporter: { + type: 'string', + short: 'r', + default: ['spec'], + multiple: true + } }, allowPositionals: true }) @@ -79,13 +84,44 @@ const config = { } try { + const pipes = [] + + const reporters = { + ...Reporters, + md: new MarkdownReporter(config), + gh: new GithubWorkflowFailuresReporter(config), + /* eslint new-cap: "off" */ + spec: new Reporters.spec() + } + + // If we're running in a GitHub action, adds the gh reporter + // by default so that we can report failures to GitHub + if (process.env.GITHUB_ACTION) { + args.values.reporter.push('gh') + } + + for (const input of args.values.reporter) { + const [name, dest] = input.split(':') + const reporter = reporters[name] + if (!reporter) { + throw new Error(`Unknown reporter: ${name}`) + } + let output = process.stdout + if (dest) { + output = createWriteStream(dest) + } + pipes.push([reporter, output]) + } + const stream = await runWithTypeScript(config) stream.on('test:fail', () => { process.exitCode = 1 }) - stream.compose(reporter).pipe(process.stdout) + for (const [reporter, output] of pipes) { + stream.compose(reporter).pipe(output) + } await finished(stream) diff --git a/lib/reporters.js b/lib/reporters.js new file mode 100644 index 0000000..6a87724 --- /dev/null +++ b/lib/reporters.js @@ -0,0 +1,114 @@ +import { Transform } from 'node:stream' +import { fileURLToPath } from 'node:url' + +function normalizeFile (file, cwd) { + let res = file + if (file.startsWith('file://')) { + try { + res = fileURLToPath(new URL(file)) + } catch (err) { + if (err.code === 'ERR_INVALID_FILE_URL_PATH') { + res = fileURLToPath(new URL(file.replace('file:///', 'file://'))) + } + } + } + res = res.replace(cwd, '') + if (res.startsWith('/') || res.startsWith('\\')) { + res = res.slice(1) + } + return res +} + +function eventToLine (event) { + return `* __${event.data.name}__, duration ${event.data.details.duration_ms}ms, line ${event.data.line}\n` +} + +export class MarkdownReporter extends Transform { + constructor (opts) { + super({ + ...opts, + objectMode: true + }) + + this._files = {} + this._cwd = opts?.cwd + } + + getFile (path) { + const file = this._files[path] || { + pass: [], + fail: [] + } + this._files[path] = file + return file + } + + _transform (event, encoding, callback) { + if (!event.data.file) { + callback() + return + } + + const path = normalizeFile(event.data.file, this._cwd) + const file = this.getFile(path) + switch (event.type) { + case 'test:pass': + file.pass.push(event) + break + case 'test:fail': + file.fail.push(event) + break + } + + callback() + } + + _flush (callback) { + this.push('# Summary\n') + for (const [path, file] of Object.entries(this._files)) { + this.push(`## ${path}\n`) + if (file.pass.length > 0) { + this.push('### :white_check_mark: Pass\n') + for (const event of file.pass) { + this.push(eventToLine(event)) + } + } + if (file.fail.length > 0) { + this.push('### :x: Fail\n') + for (const event of file.fail) { + this.push(eventToLine(event)) + } + } + } + this.push(null) + callback() + } +} + +export class GithubWorkflowFailuresReporter extends Transform { + constructor (opts) { + super({ + ...opts, + objectMode: true + }) + + this._files = {} + this._cwd = opts?.cwd + } + + _transform (event, encoding, callback) { + if (!event.data.file) { + callback() + return + } + + const path = normalizeFile(event.data.file, this._cwd) + switch (event.type) { + case 'test:fail': + this.push(`::error file=${path},line=${event.data.line}::${event.data.name}\n`) + break + } + + callback() + } +} diff --git a/lib/run.js b/lib/run.js index aa2bbba..2c0720a 100644 --- a/lib/run.js +++ b/lib/run.js @@ -58,6 +58,7 @@ export default async function runWithTypeScript (config) { } config.prefix = prefix config.setup = (test) => { + /* c8 ignore next 12 */ if (test.reporter) { for (const chunk of pushable) { test.reporter.push(chunk) diff --git a/test/cli.test.js b/test/cli.test.js index 72984d3..6df5b07 100644 --- a/test/cli.test.js +++ b/test/cli.test.js @@ -5,6 +5,8 @@ import { rejects } from 'node:assert' const borp = join(import.meta.url, '..', 'borp.js') +delete process.env.GITHUB_ACTION + test('limit concurrency', async () => { await execa('node', [ borp, diff --git a/test/coverage.test.js b/test/coverage.test.js index aff77d8..8d763c1 100644 --- a/test/coverage.test.js +++ b/test/coverage.test.js @@ -3,6 +3,7 @@ import { match, doesNotMatch } from 'node:assert' import { execa } from 'execa' import { join } from 'desm' +delete process.env.GITHUB_ACTION const borp = join(import.meta.url, '..', 'borp.js') test('coverage', async () => { diff --git a/test/reporters.test.js b/test/reporters.test.js new file mode 100644 index 0000000..754c934 --- /dev/null +++ b/test/reporters.test.js @@ -0,0 +1,172 @@ +import { MarkdownReporter, GithubWorkflowFailuresReporter } from '../lib/reporters.js' +import { test, describe } from 'node:test' +import { strictEqual } from 'node:assert' + +const cwd = process.platform === 'win32' ? 'C:\\foo' : '/foo' +const base = process.platform === 'win32' ? 'file://C:\\foo\\test\\' : 'file:///foo/test/' + +describe('MarkdownReporter', async () => { + test('should write a report', async () => { + const reporter = new MarkdownReporter({ cwd }) + + // This is skipped + reporter.write({ + type: 'test:start', + data: {} + }) + + reporter.write({ + type: 'test:pass', + data: { + name: 'add', + file: base + 'add.test.ts', + line: 1, + details: { + duration_ms: 100 + } + } + }) + + reporter.write({ + type: 'test:pass', + data: { + name: 'add2', + file: base + 'add.test.ts', + line: 2, + details: { + duration_ms: 100 + } + } + }) + + reporter.write({ + type: 'test:fail', + data: { + name: 'add3', + file: base + 'add.test.ts', + line: 10, + details: { + duration_ms: 100 + } + } + }) + reporter.end() + + let output = '' + for await (const chunk of reporter) { + output += chunk + } + + strictEqual(output.replaceAll('\\', '/'), `# Summary +## test/add.test.ts +### :white_check_mark: Pass +* __add__, duration 100ms, line 1 +* __add2__, duration 100ms, line 2 +### :x: Fail +* __add3__, duration 100ms, line 10 +`) + }) + + test('skip fail heading if no failing tests', async () => { + const reporter = new MarkdownReporter({ cwd }) + + reporter.write({ + type: 'test:pass', + data: { + name: 'add', + file: base + 'add.test.ts', + line: 1, + details: { + duration_ms: 100 + } + } + }) + + reporter.write({ + type: 'test:pass', + data: { + name: 'add2', + file: base + 'add.test.ts', + line: 2, + details: { + duration_ms: 100 + } + } + }) + + reporter.end() + + let output = '' + for await (const chunk of reporter) { + output += chunk + } + + strictEqual(output.replaceAll('\\', '/'), `# Summary +## test/add.test.ts +### :white_check_mark: Pass +* __add__, duration 100ms, line 1 +* __add2__, duration 100ms, line 2 +`) + }) +}) + +describe('GithubWorkflowFailuresReporter', async () => { + test('should write error in github format', async () => { + const reporter = new GithubWorkflowFailuresReporter({ cwd }) + + // This is skipped + reporter.write({ + type: 'test:start', + data: {} + }) + + reporter.write({ + type: 'test:pass', + data: { + name: 'add', + file: base + 'add.test.ts', + line: 1, + details: { + duration_ms: 100 + } + } + }) + + reporter.write({ + type: 'test:fail', + data: { + name: 'add2', + file: base + 'add.test.ts', + line: 2, + details: { + duration_ms: 100 + } + } + }) + + reporter.write({ + type: 'test:fail', + data: { + name: 'add3', + file: base + 'add.test.ts', + line: 10, + details: { + duration_ms: 100 + } + } + }) + reporter.end() + + let output = '' + for await (const chunk of reporter) { + output += chunk + } + + const expected = [ + '::error file=test/add.test.ts,line=2::add2\n', + '::error file=test/add.test.ts,line=10::add3\n' + ].join('') + + strictEqual(output.replaceAll('\\', '/'), expected) + }) +})