Skip to content

Commit

Permalink
refactor: use bencho for benchmarks
Browse files Browse the repository at this point in the history
  • Loading branch information
mrmlnc committed Apr 18, 2023
1 parent 42f3d96 commit 56d09b7
Show file tree
Hide file tree
Showing 34 changed files with 625 additions and 1,026 deletions.
21 changes: 6 additions & 15 deletions .github/workflows/benchmark.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,21 +3,13 @@ name: Benchmark
on:
- push

env:
BENCHMARK_LAUNCHES: 100
BENCHMARK_MAX_STDEV: 10
BENCHMARK_RETRIES: 3

jobs:
product:
name: Product benchmark
concurrency:
group: ${{ github.workflow }}-benchmark-product-${{ github.ref }}
cancel-in-progress: true
runs-on: ubuntu-latest
env:
BENCHMARK_TYPE: product
BENCHMARK_OPTIONS: '{}'
steps:
- name: Setup repository
uses: actions/checkout@v3
Expand All @@ -30,11 +22,11 @@ jobs:
- name: Compile sources
run: npm run compile
- name: Benchmark (async)
run: npm run bench-async
run: npm run bench:product:async
- name: Benchmark (stream)
run: npm run bench-stream
run: npm run bench:product:stream
- name: Benchmark (sync)
run: npm run bench-sync
run: npm run bench:product:sync

regress:
name: Regress benchmark with options (${{ matrix.benchmark_options }})
Expand All @@ -50,7 +42,6 @@ jobs:
- '{ "objectMode": true }'
- '{ "absolute": true }'
env:
BENCHMARK_TYPE: regression
BENCHMARK_OPTIONS: ${{ matrix.benchmark_options }}
steps:
- name: Setup repository
Expand All @@ -64,8 +55,8 @@ jobs:
- name: Compile sources
run: npm run compile
- name: Benchmark (async)
run: npm run bench-async
run: npm run bench:regression:async
- name: Benchmark (stream)
run: npm run bench-stream
run: npm run bench:regression:stream
- name: Benchmark (sync)
run: npm run bench-sync
run: npm run bench:regression:sync
3 changes: 3 additions & 0 deletions __snapshots__/absolute.e2e.js
Original file line number Diff line number Diff line change
Expand Up @@ -89,20 +89,23 @@ exports['Options Absolute {"pattern":"fixtures/**/*","options":{"absolute":true}
]

exports['Options Absolute {"pattern":"fixtures/../*","options":{"absolute":true}} (sync) 1'] = [
"<root>/herebyfile.mjs",
"<root>/LICENSE",
"<root>/package.json",
"<root>/README.md",
"<root>/tsconfig.json"
]

exports['Options Absolute {"pattern":"fixtures/../*","options":{"absolute":true}} (async) 1'] = [
"<root>/herebyfile.mjs",
"<root>/LICENSE",
"<root>/package.json",
"<root>/README.md",
"<root>/tsconfig.json"
]

exports['Options Absolute {"pattern":"fixtures/../*","options":{"absolute":true}} (stream) 1'] = [
"<root>/herebyfile.mjs",
"<root>/LICENSE",
"<root>/package.json",
"<root>/README.md",
Expand Down
3 changes: 3 additions & 0 deletions __snapshots__/regular.e2e.js
Original file line number Diff line number Diff line change
Expand Up @@ -5483,20 +5483,23 @@ exports['Patterns Regular (segmented lists) {"pattern":"{book.xml,library/**/a/b
]

exports['Patterns Regular (relative) {"pattern":"./*","options":{}} (sync) 1'] = [
"herebyfile.mjs",
"LICENSE",
"package.json",
"README.md",
"tsconfig.json"
]

exports['Patterns Regular (relative) {"pattern":"./*","options":{}} (async) 1'] = [
"herebyfile.mjs",
"LICENSE",
"package.json",
"README.md",
"tsconfig.json"
]

exports['Patterns Regular (relative) {"pattern":"./*","options":{}} (stream) 1'] = [
"herebyfile.mjs",
"LICENSE",
"package.json",
"README.md",
Expand Down
85 changes: 85 additions & 0 deletions herebyfile.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
import { execa } from 'execa';
import { task } from 'hereby';

const CONCURRENCY = process.env.CONCURRENCY === '1';
const REPORTER = process.env.REPORTER ?? 'compact';
const WARMUP_COUNT = process.env.WARMUP_COUNT ?? 50;
const RUNS_COUNT = process.env.RUNS_COUNT ?? 100;

const PRODUCT_ASYNC_SUITE = './out/benchmark/suites/product/async.js';
const PRODUCT_SYNC_SUITE = './out/benchmark/suites/product/sync.js';
const PRODUCT_STREAM_SUITE = './out/benchmark/suites/product/stream.js';

const REGRESSION_ASYNC_SUITE = './out/benchmark/suites/regression/async.js';
const REGRESSION_SYNC_SUITE = './out/benchmark/suites/regression/sync.js';
const REGRESSION_STREAM_SUITE = './out/benchmark/suites/regression/stream.js';

const FLATTEN_PATTERN = '*';
const DEEP_PATTERN = '**';
const PARTIAL_FLATTEN_PATTERN = '{fixtures,out}/{first,second}/*';
const PARTIAL_DEEP_PATTERN = '{fixtures,out}/**';

async function benchTask(suite, label, pattern, implementations = []) {
await execa('bencho', [
`'node ${suite} . "${pattern}" {impl}'`,
`-n "${label} {impl} ${pattern}"`,
`-w ${WARMUP_COUNT}`,
`-r ${RUNS_COUNT}`,
`-l impl=${implementations.join()}`,
`--reporter=${REPORTER}`
], {
shell: true,
stdout: 'inherit'
});
}

function makeBenchSuiteTask(type, label, suite, implementations = []) {
const asyncFlattenTask = task({
name: `bench:${type}:${label}:flatten`,
run: () => benchTask(suite, 'async', FLATTEN_PATTERN, implementations)
});

const asyncDeepTask = task({
name: `bench:${type}:${label}:deep`,
dependencies: CONCURRENCY ? [] : [asyncFlattenTask],
run: () => benchTask(suite, 'async', DEEP_PATTERN, implementations)
});

const asyncPartialFlattenTask = task({
name: `bench:${type}:${label}:partial_flatten`,
dependencies: CONCURRENCY ? [] : [asyncDeepTask],
run: () => benchTask(suite, 'async', PARTIAL_FLATTEN_PATTERN, implementations)
});

const asyncPartialDeepTask = task({
name: `bench:${type}:${label}:partial_deep`,
dependencies: CONCURRENCY ? [] : [asyncPartialFlattenTask],
run: () => benchTask(suite, 'async', PARTIAL_DEEP_PATTERN, implementations)
});

return task({
name: `bench:${type}:${label}`,
dependencies: CONCURRENCY ? [] : [asyncPartialDeepTask],
run: () => {}
});
}

export const {
productAsyncTask,
productStreamTask,
productSyncTask
} = {
productAsyncTask: makeBenchSuiteTask('product', 'async', PRODUCT_ASYNC_SUITE, ['fast-glob', 'node-glob', 'fdir']),
productStreamTask: makeBenchSuiteTask('product', 'stream', PRODUCT_STREAM_SUITE, ['fast-glob', 'node-glob']),
productSyncTask: makeBenchSuiteTask('product', 'sync', PRODUCT_SYNC_SUITE, ['fast-glob', 'node-glob', 'fdir'])
};

export const {
regressionAsyncTask,
regressionStreamTask,
regressionSyncTask
} = {
regressionAsyncTask: makeBenchSuiteTask('regression', 'async', REGRESSION_ASYNC_SUITE, ['current', 'previous']),
regressionStreamTask: makeBenchSuiteTask('regression', 'stream', REGRESSION_STREAM_SUITE, ['current', 'previous']),
regressionSyncTask: makeBenchSuiteTask('regression', 'sync', REGRESSION_SYNC_SUITE, ['current', 'previous'])
};
44 changes: 16 additions & 28 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -27,34 +27,27 @@
],
"devDependencies": {
"@nodelib/fs.macchiato": "^1.0.1",
"@types/compute-stdev": "^1.0.0",
"@types/easy-table": "^0.0.32",
"@types/glob": "^7.1.1",
"@types/glob-parent": "^5.1.0",
"@types/is-ci": "^2.0.0",
"@types/merge2": "^1.1.4",
"@types/micromatch": "^4.0.0",
"@types/minimist": "^1.2.0",
"@types/mocha": "^5.2.7",
"@types/node": "^12.7.8",
"@types/picomatch": "^2.3.0",
"@types/rimraf": "^2.0.2",
"@types/sinon": "^7.5.0",
"compute-stdev": "^1.0.0",
"easy-table": "^1.1.1",
"bencho": "^0.1.1",
"eslint": "^6.5.1",
"eslint-config-mrmlnc": "^1.1.0",
"execa": "^2.0.4",
"execa": "^7.1.1",
"fast-glob": "^3.0.4",
"fdir": "^5.1.0",
"fdir": "^6.0.1",
"glob": "^7.1.4",
"is-ci": "^2.0.0",
"log-update": "^4.0.0",
"minimist": "^1.2.0",
"hereby": "^1.8.1",
"mocha": "^6.2.1",
"rimraf": "^3.0.0",
"sinon": "^7.5.0",
"snap-shot-it": "^7.9.10",
"tiny-glob": "^0.2.6",
"typescript": "^3.6.3"
},
"dependencies": {
Expand All @@ -75,21 +68,16 @@
"test:e2e:stream": "mocha \"out/**/*.e2e.js\" -s 0 --grep \"\\(stream\\)\"",
"build": "npm run clean && npm run compile && npm run lint && npm test",
"watch": "npm run clean && npm run compile -- --sourceMap --watch",
"bench": "npm run bench-async && npm run bench-stream && npm run bench-sync",
"bench-async": "npm run bench-async-flatten && npm run bench-async-deep && npm run bench-async-partial-flatten && npm run bench-async-partial-deep",
"bench-stream": "npm run bench-stream-flatten && npm run bench-stream-deep && npm run bench-stream-partial-flatten && npm run bench-stream-partial-deep",
"bench-sync": "npm run bench-sync-flatten && npm run bench-sync-deep && npm run bench-sync-partial-flatten && npm run bench-sync-partial-deep",
"bench-async-flatten": "node ./out/benchmark --mode async --pattern \"*\"",
"bench-async-deep": "node ./out/benchmark --mode async --pattern \"**\"",
"bench-async-partial-flatten": "node ./out/benchmark --mode async --pattern \"{fixtures,out}/{first,second}/*\"",
"bench-async-partial-deep": "node ./out/benchmark --mode async --pattern \"{fixtures,out}/**\"",
"bench-stream-flatten": "node ./out/benchmark --mode stream --pattern \"*\"",
"bench-stream-deep": "node ./out/benchmark --mode stream --pattern \"**\"",
"bench-stream-partial-flatten": "node ./out/benchmark --mode stream --pattern \"{fixtures,out}/{first,second}/*\"",
"bench-stream-partial-deep": "node ./out/benchmark --mode stream --pattern \"{fixtures,out}/**\"",
"bench-sync-flatten": "node ./out/benchmark --mode sync --pattern \"*\"",
"bench-sync-deep": "node ./out/benchmark --mode sync --pattern \"**\"",
"bench-sync-partial-flatten": "node ./out/benchmark --mode sync --pattern \"{fixtures,out}/{first,second}/*\"",
"bench-sync-partial-deep": "node ./out/benchmark --mode sync --pattern \"{fixtures,out}/**\""
"bench:async": "npm run bench:product:async && npm run bench:regression:async",
"bench:stream": "npm run bench:product:stream && npm run bench:regression:stream",
"bench:sync": "npm run bench:product:sync && npm run bench:regression:sync",
"bench:product": "npm run bench:product:async && npm run bench:product:sync && npm run bench:product:stream",
"bench:product:async": "hereby bench:product:async",
"bench:product:sync": "hereby bench:product:sync",
"bench:product:stream": "hereby bench:product:stream",
"bench:regression": "npm run bench:regression:async && npm run bench:regression:sync && npm run bench:regression:stream",
"bench:regression:async": "hereby bench:regression:async",
"bench:regression:sync": "hereby bench:regression:sync",
"bench:regression:stream": "hereby bench:regression:stream"
}
}
36 changes: 0 additions & 36 deletions src/benchmark/index.ts

This file was deleted.

7 changes: 0 additions & 7 deletions src/benchmark/logger.ts

This file was deleted.

36 changes: 0 additions & 36 deletions src/benchmark/reporter.spec.ts

This file was deleted.

Loading

0 comments on commit 56d09b7

Please sign in to comment.