From 03201edcb056c91f81cd356e7983dab254e3ca14 Mon Sep 17 00:00:00 2001 From: mrmlnc Date: Sun, 16 Jun 2019 20:55:30 +0300 Subject: [PATCH] fix(settings): set the `concurrency` option to count of CPUs Infinity does not speed up, but takes CPU time on very large directories with a large number of "branches". --- README.md | 5 ++--- src/providers/provider.spec.ts | 5 +++-- src/settings.spec.ts | 3 ++- src/settings.ts | 7 +++++-- 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index ee810c9c..cb38c7a4 100644 --- a/README.md +++ b/README.md @@ -209,11 +209,11 @@ See [Options](#options-1) section. #### concurrency * Type: `number` -* Default: `Infinity` +* Default: `os.cpus().length` Specifies the maximum number of concurrent requests from a reader to read directories. -> :book: The higher the number, the higher the performance and load on the file system. If you want to read in quiet mode, set the value to a comfortable number or `4 * os.cpus().length` (4 is default size of [thread pool work scheduling][libuv_threadpool]). +> :book: The higher the number, the higher the performance and load on the file system. If you want to read in quiet mode, set the value to a comfortable number or `1`. #### cwd @@ -673,7 +673,6 @@ This software is released under the terms of the MIT license. [glob_definition]: https://en.wikipedia.org/wiki/Glob_(programming) [glob_linux_man]: http://man7.org/linux/man-pages/man3/glob.3.html [intel_ssd]: https://ark.intel.com/content/www/us/en/ark/products/93012/intel-ssd-dc-s3520-series-240gb-2-5in-sata-6gb-s-3d1-mlc.html -[libuv_threadpool]: http://docs.libuv.org/en/v1.x/threadpool.html [micromatch_backslashes]: https://github.com/micromatch/micromatch#backslashes [micromatch_braces]: https://github.com/micromatch/braces [micromatch_extended_globbing]: https://github.com/micromatch/micromatch#extended-globbing diff --git a/src/providers/provider.spec.ts b/src/providers/provider.spec.ts index a210b6cc..8c8316b8 100644 --- a/src/providers/provider.spec.ts +++ b/src/providers/provider.spec.ts @@ -66,13 +66,14 @@ describe('Providers → Provider', () => { describe('.getReaderOptions', () => { it('should return options for reader with global base (.)', () => { - const provider = getProvider(); + const settings = new Settings(); + const provider = getProvider(settings); const task = tests.task.builder().base('.').positive('*').build(); const actual = provider.getReaderOptions(task); assert.strictEqual(actual.basePath, ''); - assert.strictEqual(actual.concurrency, Infinity); + assert.strictEqual(actual.concurrency, settings.concurrency); assert.strictEqual(typeof actual.deepFilter, 'function'); assert.strictEqual(typeof actual.entryFilter, 'function'); assert.strictEqual(typeof actual.errorFilter, 'function'); diff --git a/src/settings.spec.ts b/src/settings.spec.ts index a77a7091..0323471a 100644 --- a/src/settings.spec.ts +++ b/src/settings.spec.ts @@ -1,4 +1,5 @@ import * as assert from 'assert'; +import * as os from 'os'; import Settings, { DEFAULT_FILE_SYSTEM_ADAPTER } from './settings'; @@ -25,7 +26,7 @@ describe('Settings', () => { assert.ok(settings.globstar); assert.ok(settings.onlyFiles); assert.ok(settings.unique); - assert.strictEqual(settings.concurrency, Infinity); + assert.strictEqual(settings.concurrency, os.cpus().length); assert.strictEqual(settings.cwd, process.cwd()); }); diff --git a/src/settings.ts b/src/settings.ts index 6c474fb4..afdf9516 100644 --- a/src/settings.ts +++ b/src/settings.ts @@ -1,7 +1,10 @@ import * as fs from 'fs'; +import * as os from 'os'; import { FileSystemAdapter, Pattern } from './types/index'; +const CPU_COUNT = os.cpus().length; + export const DEFAULT_FILE_SYSTEM_ADAPTER: FileSystemAdapter = { lstat: fs.lstat, lstatSync: fs.lstatSync, @@ -42,7 +45,7 @@ export interface Options { * Specifies the maximum number of concurrent requests from a reader to read * directories. * - * @default Infinity + * @default os.cpus().length */ concurrency?: number; /** @@ -154,7 +157,7 @@ export default class Settings { public readonly baseNameMatch: boolean = this._getValue(this._options.baseNameMatch, false); public readonly braceExpansion: boolean = this._getValue(this._options.braceExpansion, true); public readonly caseSensitiveMatch: boolean = this._getValue(this._options.caseSensitiveMatch, true); - public readonly concurrency: number = this._getValue(this._options.concurrency, Infinity); + public readonly concurrency: number = this._getValue(this._options.concurrency, CPU_COUNT); public readonly cwd: string = this._getValue(this._options.cwd, process.cwd()); public readonly deep: number = this._getValue(this._options.deep, Infinity); public readonly dot: boolean = this._getValue(this._options.dot, false);