diff --git a/package.json b/package.json index 4c560ea8..b5bfeec2 100644 --- a/package.json +++ b/package.json @@ -48,6 +48,7 @@ "analytics-node": "^3.4.0-beta.1", "apollo-server-express": "^2.14.2", "argon2": "^0.27.2", + "async-sema": "^3.1.1", "auth0": "^2.27.1", "axios": "^0.21.2", "azure-storage": "^2.10.3", diff --git a/src/node-file-trace.ts b/src/node-file-trace.ts index 7300e918..3e19cf38 100644 --- a/src/node-file-trace.ts +++ b/src/node-file-trace.ts @@ -6,6 +6,7 @@ import resolveDependency from './resolve-dependency'; import { isMatch } from 'micromatch'; import { sharedLibEmit } from './utils/sharedlib-emit'; import { join } from 'path'; +import { Sema } from 'async-sema'; const fsReadFile = fs.promises.readFile; const fsReadlink = fs.promises.readlink; @@ -65,6 +66,7 @@ export class Job { public processed: Set; public warnings: Set; public reasons: NodeFileTraceReasons = new Map() + private fileIOQueue: Sema; constructor ({ base = process.cwd(), @@ -79,6 +81,7 @@ export class Job { ts = true, analysis = {}, cache, + fileIOConcurrency = 128, }: NodeFileTraceOptions) { this.ts = ts; base = resolve(base); @@ -116,6 +119,9 @@ export class Job { this.paths = resolvedPaths; this.log = log; this.mixedModules = mixedModules; + this.fileIOQueue = new Sema(fileIOConcurrency, { + capacity: fileIOConcurrency * 10, + }); this.analysis = {}; if (analysis !== false) { @@ -152,8 +158,10 @@ export class Job { async readlink (path: string) { const cached = this.symlinkCache.get(path); if (cached !== undefined) return cached; + await this.fileIOQueue.acquire(); try { const link = await fsReadlink(path); + await this.fileIOQueue.release(); // also copy stat cache to symlink const stats = this.statCache.get(path); if (stats) @@ -162,6 +170,7 @@ export class Job { return link; } catch (e) { + await this.fileIOQueue.release(); if (e.code !== 'EINVAL' && e.code !== 'ENOENT' && e.code !== 'UNKNOWN') throw e; this.symlinkCache.set(path, null); @@ -187,11 +196,14 @@ export class Job { const cached = this.statCache.get(path); if (cached) return cached; try { + await this.fileIOQueue.acquire(); const stats = await fsStat(path); + await this.fileIOQueue.release(); this.statCache.set(path, stats); return stats; } catch (e) { + await this.fileIOQueue.release(); if (e.code === 'ENOENT') { this.statCache.set(path, null); return null; @@ -207,12 +219,15 @@ export class Job { async readFile (path: string): Promise { const cached = this.fileCache.get(path); if (cached !== undefined) return cached; + await this.fileIOQueue.acquire(); try { const source = (await fsReadFile(path)).toString(); this.fileCache.set(path, source); + await this.fileIOQueue.release(); return source; } catch (e) { + await this.fileIOQueue.release(); if (e.code === 'ENOENT' || e.code === 'EISDIR') { this.fileCache.set(path, null); return null; diff --git a/src/types.ts b/src/types.ts index da9201a8..a02a99ce 100644 --- a/src/types.ts +++ b/src/types.ts @@ -49,6 +49,7 @@ export interface NodeFileTraceOptions { stat?: (path: string) => Promise; readlink?: (path: string) => Promise; resolve?: (id: string, parent: string, job: Job, cjsResolve: boolean) => Promise; + fileIOConcurrency?: number; } export type NodeFileTraceReasonType = 'initial' | 'resolve' | 'dependency' | 'asset' | 'sharedlib'; diff --git a/yarn.lock b/yarn.lock index 190ffe23..61cd9db2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3047,6 +3047,11 @@ async-retry@^1.2.1, async-retry@^1.3.1: dependencies: retry "0.12.0" +async-sema@^3.1.1: + version "3.1.1" + resolved "https://registry.npmjs.org/async-sema/-/async-sema-3.1.1.tgz#e527c08758a0f8f6f9f15f799a173ff3c40ea808" + integrity sha512-tLRNUXati5MFePdAk8dw7Qt7DpxPB60ofAgn8WRhW6a2rcimZnYBP9oxHiv0OHy+Wz7kPMG+t4LGdt31+4EmGg== + async@0.9.x: version "0.9.2" resolved "https://registry.yarnpkg.com/async/-/async-0.9.2.tgz#aea74d5e61c1f899613bf64bda66d4c78f2fd17d"