fix: add concurrency limit to readlink

package.json

@@ -21,6 +21,7 @@
   "dependencies": {
     "@mapbox/node-pre-gyp": "^1.0.5",
     "acorn": "^8.6.0",
+    "async-sema": "^3.1.1",
     "bindings": "^1.4.0",
     "estree-walker": "2.0.2",
     "glob": "^7.1.3",

src/node-file-trace.ts

@@ -6,6 +6,7 @@ import resolveDependency from './resolve-dependency';
 import { isMatch } from 'micromatch';
 import { sharedLibEmit } from './utils/sharedlib-emit';
 import { join } from 'path';
+import { Sema } from 'async-sema';
 
 const fsReadFile = fs.promises.readFile;
 const fsReadlink = fs.promises.readlink;
@@ -65,6 +66,7 @@ export class Job {
   public processed: Set<string>;
   public warnings: Set<Error>;
   public reasons: NodeFileTraceReasons = new Map()
+  private fileIOQueue: Sema;
 
   constructor ({
     base = process.cwd(),
@@ -79,6 +81,7 @@ export class Job {
     ts = true,
     analysis = {},
     cache,
+    fileIOConcurrency = Number(process.env.FILE_IO_CONCURRENCY) || 1024,
   }: NodeFileTraceOptions) {
     this.ts = ts;
     base = resolve(base);
@@ -116,6 +119,7 @@ export class Job {
     this.paths = resolvedPaths;
     this.log = log;
     this.mixedModules = mixedModules;
+    this.fileIOQueue = new Sema(fileIOConcurrency);
 
     this.analysis = {};
     if (analysis !== false) {
@@ -152,8 +156,10 @@ export class Job {
   async readlink (path: string) {
     const cached = this.symlinkCache.get(path);
     if (cached !== undefined) return cached;
+    await this.fileIOQueue.acquire();
     try {
       const link = await fsReadlink(path);
+      await this.fileIOQueue.release();
       // also copy stat cache to symlink
       const stats = this.statCache.get(path);
       if (stats)
@@ -162,6 +168,7 @@ export class Job {
       return link;
     }
     catch (e) {
+      await this.fileIOQueue.release();
       if (e.code !== 'EINVAL' && e.code !== 'ENOENT' && e.code !== 'UNKNOWN')
         throw e;
       this.symlinkCache.set(path, null);
@@ -186,12 +193,15 @@ export class Job {
   async stat (path: string) {
     const cached = this.statCache.get(path);
     if (cached) return cached;
+    await this.fileIOQueue.acquire();
     try {
       const stats = await fsStat(path);
+      await this.fileIOQueue.release();
       this.statCache.set(path, stats);
       return stats;
     }
     catch (e) {
+      await this.fileIOQueue.release();
       if (e.code === 'ENOENT') {
         this.statCache.set(path, null);
         return null;
@@ -207,12 +217,15 @@ export class Job {
   async readFile (path: string): Promise<string | Buffer | null> {
     const cached = this.fileCache.get(path);
     if (cached !== undefined) return cached;
+    await this.fileIOQueue.acquire();
     try {
       const source = (await fsReadFile(path)).toString();
       this.fileCache.set(path, source);
+      await this.fileIOQueue.release();
       return source;
     }
     catch (e) {
+      await this.fileIOQueue.release();
       if (e.code === 'ENOENT' || e.code === 'EISDIR') {
         this.fileCache.set(path, null);
         return null;

src/types.ts

@@ -49,6 +49,7 @@ export interface NodeFileTraceOptions {
   stat?: (path: string) => Promise<Stats | null>;
   readlink?: (path: string) => Promise<string | null>;
   resolve?: (id: string, parent: string, job: Job, cjsResolve: boolean) => Promise<string | string[]>;
+  fileIOConcurrency?: number;
 }
 
 export type NodeFileTraceReasonType = 'initial' | 'resolve' | 'dependency' | 'asset' | 'sharedlib';

yarn.lock

@@ -3047,6 +3047,11 @@ async-retry@^1.2.1, async-retry@^1.3.1:
   dependencies:
     retry "0.12.0"
 
+async-sema@^3.1.1:
+  version "3.1.1"
+  resolved "https://registry.npmjs.org/async-sema/-/async-sema-3.1.1.tgz#e527c08758a0f8f6f9f15f799a173ff3c40ea808"
+  integrity sha512-tLRNUXati5MFePdAk8dw7Qt7DpxPB60ofAgn8WRhW6a2rcimZnYBP9oxHiv0OHy+Wz7kPMG+t4LGdt31+4EmGg==
+
 async@0.9.x:
   version "0.9.2"
   resolved "https://registry.yarnpkg.com/async/-/async-0.9.2.tgz#aea74d5e61c1f899613bf64bda66d4c78f2fd17d"