fix: add concurrency limit to fs calls

package.json

@@ -21,6 +21,7 @@
   "dependencies": {
     "@mapbox/node-pre-gyp": "^1.0.5",
     "acorn": "^8.6.0",
+    "async-sema": "^3.1.1",
     "bindings": "^1.4.0",
     "estree-walker": "2.0.2",
     "glob": "^7.1.3",

readme.md

@@ -118,6 +118,18 @@ By its nature of integrating into existing build systems, the TypeScript
 compiler is not included in this project - rather the TypeScript transform
 layer requires separate integration into the `readFile` hook.
 
+#### File IO Concurrency
+
+In some large projects, the file tracing logic may process many files at the same time. In this case, if you do not limit the number of concurrent files IO, OOM problems are likely to occur.
+
+We use a default of 1024 concurrency to balance performance and memory usage for fs operations. You can increase this value to a higher number for faster speed, but be aware of the memory issues if the concurrency is too high.
+
+```js
+const { fileList } = await nodeFileTrace(files, {
+  fileIOConcurrency: 2048,
+});
+```
+
 #### Analysis
 
 Analysis options allow customizing how much analysis should be performed to exactly work out the dependency list.

src/node-file-trace.ts

@@ -6,6 +6,7 @@ import resolveDependency from './resolve-dependency';
 import { isMatch } from 'micromatch';
 import { sharedLibEmit } from './utils/sharedlib-emit';
 import { join } from 'path';
+import { Sema } from 'async-sema';
 
 const fsReadFile = fs.promises.readFile;
 const fsReadlink = fs.promises.readlink;
@@ -65,6 +66,7 @@ export class Job {
   public processed: Set<string>;
   public warnings: Set<Error>;
   public reasons: NodeFileTraceReasons = new Map()
+  private fileIOQueue: Sema;
 
   constructor ({
     base = process.cwd(),
@@ -79,6 +81,9 @@ export class Job {
     ts = true,
     analysis = {},
     cache,
+    // we use a default of 1024 concurrency to balance
+    // performance and memory usage for fs operations
+    fileIOConcurrency = 1024,
   }: NodeFileTraceOptions) {
     this.ts = ts;
     base = resolve(base);
@@ -116,6 +121,7 @@ export class Job {
     this.paths = resolvedPaths;
     this.log = log;
     this.mixedModules = mixedModules;
+    this.fileIOQueue = new Sema(fileIOConcurrency);
 
     this.analysis = {};
     if (analysis !== false) {
@@ -152,6 +158,7 @@ export class Job {
   async readlink (path: string) {
     const cached = this.symlinkCache.get(path);
     if (cached !== undefined) return cached;
+    await this.fileIOQueue.acquire();
     try {
       const link = await fsReadlink(path);
       // also copy stat cache to symlink
@@ -167,6 +174,9 @@ export class Job {
       this.symlinkCache.set(path, null);
       return null;
     }
+    finally {
+      this.fileIOQueue.release();
+    }
   }
 
   async isFile (path: string) {
@@ -186,6 +196,7 @@ export class Job {
   async stat (path: string) {
     const cached = this.statCache.get(path);
     if (cached) return cached;
+    await this.fileIOQueue.acquire();
     try {
       const stats = await fsStat(path);
       this.statCache.set(path, stats);
@@ -198,6 +209,9 @@ export class Job {
       }
       throw e;
     }
+    finally {
+      this.fileIOQueue.release();
+    }
   }
 
   async resolve (id: string, parent: string, job: Job, cjsResolve: boolean): Promise<string | string[]> {
@@ -207,6 +221,7 @@ export class Job {
   async readFile (path: string): Promise<string | Buffer | null> {
     const cached = this.fileCache.get(path);
     if (cached !== undefined) return cached;
+    await this.fileIOQueue.acquire();
     try {
       const source = (await fsReadFile(path)).toString();
       this.fileCache.set(path, source);
@@ -219,6 +234,9 @@ export class Job {
       }
       throw e;
     }
+    finally {
+      this.fileIOQueue.release();
+    }
   }
 
   async realpath (path: string, parent?: string, seen = new Set()): Promise<string> {

src/types.ts

@@ -49,6 +49,7 @@ export interface NodeFileTraceOptions {
   stat?: (path: string) => Promise<Stats | null>;
   readlink?: (path: string) => Promise<string | null>;
   resolve?: (id: string, parent: string, job: Job, cjsResolve: boolean) => Promise<string | string[]>;
+  fileIOConcurrency?: number;
 }
 
 export type NodeFileTraceReasonType = 'initial' | 'resolve' | 'dependency' | 'asset' | 'sharedlib';

yarn.lock

@@ -3047,6 +3047,11 @@ async-retry@^1.2.1, async-retry@^1.3.1:
   dependencies:
     retry "0.12.0"
 
+async-sema@^3.1.1:
+  version "3.1.1"
+  resolved "https://registry.npmjs.org/async-sema/-/async-sema-3.1.1.tgz#e527c08758a0f8f6f9f15f799a173ff3c40ea808"
+  integrity sha512-tLRNUXati5MFePdAk8dw7Qt7DpxPB60ofAgn8WRhW6a2rcimZnYBP9oxHiv0OHy+Wz7kPMG+t4LGdt31+4EmGg==
+
 async@0.9.x:
   version "0.9.2"
   resolved "https://registry.yarnpkg.com/async/-/async-0.9.2.tgz#aea74d5e61c1f899613bf64bda66d4c78f2fd17d"