fix: add concurrency limit to readlink

vercel · Jul 15, 2022 · 62c49bf · 62c49bf
1 parent 11c6888
commit 62c49bf
Show file tree

Hide file tree

Showing 4 changed files with 20 additions and 1 deletion.
diff --git a/package.json b/package.json
@@ -92,6 +92,7 @@
     "mysql": "^2.17.1",
     "npm": "^6.14.6",
     "oracledb": "^4.2.0",
+    "p-queue": "^6.6.2",
     "paraphrase": "1.8.0",
     "passport": "^0.4.0",
     "passport-google-oauth": "^2.0.0",

diff --git a/src/node-file-trace.ts b/src/node-file-trace.ts
@@ -6,6 +6,7 @@ import resolveDependency from './resolve-dependency';
 import { isMatch } from 'micromatch';
 import { sharedLibEmit } from './utils/sharedlib-emit';
 import { join } from 'path';
+import PromiseQueue from 'p-queue';
 
 const fsReadFile = fs.promises.readFile;
 const fsReadlink = fs.promises.readlink;
@@ -65,6 +66,7 @@ export class Job {
   public processed: Set<string>;
   public warnings: Set<Error>;
   public reasons: NodeFileTraceReasons = new Map()
+  private readLinkQueue: PromiseQueue;
 
   constructor ({
     base = process.cwd(),
@@ -79,6 +81,7 @@ export class Job {
     ts = true,
     analysis = {},
     cache,
+    readlinkConcurrency = 128,
   }: NodeFileTraceOptions) {
     this.ts = ts;
     base = resolve(base);
@@ -116,6 +119,7 @@ export class Job {
     this.paths = resolvedPaths;
     this.log = log;
     this.mixedModules = mixedModules;
+    this.readLinkQueue = new PromiseQueue({ concurrency: readlinkConcurrency });
 
     this.analysis = {};
     if (analysis !== false) {
@@ -153,7 +157,7 @@ export class Job {
     const cached = this.symlinkCache.get(path);
     if (cached !== undefined) return cached;
     try {
-      const link = await fsReadlink(path);
+      const link = await this.readLinkQueue.add(() => fsReadlink(path));
       // also copy stat cache to symlink
       const stats = this.statCache.get(path);
       if (stats)

diff --git a/src/types.ts b/src/types.ts
@@ -49,6 +49,7 @@ export interface NodeFileTraceOptions {
   stat?: (path: string) => Promise<Stats | null>;
   readlink?: (path: string) => Promise<string | null>;
   resolve?: (id: string, parent: string, job: Job, cjsResolve: boolean) => Promise<string | string[]>;
+  readlinkConcurrency?: number;
 }
 
 export type NodeFileTraceReasonType = 'initial' | 'resolve' | 'dependency' | 'asset' | 'sharedlib';

diff --git a/yarn.lock b/yarn.lock
@@ -6017,6 +6017,11 @@ eventemitter3@^3.1.0:
   resolved "https://registry.yarnpkg.com/eventemitter3/-/eventemitter3-3.1.2.tgz#2d3d48f9c346698fce83a85d7d664e98535df6e7"
   integrity sha512-tvtQIeLVHjDkJYnzf2dgVMxfuSGJeM/7UCG17TT4EumTfNtF+0nebF/4zWOIkCreAbtNqhGEboB6BWrwqNaw4Q==
 
+eventemitter3@^4.0.4:
+  version "4.0.7"
+  resolved "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz#2de9b68f6528d5644ef5c59526a1b4a07306169f"
+  integrity sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==
+
 events@1.1.1:
   version "1.1.1"
   resolved "https://registry.yarnpkg.com/events/-/events-1.1.1.tgz#9ebdb7635ad099c70dcc4c2a1f5004288e8bd924"
@@ -11960,6 +11965,14 @@ p-map@^2.1.0:
   resolved "https://registry.yarnpkg.com/p-map/-/p-map-2.1.0.tgz#310928feef9c9ecc65b68b17693018a665cea175"
   integrity sha512-y3b8Kpd8OAN444hxfBbFfj1FY/RjtTd8tzYwhUqNYXx0fXx2iX4maP4Qr6qhIKbQXI02wTLAda4fYUbDagTUFw==
 
+p-queue@^6.6.2:
+  version "6.6.2"
+  resolved "https://registry.npmjs.org/p-queue/-/p-queue-6.6.2.tgz#2068a9dcf8e67dd0ec3e7a2bcb76810faa85e426"
+  integrity sha512-RwFpb72c/BhQLEXIZ5K2e+AhgNVmIejGlTgiB9MzZ0e93GRvqZ7uSi0dvRF7/XIXDeNkra2fNHBxTyPDGySpjQ==
+  dependencies:
+    eventemitter3 "^4.0.4"
+    p-timeout "^3.2.0"
+
 p-timeout@^2.0.1:
   version "2.0.1"
   resolved "https://registry.yarnpkg.com/p-timeout/-/p-timeout-2.0.1.tgz#d8dd1979595d2dc0139e1fe46b8b646cb3cdf038"