module: support main w/o extension, pjson cache

This adds support for ensuring that the top-level main into Node is
supported loading when it has no extension for backwards-compat with
NodeJS bin workflows.

In addition package.json caching is implemented in the module lookup
process.

Backport-PR-URL: #18923
PR-URL: #18728
Reviewed-By: Benjamin Gruenbaum <benjamingr@gmail.com>

doc/api/esm.md

@@ -113,17 +113,22 @@ The resolve hook returns the resolved file URL and module format for a
 given module specifier and parent file URL:
 
 ```js
-import url from 'url';
+const baseURL = new URL('file://');
+baseURL.pathname = process.cwd() + '/';
 
-export async function resolve(specifier, parentModuleURL, defaultResolver) {
+export async function resolve(specifier,
+                              parentModuleURL = baseURL,
+                              defaultResolver) {
   return {
     url: new URL(specifier, parentModuleURL).href,
     format: 'esm'
   };
 }
 ```
 
-The default NodeJS ES module resolution function is provided as a third
+The parentURL is provided as `undefined` when performing main Node.js load itself.
+
+The default Node.js ES module resolution function is provided as a third
 argument to the resolver for easy compatibility workflows.
 
 In addition to returning the resolved file URL value, the resolve hook also
@@ -152,7 +157,10 @@ import Module from 'module';
 const builtins = Module.builtinModules;
 const JS_EXTENSIONS = new Set(['.js', '.mjs']);
 
-export function resolve(specifier, parentModuleURL/*, defaultResolve */) {
+const baseURL = new URL('file://');
+baseURL.pathname = process.cwd() + '/';
+
+export function resolve(specifier, parentModuleURL = baseURL, defaultResolve) {
   if (builtins.includes(specifier)) {
     return {
       url: specifier,

lib/internal/bootstrap_node.js

@@ -105,6 +105,7 @@
       process.emitWarning(
         'The ESM module loader is experimental.',
         'ExperimentalWarning', undefined);
+      NativeModule.require('internal/process/modules').setup();
     }
 
 

lib/internal/loader/DefaultResolve.js

@@ -2,7 +2,6 @@
 
 const { URL } = require('url');
 const CJSmodule = require('module');
-const internalURLModule = require('internal/url');
 const internalFS = require('internal/fs');
 const NativeModule = require('native_module');
 const { extname } = require('path');
@@ -11,6 +10,7 @@ const preserveSymlinks = !!process.binding('config').preserveSymlinks;
 const errors = require('internal/errors');
 const { resolve: moduleWrapResolve } = internalBinding('module_wrap');
 const StringStartsWith = Function.call.bind(String.prototype.startsWith);
+const { getURLFromFilePath, getPathFromURL } = require('internal/url');
 
 const realpathCache = new Map();
 
@@ -57,7 +57,8 @@ function resolve(specifier, parentURL) {
 
   let url;
   try {
-    url = search(specifier, parentURL);
+    url = search(specifier,
+                 parentURL || getURLFromFilePath(`${process.cwd()}/`).href);
   } catch (e) {
     if (typeof e.message === 'string' &&
         StringStartsWith(e.message, 'Cannot find module'))
@@ -66,17 +67,27 @@ function resolve(specifier, parentURL) {
   }
 
   if (!preserveSymlinks) {
-    const real = realpathSync(internalURLModule.getPathFromURL(url), {
+    const real = realpathSync(getPathFromURL(url), {
       [internalFS.realpathCacheKey]: realpathCache
     });
     const old = url;
-    url = internalURLModule.getURLFromFilePath(real);
+    url = getURLFromFilePath(real);
     url.search = old.search;
     url.hash = old.hash;
   }
 
   const ext = extname(url.pathname);
-  return { url: `${url}`, format: extensionFormatMap[ext] || ext };
+
+  let format = extensionFormatMap[ext];
+  if (!format) {
+    const isMain = parentURL === undefined;
+    if (isMain)
+      format = 'cjs';
+    else
+      throw new errors.Error('ERR_UNKNOWN_FILE_EXTENSION', url.pathname);
+  }
+
+  return { url: `${url}`, format };
 }
 
 module.exports = resolve;

lib/internal/loader/Loader.js

@@ -1,51 +1,21 @@
 'use strict';
 
-const path = require('path');
-const { getURLFromFilePath, URL } = require('internal/url');
 const errors = require('internal/errors');
-
 const ModuleMap = require('internal/loader/ModuleMap');
 const ModuleJob = require('internal/loader/ModuleJob');
 const defaultResolve = require('internal/loader/DefaultResolve');
 const createDynamicModule = require('internal/loader/CreateDynamicModule');
 const translators = require('internal/loader/Translators');
-const { setImportModuleDynamicallyCallback } = internalBinding('module_wrap');
+
 const FunctionBind = Function.call.bind(Function.prototype.bind);
 
 const debug = require('util').debuglog('esm');
 
-// Returns a file URL for the current working directory.
-function getURLStringForCwd() {
-  try {
-    return getURLFromFilePath(`${process.cwd()}/`).href;
-  } catch (e) {
-    e.stack;
-    // If the current working directory no longer exists.
-    if (e.code === 'ENOENT') {
-      return undefined;
-    }
-    throw e;
-  }
-}
-
-function normalizeReferrerURL(referrer) {
-  if (typeof referrer === 'string' && path.isAbsolute(referrer)) {
-    return getURLFromFilePath(referrer).href;
-  }
-  return new URL(referrer).href;
-}
-
 /* A Loader instance is used as the main entry point for loading ES modules.
  * Currently, this is a singleton -- there is only one used for loading
  * the main module and everything in its dependency graph. */
 class Loader {
-  constructor(base = getURLStringForCwd()) {
-    if (typeof base !== 'string')
-      throw new errors.TypeError('ERR_INVALID_ARG_TYPE', 'base', 'string');
-
-    this.base = base;
-    this.isMain = true;
-
+  constructor() {
     // methods which translate input code or other information
     // into es modules
     this.translators = translators;
@@ -71,8 +41,9 @@ class Loader {
     this._dynamicInstantiate = undefined;
   }
 
-  async resolve(specifier, parentURL = this.base) {
-    if (typeof parentURL !== 'string')
+  async resolve(specifier, parentURL) {
+    const isMain = parentURL === undefined;
+    if (!isMain && typeof parentURL !== 'string')
       throw new errors.TypeError('ERR_INVALID_ARG_TYPE', 'parentURL', 'string');
 
     const { url, format } =
@@ -93,7 +64,7 @@ class Loader {
     return { url, format };
   }
 
-  async import(specifier, parent = this.base) {
+  async import(specifier, parent) {
     const job = await this.getModuleJob(specifier, parent);
     const module = await job.run();
     return module.namespace();
@@ -107,7 +78,7 @@ class Loader {
       this._dynamicInstantiate = FunctionBind(dynamicInstantiate, null);
   }
 
-  async getModuleJob(specifier, parentURL = this.base) {
+  async getModuleJob(specifier, parentURL) {
     const { url, format } = await this.resolve(specifier, parentURL);
     let job = this.moduleMap.get(url);
     if (job !== undefined)
@@ -134,24 +105,16 @@ class Loader {
     }
 
     let inspectBrk = false;
-    if (this.isMain) {
-      if (process._breakFirstLine) {
-        delete process._breakFirstLine;
-        inspectBrk = true;
-      }
-      this.isMain = false;
+    if (process._breakFirstLine) {
+      delete process._breakFirstLine;
+      inspectBrk = true;
     }
     job = new ModuleJob(this, url, loaderInstance, inspectBrk);
     this.moduleMap.set(url, job);
     return job;
   }
-
-  static registerImportDynamicallyCallback(loader) {
-    setImportModuleDynamicallyCallback(async (referrer, specifier) => {
-      return loader.import(specifier, normalizeReferrerURL(referrer));
-    });
-  }
 }
 
 Object.setPrototypeOf(Loader.prototype, null);
+
 module.exports = Loader;

lib/internal/loader/Translators.js

@@ -19,7 +19,7 @@ const JsonParse = JSON.parse;
 const translators = new SafeMap();
 module.exports = translators;
 
-// Stragety for loading a standard JavaScript module
+// Strategy for loading a standard JavaScript module
 translators.set('esm', async (url) => {
   const source = `${await readFileAsync(new URL(url))}`;
   debug(`Translating StandardModule ${url}`);
@@ -62,7 +62,7 @@ translators.set('builtin', async (url) => {
   });
 });
 
-// Stragety for loading a node native module
+// Strategy for loading a node native module
 translators.set('addon', async (url) => {
   debug(`Translating NativeModule ${url}`);
   return createDynamicModule(['default'], url, (reflect) => {
@@ -74,7 +74,7 @@ translators.set('addon', async (url) => {
   });
 });
 
-// Stragety for loading a JSON file
+// Strategy for loading a JSON file
 translators.set('json', async (url) => {
   debug(`Translating JSONModule ${url}`);
   return createDynamicModule(['default'], url, (reflect) => {

lib/internal/process/modules.js

@@ -0,0 +1,47 @@
+'use strict';
+
+const {
+  setImportModuleDynamicallyCallback
+} = internalBinding('module_wrap');
+
+const { getURLFromFilePath } = require('internal/url');
+const Loader = require('internal/loader/Loader');
+const path = require('path');
+const { URL } = require('url');
+
+function normalizeReferrerURL(referrer) {
+  if (typeof referrer === 'string' && path.isAbsolute(referrer)) {
+    return getURLFromFilePath(referrer).href;
+  }
+  return new URL(referrer).href;
+}
+
+let loaderResolve;
+exports.loaderPromise = new Promise((resolve, reject) => {
+  loaderResolve = resolve;
+});
+
+exports.ESMLoader = undefined;
+
+exports.setup = function() {
+  let ESMLoader = new Loader();
+  const loaderPromise = (async () => {
+    const userLoader = process.binding('config').userLoader;
+    if (userLoader) {
+      const hooks = await ESMLoader.import(
+        userLoader, getURLFromFilePath(`${process.cwd()}/`).href);
+      ESMLoader = new Loader();
+      ESMLoader.hook(hooks);
+      exports.ESMLoader = ESMLoader;
+    }
+    return ESMLoader;
+  })();
+  loaderResolve(loaderPromise);
+
+  setImportModuleDynamicallyCallback(async (referrer, specifier) => {
+    const loader = await loaderPromise;
+    return loader.import(specifier, normalizeReferrerURL(referrer));
+  });
+
+  exports.ESMLoader = ESMLoader;
+};

lib/module.js

@@ -24,7 +24,6 @@
 const NativeModule = require('native_module');
 const util = require('util');
 const { decorateErrorStack } = require('internal/util');
-const internalModule = require('internal/module');
 const { getURLFromFilePath } = require('internal/url');
 const vm = require('vm');
 const assert = require('assert').ok;
@@ -35,6 +34,7 @@ const {
   internalModuleReadFile,
   internalModuleStat
 } = process.binding('fs');
+const internalModule = require('internal/module');
 const preserveSymlinks = !!process.binding('config').preserveSymlinks;
 const experimentalModules = !!process.binding('config').experimentalModules;
 
@@ -43,10 +43,9 @@ const errors = require('internal/errors');
 module.exports = Module;
 
 // these are below module.exports for the circular reference
-const Loader = require('internal/loader/Loader');
+const internalESModule = require('internal/process/modules');
 const ModuleJob = require('internal/loader/ModuleJob');
 const createDynamicModule = require('internal/loader/CreateDynamicModule');
-let ESMLoader;
 
 function stat(filename) {
   filename = path.toNamespacedPath(filename);
@@ -444,7 +443,6 @@ Module._resolveLookupPaths = function(request, parent, newReturn) {
   return (newReturn ? parentDir : [id, parentDir]);
 };
 
-
 // Check the cache for the requested file.
 // 1. If a module already exists in the cache: return its exports object.
 // 2. If the module is native: call `NativeModule.require()` with the
@@ -457,22 +455,10 @@ Module._load = function(request, parent, isMain) {
     debug('Module._load REQUEST %s parent: %s', request, parent.id);
   }
 
-  if (isMain && experimentalModules) {
-    (async () => {
-      // loader setup
-      if (!ESMLoader) {
-        ESMLoader = new Loader();
-        const userLoader = process.binding('config').userLoader;
-        if (userLoader) {
-          ESMLoader.isMain = false;
-          const hooks = await ESMLoader.import(userLoader);
-          ESMLoader = new Loader();
-          ESMLoader.hook(hooks);
-        }
-      }
-      Loader.registerImportDynamicallyCallback(ESMLoader);
-      await ESMLoader.import(getURLFromFilePath(request).pathname);
-    })()
+  if (experimentalModules && isMain) {
+    internalESModule.loaderPromise.then((loader) => {
+      return loader.import(getURLFromFilePath(request).pathname);
+    })
     .catch((e) => {
       decorateErrorStack(e);
       console.error(e);
@@ -575,7 +561,8 @@ Module.prototype.load = function(filename) {
   Module._extensions[extension](this, filename);
   this.loaded = true;
 
-  if (ESMLoader) {
+  if (experimentalModules) {
+    const ESMLoader = internalESModule.ESMLoader;
     const url = getURLFromFilePath(filename);
     const urlString = `${url}`;
     const exports = this.exports;

node.gyp

@@ -115,6 +115,7 @@
       'lib/internal/net.js',
       'lib/internal/module.js',
       'lib/internal/os.js',
+      'lib/internal/process/modules.js',
       'lib/internal/process/next_tick.js',
       'lib/internal/process/promises.js',
       'lib/internal/process/stdio.js',