Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Modules bootstrap refactoring #29937

Closed
wants to merge 14 commits into from
8 changes: 4 additions & 4 deletions doc/api/esm.md
Original file line number Diff line number Diff line change
Expand Up @@ -886,13 +886,13 @@ _isMain_ is **true** when resolving the Node.js application entry point.
> 1. Throw a _Module Not Found_ error.
> 1. If _pjson.exports_ is not **null** or **undefined**, then
> 1. If _pjson.exports_ is a String or Array, then
> 1. Return _PACKAGE_EXPORTS_TARGET_RESOLVE(packageURL, pjson.exports,
> "")_.
> 1. Return **PACKAGE_EXPORTS_TARGET_RESOLVE**(_packageURL_,
> _pjson.exports_, "")_.
> 1. If _pjson.exports is an Object, then
> 1. If _pjson.exports_ contains a _"."_ property, then
> 1. Let _mainExport_ be the _"."_ property in _pjson.exports_.
> 1. Return _PACKAGE_EXPORTS_TARGET_RESOLVE(packageURL, mainExport,
> "")_.
> 1. Return **PACKAGE_EXPORTS_TARGET_RESOLVE**(_packageURL_,
> _mainExport_, "")_.
> 1. If _pjson.main_ is a String, then
> 1. Let _resolvedMain_ be the URL resolution of _packageURL_, "/", and
> _pjson.main_.
Expand Down
5 changes: 4 additions & 1 deletion lib/internal/bootstrap/loaders.js
Original file line number Diff line number Diff line change
Expand Up @@ -220,7 +220,10 @@ NativeModule.prototype.compileForPublicLoader = function(needToSyncExports) {
this.compile();
if (needToSyncExports) {
if (!this.exportKeys) {
this.exportKeys = Object.keys(this.exports);
// When using --expose-internals, we do not want to reflect the named
// exports from core modules as this can trigger unnecessary getters.
const internal = this.id.startsWith('internal/');
this.exportKeys = internal ? [] : Object.keys(this.exports);
}
this.getESMFacade();
this.syncExports();
Expand Down
69 changes: 60 additions & 9 deletions lib/internal/bootstrap/pre_execution.js
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ const { Object, SafeWeakMap } = primordials;
const { getOptionValue } = require('internal/options');
const { Buffer } = require('buffer');
const { ERR_MANIFEST_ASSERT_INTEGRITY } = require('internal/errors').codes;
const path = require('path');

function prepareMainThreadExecution(expandArgv1 = false) {
// Patch the process object with legacy properties and normalizations
Expand Down Expand Up @@ -404,7 +405,6 @@ function initializeESMLoader() {
'The ESM module loader is experimental.',
'ExperimentalWarning', undefined);
}

const {
setImportModuleDynamicallyCallback,
setInitializeImportMetaObjectCallback
Expand All @@ -414,14 +414,6 @@ function initializeESMLoader() {
// track of for different ESM modules.
setInitializeImportMetaObjectCallback(esm.initializeImportMetaObject);
setImportModuleDynamicallyCallback(esm.importModuleDynamicallyCallback);
const userLoader = getOptionValue('--experimental-loader');
// If --experimental-loader is specified, create a loader with user hooks.
// Otherwise create the default loader.
if (userLoader) {
const { emitExperimentalWarning } = require('internal/util');
emitExperimentalWarning('--experimental-loader');
}
esm.initializeLoader(process.cwd(), userLoader);
}
}

Expand All @@ -446,11 +438,70 @@ function loadPreloadModules() {
}
}

function resolveMainPath(main) {
const { toRealPath, Module: CJSModule } =
require('internal/modules/cjs/loader');

// Note extension resolution for the main entry point can be deprecated in a
// future major.
let mainPath = CJSModule._findPath(path.resolve(main), null, true);
if (!mainPath)
return;

const preserveSymlinksMain = getOptionValue('--preserve-symlinks-main');
if (!preserveSymlinksMain)
mainPath = toRealPath(mainPath);

return mainPath;
}

function shouldUseESMLoader(mainPath) {
const experimentalModules = getOptionValue('--experimental-modules');
if (!experimentalModules)
return false;
const userLoader = getOptionValue('--experimental-loader');
if (userLoader)
return true;
// Determine the module format of the main
if (mainPath && mainPath.endsWith('.mjs'))
return true;
if (!mainPath || mainPath.endsWith('.cjs'))
return false;
const { readPackageScope } = require('internal/modules/cjs/loader');
const pkg = readPackageScope(mainPath);
return pkg && pkg.data.type === 'module';
}

function runMainESM(mainPath) {
const esmLoader = require('internal/process/esm_loader');
const { pathToFileURL } = require('internal/url');
const { hasUncaughtExceptionCaptureCallback } =
require('internal/process/execution');
return esmLoader.initializeLoader().then(() => {
const main = path.isAbsolute(mainPath) ?
pathToFileURL(mainPath).href : mainPath;
return esmLoader.ESMLoader.import(main).catch((e) => {
if (hasUncaughtExceptionCaptureCallback()) {
process._fatalException(e);
return;
}
internalBinding('errors').triggerUncaughtException(
e,
true /* fromPromise */
);
});
});
}


module.exports = {
patchProcessObject,
resolveMainPath,
runMainESM,
setupCoverageHooks,
setupWarningHandler,
setupDebugEnv,
shouldUseESMLoader,
prepareMainThreadExecution,
initializeDeprecations,
initializeESMLoader,
Expand Down
9 changes: 4 additions & 5 deletions lib/internal/main/run_main_module.js
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,7 @@ const CJSModule = require('internal/modules/cjs/loader').Module;

markBootstrapComplete();

// Note: this actually tries to run the module as a ESM first if
guybedford marked this conversation as resolved.
Show resolved Hide resolved
// --experimental-modules is on.
// TODO(joyeecheung): can we move that logic to here? Note that this
// is an undocumented method available via `require('module').runMain`
CJSModule.runMain();
// Note: this loads the module through the ESM loader if
// --experimental-loader is provided or --experimental-modules is on
// and the module is determined to be an ES module
CJSModule.runMain(process.argv[1]);
5 changes: 3 additions & 2 deletions lib/internal/main/worker_thread.js
Original file line number Diff line number Diff line change
Expand Up @@ -140,8 +140,9 @@ port.on('message', (message) => {
const { evalScript } = require('internal/process/execution');
evalScript('[worker eval]', filename);
} else {
process.argv[1] = filename; // script filename
require('module').runMain();
// script filename
const CJSModule = require('internal/modules/cjs/loader').Module;
CJSModule.runMain(process.argv[1] = filename);
}
} else if (message.type === STDIO_PAYLOAD) {
const { stream, chunk, encoding } = message;
Expand Down
66 changes: 36 additions & 30 deletions lib/internal/modules/cjs/loader.js
Original file line number Diff line number Diff line change
Expand Up @@ -69,9 +69,14 @@ const {
ERR_REQUIRE_ESM
} = require('internal/errors').codes;
const { validateString } = require('internal/validators');
const {
resolveMainPath,
shouldUseESMLoader,
runMainESM
} = require('internal/bootstrap/pre_execution');
const pendingDeprecation = getOptionValue('--pending-deprecation');

module.exports = { wrapSafe, Module };
module.exports = { wrapSafe, Module, toRealPath, readPackageScope };

let asyncESM, ModuleJob, ModuleWrap, kInstantiated;

Expand Down Expand Up @@ -810,6 +815,10 @@ Module.prototype.load = function(filename) {
this.paths = Module._nodeModulePaths(path.dirname(filename));

const extension = findLongestRegisteredExtension(filename);
// allow .mjs to be overridden
if (filename.endsWith('.mjs') && !Module._extensions['.mjs']) {
throw new ERR_REQUIRE_ESM(filename);
}
Module._extensions[extension](this, filename);
this.loaded = true;

Expand All @@ -823,14 +832,19 @@ Module.prototype.load = function(filename) {
if (module !== undefined && module.module !== undefined) {
if (module.module.getStatus() >= kInstantiated)
module.module.setExport('default', exports);
} else { // preemptively cache
} else {
// Preemptively cache
// We use a function to defer promise creation for async hooks.
ESMLoader.moduleMap.set(
url,
new ModuleJob(ESMLoader, url, () =>
// Module job creation will start promises.
// We make it a function to lazily trigger those promises
// for async hooks compatibility.
() => new ModuleJob(ESMLoader, url, () =>
new ModuleWrap(url, undefined, ['default'], function() {
this.setExport('default', exports);
})
)
, false /* isMain */, false /* inspectBrk */)
);
}
}
Expand Down Expand Up @@ -859,15 +873,15 @@ Module.prototype.require = function(id) {
var resolvedArgv;
let hasPausedEntry = false;

function wrapSafe(filename, content) {
function wrapSafe(filename, content, cjsModuleInstance) {
if (patched) {
const wrapper = Module.wrap(content);
return vm.runInThisContext(wrapper, {
filename,
lineOffset: 0,
displayErrors: true,
importModuleDynamically: experimentalModules ? async (specifier) => {
const loader = await asyncESM.loaderPromise;
const loader = asyncESM.ESMLoader;
return loader.import(specifier, normalizeReferrerURL(filename));
} : undefined,
});
Expand All @@ -892,17 +906,16 @@ function wrapSafe(filename, content) {
]
);
} catch (err) {
if (experimentalModules) {
if (experimentalModules && process.mainModule === cjsModuleInstance)
enrichCJSError(err);
}
throw err;
}

if (experimentalModules) {
const { callbackMap } = internalBinding('module_wrap');
callbackMap.set(compiled.cacheKey, {
importModuleDynamically: async (specifier) => {
const loader = await asyncESM.loaderPromise;
const loader = asyncESM.ESMLoader;
return loader.import(specifier, normalizeReferrerURL(filename));
}
});
Expand All @@ -925,7 +938,7 @@ Module.prototype._compile = function(content, filename) {
}

maybeCacheSourceMap(filename, content, this);
const compiledWrapper = wrapSafe(filename, content);
const compiledWrapper = wrapSafe(filename, content, this);

var inspectorWrapper = null;
if (getOptionValue('--inspect-brk') && process._eval == null) {
Expand Down Expand Up @@ -981,7 +994,11 @@ Module._extensions['.js'] = function(module, filename) {
'files in that package scope as ES modules.\nInstead rename ' +
`${basename} to end in .cjs, change the requiring code to use ` +
'import(), or remove "type": "module" from ' +
`${path.resolve(pkg.path, 'package.json')}.`
`${path.resolve(pkg.path, 'package.json')}.`,
undefined,
undefined,
undefined,
true
);
warnRequireESM = false;
}
Expand Down Expand Up @@ -1024,26 +1041,15 @@ Module._extensions['.node'] = function(module, filename) {
return process.dlopen(module, path.toNamespacedPath(filename));
};

Module._extensions['.mjs'] = function(module, filename) {
throw new ERR_REQUIRE_ESM(filename);
};

// Bootstrap main module.
Module.runMain = function() {
// Load the main module--the command line argument.
if (experimentalModules) {
asyncESM.loaderPromise.then((loader) => {
return loader.import(pathToFileURL(process.argv[1]).href);
})
.catch((e) => {
internalBinding('errors').triggerUncaughtException(
e,
true /* fromPromise */
);
});
return;
Module.runMain = function(main = process.argv[1]) {
const resolvedMain = resolveMainPath(main);
const useESMLoader = shouldUseESMLoader(resolvedMain);
if (useESMLoader) {
runMainESM(resolvedMain || main);
} else {
Module._load(main, null, true);
}
Module._load(process.argv[1], null, true);
};

function createRequireFromPath(filename) {
Expand Down Expand Up @@ -1164,7 +1170,7 @@ Module.Module = Module;

// We have to load the esm things after module.exports!
if (experimentalModules) {
asyncESM = require('internal/process/esm_loader');
ModuleJob = require('internal/modules/esm/module_job');
asyncESM = require('internal/process/esm_loader');
({ ModuleWrap, kInstantiated } = internalBinding('module_wrap'));
}
11 changes: 9 additions & 2 deletions lib/internal/modules/esm/loader.js
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ const createDynamicModule = require(
'internal/modules/esm/create_dynamic_module');
const { translators } = require('internal/modules/esm/translators');
const { ModuleWrap } = internalBinding('module_wrap');
const { getOptionValue } = require('internal/options');

const debug = require('internal/util/debuglog').debuglog('esm');

Expand Down Expand Up @@ -118,7 +119,7 @@ class Loader {
url = pathToFileURL(`${process.cwd()}/[eval${++this.evalIndex}]`).href
) {
const evalInstance = (url) => new ModuleWrap(url, undefined, source, 0, 0);
const job = new ModuleJob(this, url, evalInstance, false);
const job = new ModuleJob(this, url, evalInstance, false, false);
this.moduleMap.set(url, job);
const { module, result } = await job.run();
return {
Expand Down Expand Up @@ -146,6 +147,9 @@ class Loader {
async getModuleJob(specifier, parentURL) {
const { url, format } = await this.resolve(specifier, parentURL);
let job = this.moduleMap.get(url);
// CommonJS will set functions for lazy job evaluation.
if (typeof job === 'function')
this.moduleMap.set(url, job = job());
if (job !== undefined)
return job;

Expand All @@ -169,7 +173,10 @@ class Loader {
loaderInstance = translators.get(format);
}

job = new ModuleJob(this, url, loaderInstance, parentURL === undefined);
const inspectBrk = parentURL === undefined &&
format === 'module' && getOptionValue('--inspect-brk');
job = new ModuleJob(this, url, loaderInstance, parentURL === undefined,
inspectBrk);
this.moduleMap.set(url, job);
return job;
}
Expand Down
8 changes: 4 additions & 4 deletions lib/internal/modules/esm/module_job.js
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ const {
const { ModuleWrap } = internalBinding('module_wrap');

const { decorateErrorStack } = require('internal/util');
const { getOptionValue } = require('internal/options');
const assert = require('internal/assert');
const resolvedPromise = SafePromise.resolve();

Expand All @@ -22,9 +21,10 @@ let hasPausedEntry = false;
class ModuleJob {
// `loader` is the Loader instance used for loading dependencies.
// `moduleProvider` is a function
constructor(loader, url, moduleProvider, isMain) {
constructor(loader, url, moduleProvider, isMain, inspectBrk) {
this.loader = loader;
this.isMain = isMain;
this.inspectBrk = inspectBrk;

// This is a Promise<{ module, reflect }>, whose fields will be copied
// onto `this` by `link()` below once it has been resolved.
Expand Down Expand Up @@ -83,12 +83,12 @@ class ModuleJob {
};
await addJobsToDependencyGraph(this);
try {
if (!hasPausedEntry && this.isMain && getOptionValue('--inspect-brk')) {
if (!hasPausedEntry && this.inspectBrk) {
hasPausedEntry = true;
const initWrapper = internalBinding('inspector').callAndPauseOnStart;
initWrapper(this.module.instantiate, this.module);
} else {
this.module.instantiate();
this.module.instantiate(true);
}
} catch (e) {
decorateErrorStack(e);
Expand Down
3 changes: 2 additions & 1 deletion lib/internal/modules/esm/module_map.js
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,8 @@ class ModuleMap extends SafeMap {
}
set(url, job) {
validateString(url, 'url');
if (job instanceof ModuleJob !== true) {
if (job instanceof ModuleJob !== true &&
typeof job !== 'function') {
throw new ERR_INVALID_ARG_TYPE('job', 'ModuleJob', job);
}
debug(`Storing ${url} in ModuleMap`);
Expand Down
Loading