From a979dd6364770f3ef149f1d24795c825e8ea2d9d Mon Sep 17 00:00:00 2001 From: Jonas Kello Date: Sun, 10 Oct 2021 22:57:04 +0200 Subject: [PATCH] Update esm loader hooks API (#1457) * Initial commit * Update hooks * wip impl of load * Expose old hooks for backward compat * Some logging * Add raw copy of default get format * Adapt defaultGetFormat() from node source * Fix defaultTransformSource * Add missing newline * Fix require * Check node version to avoid deprecation warning * Remove load from old hooks * Add some comments * Use versionGte * Remove logging * Refine comments * Wording * Use format hint if available * One more comment * Nitpicky changes to comments * Update index.ts * lint-fix * attempt at downloading node nightly in tests * fix * fix * Windows install of node nightly * update version checks to be ready for node backporting * Add guards for undefined source * More error info * Skip source transform for builtin and commonjs * Update transpile-only.mjs * Tweak `createEsmHooks` type * fix test to accomodate new api Co-authored-by: Andrew Bradley --- .github/workflows/continuous-integration.yml | 31 ++++++- dist-raw/node-esm-default-get-format.js | 83 ++++++++++++++++++ esm.mjs | 1 + esm/transpile-only.mjs | 1 + package.json | 7 +- src/esm.ts | 92 +++++++++++++++++++- src/index.ts | 39 ++++++--- tests/esm-custom-loader/loader.mjs | 2 +- 8 files changed, 235 insertions(+), 21 deletions(-) create mode 100644 dist-raw/node-esm-default-get-format.js diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index e279668f4..bc954cd9e 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -48,7 +48,7 @@ jobs: matrix: os: [ubuntu, windows] # Don't forget to add all new flavors to this list! - flavor: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + flavor: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11] include: # Node 12.15 # TODO Add comments about why we test 12.15; I think git blame says it's because of an ESM behavioral change that happened at 12.16 @@ -112,14 +112,43 @@ jobs: typescript: next typescriptFlag: next downgradeNpm: true + # Node nightly + - flavor: 11 + node: nightly + nodeFlag: nightly + typescript: latest + typescriptFlag: latest + downgradeNpm: true steps: # checkout code - uses: actions/checkout@v2 # install node - name: Use Node.js ${{ matrix.node }} + if: matrix.node != 'nightly' uses: actions/setup-node@v1 with: node-version: ${{ matrix.node }} + - name: Use Node.js 16, will be subsequently overridden by download of nightly + if: matrix.node == 'nightly' + uses: actions/setup-node@v1 + with: + node-version: 16 + - name: Download Node.js nightly + if: matrix.node == 'nightly' && matrix.os == 'ubuntu' + run: | + export N_PREFIX=$(pwd)/n + npm install -g n + n nightly + sudo cp "${N_PREFIX}/bin/node" "$(which node)" + node --version + - name: Download Node.js nightly + if: matrix.node == 'nightly' && matrix.os == 'windows' + run: | + $version = (Invoke-WebRequest https://nodejs.org/download/nightly/index.json | ConvertFrom-json)[0].version + $url = "https://nodejs.org/download/nightly/$version/win-x64/node.exe" + $targetPath = (Get-Command node.exe).Source + Invoke-WebRequest -Uri $url -OutFile $targetPath + node --version # lint, build, test # Downgrade from npm 7 to 6 because 7 still seems buggy to me - if: ${{ matrix.downgradeNpm }} diff --git a/dist-raw/node-esm-default-get-format.js b/dist-raw/node-esm-default-get-format.js new file mode 100644 index 000000000..d8af956f3 --- /dev/null +++ b/dist-raw/node-esm-default-get-format.js @@ -0,0 +1,83 @@ +// Copied from https://github.com/raw/nodejs/node/v15.3.0/lib/internal/modules/esm/get_format.js +// Then modified to suite our needs. +// Formatting is intentionally bad to keep the diff as small as possible, to make it easier to merge +// upstream changes and understand our modifications. + +'use strict'; +const { + RegExpPrototypeExec, + StringPrototypeStartsWith, +} = require('./node-primordials'); +const { extname } = require('path'); +const { getOptionValue } = require('./node-options'); + +const experimentalJsonModules = getOptionValue('--experimental-json-modules'); +const experimentalSpeciferResolution = + getOptionValue('--experimental-specifier-resolution'); +const experimentalWasmModules = getOptionValue('--experimental-wasm-modules'); +const { getPackageType } = require('./node-esm-resolve-implementation.js').createResolve({tsExtensions: [], jsExtensions: []}); +const { URL, fileURLToPath } = require('url'); +const { ERR_UNKNOWN_FILE_EXTENSION } = require('./node-errors').codes; + +const extensionFormatMap = { + '__proto__': null, + '.cjs': 'commonjs', + '.js': 'module', + '.mjs': 'module' +}; + +const legacyExtensionFormatMap = { + '__proto__': null, + '.cjs': 'commonjs', + '.js': 'commonjs', + '.json': 'commonjs', + '.mjs': 'module', + '.node': 'commonjs' +}; + +if (experimentalWasmModules) + extensionFormatMap['.wasm'] = legacyExtensionFormatMap['.wasm'] = 'wasm'; + +if (experimentalJsonModules) + extensionFormatMap['.json'] = legacyExtensionFormatMap['.json'] = 'json'; + +function defaultGetFormat(url, context, defaultGetFormatUnused) { + if (StringPrototypeStartsWith(url, 'node:')) { + return { format: 'builtin' }; + } + const parsed = new URL(url); + if (parsed.protocol === 'data:') { + const [ , mime ] = RegExpPrototypeExec( + /^([^/]+\/[^;,]+)(?:[^,]*?)(;base64)?,/, + parsed.pathname, + ) || [ null, null, null ]; + const format = ({ + '__proto__': null, + 'text/javascript': 'module', + 'application/json': experimentalJsonModules ? 'json' : null, + 'application/wasm': experimentalWasmModules ? 'wasm' : null + })[mime] || null; + return { format }; + } else if (parsed.protocol === 'file:') { + const ext = extname(parsed.pathname); + let format; + if (ext === '.js') { + format = getPackageType(parsed.href) === 'module' ? 'module' : 'commonjs'; + } else { + format = extensionFormatMap[ext]; + } + if (!format) { + if (experimentalSpeciferResolution === 'node') { + process.emitWarning( + 'The Node.js specifier resolution in ESM is experimental.', + 'ExperimentalWarning'); + format = legacyExtensionFormatMap[ext]; + } else { + throw new ERR_UNKNOWN_FILE_EXTENSION(ext, fileURLToPath(url)); + } + } + return { format: format || null }; + } + return { format: null }; +} +exports.defaultGetFormat = defaultGetFormat; diff --git a/esm.mjs b/esm.mjs index 2a11ac36e..4d404070d 100644 --- a/esm.mjs +++ b/esm.mjs @@ -6,6 +6,7 @@ const require = createRequire(fileURLToPath(import.meta.url)); const esm = require('./dist/esm'); export const { resolve, + load, getFormat, transformSource, } = esm.registerAndCreateEsmHooks(); diff --git a/esm/transpile-only.mjs b/esm/transpile-only.mjs index c19132284..07b2c7ae6 100644 --- a/esm/transpile-only.mjs +++ b/esm/transpile-only.mjs @@ -6,6 +6,7 @@ const require = createRequire(fileURLToPath(import.meta.url)); const esm = require('../dist/esm'); export const { resolve, + load, getFormat, transformSource, } = esm.registerAndCreateEsmHooks({ transpileOnly: true }); diff --git a/package.json b/package.json index f00637cbc..8ed02f65f 100644 --- a/package.json +++ b/package.json @@ -70,10 +70,9 @@ "pre-debug": "npm run build-tsc && npm run build-pack", "coverage-report": "nyc report --reporter=lcov", "prepare": "npm run clean && npm run build-nopack", - "api-extractor": "api-extractor run --local --verbose" - }, - "engines": { - "node": ">=12.0.0" + "api-extractor": "api-extractor run --local --verbose", + "esm-usage-example": "npm run build-tsc && cd esm-usage-example && node --experimental-specifier-resolution node --loader ../esm.mjs ./index", + "esm-usage-example2": "npm run build-tsc && cd tests && TS_NODE_PROJECT=./module-types/override-to-cjs/tsconfig.json node --loader ../esm.mjs ./module-types/override-to-cjs/test.cjs" }, "repository": { "type": "git", diff --git a/src/esm.ts b/src/esm.ts index ab1638eb1..5502d0155 100644 --- a/src/esm.ts +++ b/src/esm.ts @@ -1,4 +1,10 @@ -import { getExtensions, register, RegisterOptions, Service } from './index'; +import { + register, + getExtensions, + RegisterOptions, + Service, + versionGteLt, +} from './index'; import { parse as parseUrl, format as formatUrl, @@ -12,9 +18,24 @@ import { normalizeSlashes } from './util'; const { createResolve, } = require('../dist-raw/node-esm-resolve-implementation'); +const { defaultGetFormat } = require('../dist-raw/node-esm-default-get-format'); // Note: On Windows, URLs look like this: file:///D:/dev/@TypeStrong/ts-node-examples/foo.ts +// NOTE ABOUT MULTIPLE EXPERIMENTAL LOADER APIS +// +// At the time of writing, this file implements 2x different loader APIs. +// Node made a breaking change to the loader API in https://github.com/nodejs/node/pull/37468 +// +// We check the node version number and export either the *old* or the *new* API. +// +// Today, we are implementing the *new* API on top of our implementation of the *old* API, +// which relies on copy-pasted code from the *old* hooks implementation in node. +// +// In the future, we will likely invert this: we will copy-paste the *new* API implementation +// from node, build our implementation of the *new* API on top of it, and implement the *old* +// hooks API as a shim to the *new* API. + /** @internal */ export function registerAndCreateEsmHooks(opts?: RegisterOptions) { // Automatically performs registration just like `-r ts-node/register` @@ -32,7 +53,24 @@ export function createEsmHooks(tsNodeService: Service) { preferTsExts: tsNodeService.options.preferTsExts, }); - return { resolve, getFormat, transformSource }; + // The hooks API changed in node version X so we need to check for backwards compatibility. + // TODO: When the new API is backported to v12, v14, v16, update these version checks accordingly. + const newHooksAPI = + versionGteLt(process.versions.node, '17.0.0') || + versionGteLt(process.versions.node, '16.999.999', '17.0.0') || + versionGteLt(process.versions.node, '14.999.999', '15.0.0') || + versionGteLt(process.versions.node, '12.999.999', '13.0.0'); + + // Explicit return type to avoid TS's non-ideal inferred type + const hooksAPI: { + resolve: typeof resolve; + getFormat: typeof getFormat | undefined; + transformSource: typeof transformSource | undefined; + load: typeof load | undefined; + } = newHooksAPI + ? { resolve, load, getFormat: undefined, transformSource: undefined } + : { resolve, getFormat, transformSource, load: undefined }; + return hooksAPI; function isFileUrlOrNodeStyleSpecifier(parsed: UrlWithStringQuery) { // We only understand file:// URLs, but in node, the specifier can be a node-style `./foo` or `foo` @@ -76,6 +114,52 @@ export function createEsmHooks(tsNodeService: Service) { ); } + // `load` from new loader hook API (See description at the top of this file) + async function load( + url: string, + context: { format: Format | null | undefined }, + defaultLoad: typeof load + ): Promise<{ format: Format; source: string | Buffer | undefined }> { + // If we get a format hint from resolve() on the context then use it + // otherwise call the old getFormat() hook using node's old built-in defaultGetFormat() that ships with ts-node + const format = + context.format ?? + (await getFormat(url, context, defaultGetFormat)).format; + + let source = undefined; + if (format !== 'builtin' && format !== 'commonjs') { + // Call the new defaultLoad() to get the source + const { source: rawSource } = await defaultLoad( + url, + { format }, + defaultLoad + ); + + if (rawSource === undefined || rawSource === null) { + throw new Error( + `Failed to load raw source: Format was '${format}' and url was '${url}''.` + ); + } + + // Emulate node's built-in old defaultTransformSource() so we can re-use the old transformSource() hook + const defaultTransformSource: typeof transformSource = async ( + source, + _context, + _defaultTransformSource + ) => ({ source }); + + // Call the old hook + const { source: transformedSource } = await transformSource( + rawSource, + { url, format }, + defaultTransformSource + ); + source = transformedSource; + } + + return { format, source }; + } + type Format = 'builtin' | 'commonjs' | 'dynamic' | 'json' | 'module' | 'wasm'; async function getFormat( url: string, @@ -129,6 +213,10 @@ export function createEsmHooks(tsNodeService: Service) { context: { url: string; format: Format }, defaultTransformSource: typeof transformSource ): Promise<{ source: string | Buffer }> { + if (source === null || source === undefined) { + throw new Error('No source'); + } + const defer = () => defaultTransformSource(source, context, defaultTransformSource); diff --git a/src/index.ts b/src/index.ts index 5f9c5c9aa..7570d07a0 100644 --- a/src/index.ts +++ b/src/index.ts @@ -47,18 +47,31 @@ export type { const engineSupportsPackageTypeField = parseInt(process.versions.node.split('.')[0], 10) >= 12; -function versionGte(version: string, requirement: string) { - const [major, minor, patch, extra] = version - .split(/[\.-]/) - .map((s) => parseInt(s, 10)); - const [reqMajor, reqMinor, reqPatch] = requirement - .split('.') - .map((s) => parseInt(s, 10)); - return ( - major > reqMajor || - (major === reqMajor && - (minor > reqMinor || (minor === reqMinor && patch >= reqPatch))) - ); +/** @internal */ +export function versionGteLt( + version: string, + gteRequirement: string, + ltRequirement?: string +) { + const [major, minor, patch, extra] = parse(version); + const [gteMajor, gteMinor, gtePatch] = parse(gteRequirement); + const isGte = + major > gteMajor || + (major === gteMajor && + (minor > gteMinor || (minor === gteMinor && patch >= gtePatch))); + let isLt = true; + if (ltRequirement) { + const [ltMajor, ltMinor, ltPatch] = parse(ltRequirement); + isLt = + major < ltMajor || + (major === ltMajor && + (minor < ltMinor || (minor === ltMinor && patch < ltPatch))); + } + return isGte && isLt; + + function parse(requirement: string) { + return requirement.split(/[\.-]/).map((s) => parseInt(s, 10)); + } } /** @@ -570,7 +583,7 @@ export function create(rawOptions: CreateOptions = {}): Service { ); } // Top-level await was added in TS 3.8 - const tsVersionSupportsTla = versionGte(ts.version, '3.8.0'); + const tsVersionSupportsTla = versionGteLt(ts.version, '3.8.0'); if (options.experimentalReplAwait === true && !tsVersionSupportsTla) { throw new Error( 'Experimental REPL await is not compatible with TypeScript versions older than 3.8' diff --git a/tests/esm-custom-loader/loader.mjs b/tests/esm-custom-loader/loader.mjs index 3b0ee683c..bf82e766b 100755 --- a/tests/esm-custom-loader/loader.mjs +++ b/tests/esm-custom-loader/loader.mjs @@ -11,6 +11,6 @@ const tsNodeInstance = register({ }, }); -export const { resolve, getFormat, transformSource } = createEsmHooks( +export const { resolve, getFormat, transformSource, load } = createEsmHooks( tsNodeInstance );