Skip to content

Commit

Permalink
refactor(bundle-source): Isolate transform (#2685)
Browse files Browse the repository at this point in the history 
This isolates a transformSource function so that it can be applied for
the Endo Base-64 Zip bundle format in a subsequent change.
The change reveals that the transformSource function is necessarilly
async--it uses the Mozilla SourceMapConsumer, which in turn relies on a
pool of workers for performance.
This in turn suggests the next phase of work will be adding support for
asynchronous transforms to the Endo archiver in the Compartment Mapper.

Refactoring revealed an opportunity to parallelize transforms, which
might better leverage the source map worker pool, and also revealed a
resource leak since we did not previously release source map workers.
  • Loading branch information
@kriskowal
kriskowal authored Mar 19, 2021
1 parent f64c745 commit b5bda04
Showing 1 changed file with 115 additions and 88 deletions.
203 changes: 115 additions & 88 deletions packages/bundle-source/src/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,102 @@ export function tildotPlugin() {
},
};
}

function rewriteComment(node, unmapLoc) {
node.type = 'CommentBlock';
// Within comments...
node.value = node.value
// ...strip extraneous comment whitespace
.replace(/^\s+/gm, ' ')
// ...replace HTML comments with a defanged version to pass SES restrictions.
.replace(HTML_COMMENT_START_RE, '<!X-')
.replace(HTML_COMMENT_END_RE, '-X>')
// ...replace import expressions with a defanged version to pass SES restrictions.
.replace(IMPORT_RE, 'X$1$2')
// ...replace end-of-comment markers
.replace(/\*\//g, '*X/');
if (unmapLoc) {
unmapLoc(node.loc);
}
// console.log(JSON.stringify(node, undefined, 2));
}

async function makeLocationUnmapper({ sourceMap, ast }) {
// We rearrange the rolled-up chunk according to its sourcemap to move
// its source lines back to the right place.
// eslint-disable-next-line no-await-in-loop
const consumer = await new SourceMapConsumer(sourceMap);
try {
const unmapped = new WeakSet();
let lastPos = { ...ast.loc.start };
return loc => {
if (!loc || unmapped.has(loc)) {
return;
}
// Make sure things start at least at the right place.
loc.end = { ...loc.start };
for (const pos of ['start', 'end']) {
if (loc[pos]) {
const newPos = consumer.originalPositionFor(loc[pos]);
if (newPos.source !== null) {
lastPos = {
line: newPos.line,
column: newPos.column,
};
}
loc[pos] = lastPos;
}
}
unmapped.add(loc);
};
} finally {
consumer.destroy();
}
}

function transformAst(ast, unmapLoc) {
babelTraverse(ast, {
enter(p) {
const { loc, leadingComments, trailingComments } = p.node;
if (p.node.comments) {
p.node.comments = [];
}
// Rewrite all comments.
(leadingComments || []).forEach(node => rewriteComment(node, unmapLoc));
if (p.node.type.startsWith('Comment')) {
rewriteComment(p.node, unmapLoc);
}
// If not a comment, and we are unmapping the source maps,
// then do it for this location.
if (unmapLoc) {
unmapLoc(loc);
}
(trailingComments || []).forEach(node => rewriteComment(node, unmapLoc));
},
});
}

async function transformSource(code, { sourceMap, useLocationUnmap }) {
// Parse the rolled-up chunk with Babel.
// We are prepared for different module systems.
const ast = (babelParser.parse || babelParser)(code, {
plugins: ['bigInt'],
});

let unmapLoc;
if (useLocationUnmap) {
unmapLoc = await makeLocationUnmapper({
sourceMap,
ast,
});
}

transformAst(ast, unmapLoc);

// Now generate the sources with the new positions.
return babelGenerate(ast, { retainLines: true });
}

/** @type {BundleSource} */
export default async function bundleSource(
startFilename,
Expand Down Expand Up @@ -86,97 +182,28 @@ export default async function bundleSource(
// Create a source bundle.
const sourceBundle = {};
let entrypoint;
for (const chunk of output) {
if (chunk.isAsset) {
throw Error(`unprepared for assets: ${chunk.fileName}`);
}
const { code, fileName, isEntry } = chunk;
if (isEntry) {
entrypoint = fileName;
}

// Parse the rolled-up chunk with Babel.
// We are prepared for different module systems.
const ast = (babelParser.parse || babelParser)(code, {
plugins: ['bigInt'],
});

let unmapLoc;
if (
moduleFormat === 'nestedEvaluate' &&
!fileName.startsWith('_virtual/')
) {
// We rearrange the rolled-up chunk according to its sourcemap to move
// its source lines back to the right place.
// eslint-disable-next-line no-await-in-loop
const consumer = await new SourceMapConsumer(chunk.map);
const unmapped = new WeakSet();
let lastPos = { ...ast.loc.start };
unmapLoc = loc => {
if (!loc || unmapped.has(loc)) {
return;
}
// Make sure things start at least at the right place.
loc.end = { ...loc.start };
for (const pos of ['start', 'end']) {
if (loc[pos]) {
const newPos = consumer.originalPositionFor(loc[pos]);
if (newPos.source !== null) {
lastPos = {
line: newPos.line,
column: newPos.column,
};
}
loc[pos] = lastPos;
}
}
unmapped.add(loc);
};
}

const rewriteComment = node => {
node.type = 'CommentBlock';
// Within comments...
node.value = node.value
// ...strip extraneous comment whitespace
.replace(/^\s+/gm, ' ')
// ...replace HTML comments with a defanged version to pass SES restrictions.
.replace(HTML_COMMENT_START_RE, '<!X-')
.replace(HTML_COMMENT_END_RE, '-X>')
// ...replace import expressions with a defanged version to pass SES restrictions.
.replace(IMPORT_RE, 'X$1$2')
// ...replace end-of-comment markers
.replace(/\*\//g, '*X/');
if (unmapLoc) {
unmapLoc(node.loc);
await Promise.all(
output.map(async chunk => {
if (chunk.isAsset) {
throw Error(`unprepared for assets: ${chunk.fileName}`);
}
const { code, fileName, isEntry } = chunk;
if (isEntry) {
entrypoint = fileName;
}
// console.log(JSON.stringify(node, undefined, 2));
};

babelTraverse(ast, {
enter(p) {
const { loc, leadingComments, trailingComments } = p.node;
if (p.node.comments) {
p.node.comments = [];
}
// Rewrite all comments.
(leadingComments || []).forEach(rewriteComment);
if (p.node.type.startsWith('Comment')) {
rewriteComment(p.node);
}
// If not a comment, and we are unmapping the source maps,
// then do it for this location.
if (unmapLoc) {
unmapLoc(loc);
}
(trailingComments || []).forEach(rewriteComment);
},
});
const useLocationUnmap =
moduleFormat === 'nestedEvaluate' && !fileName.startsWith('_virtual/');

// Now generate the sources with the new positions.
sourceBundle[fileName] = babelGenerate(ast, { retainLines: true }).code;
// console.log(`==== sourceBundle[${fileName}]\n${sourceBundle[fileName]}\n====`);
}
const { code: transformedCode } = await transformSource(code, {
sourceMap: chunk.map,
useLocationUnmap,
});
sourceBundle[fileName] = transformedCode;

// console.log(`==== sourceBundle[${fileName}]\n${sourceBundle[fileName]}\n====`);
}),
);

if (!entrypoint) {
throw Error('No entrypoint found in output bundle');
Expand Down

0 comments on commit b5bda04

Please sign in to comment.