Skip to content

Commit

Permalink
lbt/bundle/Builder: Add source map transformation
Browse files Browse the repository at this point in the history 
Inspired by #282 and
https://github.com/SAP/ui5-builder/compare/sourcemap
  • Loading branch information
@RandomByte
RandomByte committed Nov 2, 2021
1 parent 4701487 commit b030373
Show file tree
Hide file tree
Showing 9 changed files with 919 additions and 92 deletions.
213 changes: 142 additions & 71 deletions lib/lbt/bundle/Builder.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,10 @@
// for consistency of write calls, we generally allow template literals
"use strict";

const terser = require("terser");
const path = require("path");
const {pd} = require("pretty-data");
const {parseJS, Syntax} = require("../utils/parseUtils");
// const MOZ_SourceMap = require("source-map");
const {encode: encodeMappings, decode: decodeMappings} = require("sourcemap-codec");

const {isMethodCall} = require("../utils/ASTUtils");
const ModuleName = require("../utils/ModuleName");
Expand All @@ -18,7 +18,7 @@ const {SectionType} = require("./BundleDefinition");
const BundleWriter = require("./BundleWriter");
const log = require("@ui5/logger").getLogger("lbt:bundle:Builder");

const copyrightCommentsPattern = /copyright|\(c\)(?:[0-9]+|\s+[0-9A-za-z])|released under|license|\u00a9|^@ui5-bundle-raw-include |^@ui5-bundle /i;
const sourcemapPattern = /\/\/# sourceMappingURL=data:application\/json;charset=utf-8;base64,(.+)/i;
const xmlHtmlPrePattern = /<(?:\w+:)?pre\b/;

const strReplacements = {
Expand Down Expand Up @@ -147,6 +147,7 @@ class BundleBuilder {

this.options = options || {};
this.optimize = !!this.options.optimize;
this.options.sourceMap = this.options.sourceMap === undefined ? true : this.options.sourceMap;

// when decorateBootstrapModule is set to false, we don't write the optimized flag
// and don't write the try catch wrapper
Expand All @@ -156,6 +157,14 @@ class BundleBuilder {
// TODO is the following condition ok or should the availability of jquery.sap.global.js be configurable?
this.jqglobalAvailable = !resolvedModule.containsGlobal;
this.openModule(resolvedModule.name);

this._sourceMap = {
version: 3,
file: path.posix.basename(resolvedModule.name),
sections: [],
};
this._bundleName = resolvedModule.name;

let bundleInfos = [];
// create all sections in sequence
for ( const section of resolvedModule.sections ) {
Expand Down Expand Up @@ -184,6 +193,7 @@ class BundleBuilder {
return {
name: module.name,
content: this.outW.toString(),
sourceMap: this.options.sourceMap ? JSON.stringify(this._sourceMap) : null,
bundleInfo: bundleInfo
};
}
Expand Down Expand Up @@ -213,11 +223,9 @@ class BundleBuilder {
this.outW.writeln(`if (oError.name != "Restart") { throw oError; }`);
this.outW.writeln(`}`);
}
/* NODE-TODO
if ( writeSourceMap && writeSourceMapAnnotation ) {
outW.ensureNewLine();
outW.write("//# sourceMappingURL=" + moduleName.getBaseName().replaceFirst("\\.js$", ".js.map"));
}*/
if (this.options.sourceMap) {
this.outW.writeln(`//# sourceMappingURL=${path.basename(resolvedModule.name)}.map`);
}
}

addSection(section) {
Expand Down Expand Up @@ -281,12 +289,8 @@ class BundleBuilder {
}

async writeRawModule(module, resource) {
let fileContent = await resource.buffer();
if ( /\.js$/.test(module) ) {
fileContent = await this.compressJS( fileContent, resource );
}
this.outW.ensureNewLine();
this.outW.write( fileContent );
this.outW.write( (await resource.buffer()).toString() );
}

async writePreloadFunction(section) {
Expand Down Expand Up @@ -330,60 +334,80 @@ class BundleBuilder {
// this.afterWriteFunctionPreloadSection();
}

async compressJS(fileContent, resource) {
if ( this.optimize ) {
const result = await terser.minify({
[resource.name]: String(fileContent)
}, {
compress: false, // TODO configure?
output: {
comments: copyrightCommentsPattern,
wrap_func_args: false
}
// , outFileName: resource.name
// , outSourceMap: true
});
// console.log(result.map);
// const map = new MOZ_SourceMap.SourceMapConsumer(result.map);
// map.eachMapping(function (m) { console.log(m); }); // console.log(map);
fileContent = result.code;
// throw new Error();
}
return fileContent;
}

beforeWriteFunctionPreloadSection(sequence) {
// simple version: just sort alphabetically
sequence.sort();
}

async addSourceMap(map) {
if (!map) {
throw new Error("No source map provided");
}
this._sourceMap.sections.push({
offset: {
line: this.outW.lineOffset,
column: this.outW.columnOffset
},
map
});
}

async rewriteAMDModules(sequence) {
if ( this.options.usePredefineCalls ) {
const outW = this.outW;

const remaining = [];
for ( const module of sequence ) {
if ( /\.js$/.test(module) ) {
// console.log("Processing " + module);
const resource = await this.pool.findResourceWithInfo(module);
let code = await resource.buffer();
code = rewriteDefine(this.targetBundleFormat, code, module);
if ( code ) {
outW.startSegment(module);
for ( const moduleName of sequence ) {
if ( /\.js$/.test(moduleName) ) {
// console.log("Processing " + moduleName);
const resource = await this.pool.findResourceWithInfo(moduleName);
let moduleCode = (await resource.buffer()).toString();
let moduleSourceMap = false;
if (this.options.sourceMap) {
moduleSourceMap = null;
try {
const sourceMapResource = await this.pool.findResource(`${moduleName}.map`);
moduleSourceMap = (await sourceMapResource.buffer()).toString();
} catch (e) {
// No input sourcemap
// TODO: Differentiate real errors from file not found
}

if (!moduleSourceMap) {
const inlineSourcemap = moduleCode.match(sourcemapPattern);
if (inlineSourcemap) {
moduleSourceMap = Buffer.from(inlineSourcemap[1], "base64").toString();
}
}
// Strip sourceMappingURL from module code to be bundled
// It has no effect and might be cause for confusion
moduleCode = moduleCode.replace(/\/\/# sourceMappingURL=.*/, "");
}

const {code, sourceMap} = await rewriteDefine({
targetBundleFormat: this.targetBundleFormat,
moduleCode, moduleName, moduleSourceMap
});
if (code) {
outW.startSegment(moduleName);
outW.ensureNewLine();
const fileContent = await this.compressJS(code, resource);
outW.write( fileContent );
if (sourceMap) {
sourceMap.sourceRoot = path.posix.relative(
path.dirname(this._bundleName), path.dirname(moduleName));
await this.addSourceMap(sourceMap);
}
outW.write( code );
outW.ensureNewLine();
const compressedSize = outW.endSegment();
log.verbose(" %s (%d,%d)", module,
log.verbose(" %s (%d,%d)", moduleName,
resource.info != null ? resource.info.size : -1, compressedSize);
} else {
// keep unprocessed modules
remaining.push(module);
remaining.push(moduleName);
}
} else {
// keep unprocessed modules
remaining.push(module);
remaining.push(moduleName);
}
}

Expand All @@ -408,22 +432,20 @@ class BundleBuilder {
const outW = this.outW;

if ( /\.js$/.test(module) && (info == null || !info.requiresTopLevelScope) ) {
const compressedContent = await this.compressJS( await resource.buffer(), resource );
outW.write(`function(){`);
outW.write( compressedContent );
outW.write( (await resource.buffer()).toString() );
this.exportGlobalNames(info);
outW.ensureNewLine();
outW.write(`}`);
} else if ( /\.js$/.test(module) /* implicitly: && info != null && info.requiresTopLevelScope */ ) {
log.warn("**** warning: module %s requires top level scope" +
" and can only be embedded as a string (requires 'eval')", module);
const compressedContent = await this.compressJS( await resource.buffer(), resource );
outW.write( makeStringLiteral( compressedContent ) );
outW.write( makeStringLiteral( (await resource.buffer()).toString() ) );
} else if ( /\.html$/.test(module) ) {
const fileContent = await resource.buffer();
const fileContent = (await resource.buffer()).toString();
outW.write( makeStringLiteral( fileContent ) );
} else if ( /\.json$/.test(module) ) {
let fileContent = await resource.buffer();
let fileContent = (await resource.buffer()).toString();
if ( this.optimize ) {
try {
fileContent = JSON.stringify( JSON.parse( fileContent) );
Expand All @@ -434,13 +456,13 @@ class BundleBuilder {
}
outW.write(makeStringLiteral(fileContent));
} else if ( /\.xml$/.test(module) ) {
let fileContent = await resource.buffer();
let fileContent = (await resource.buffer()).toString();
if ( this.optimize ) {
// For XML we use the pretty data
// Do not minify if XML(View) contains an <*:pre> tag,
// because whitespace of HTML <pre> should be preserved (should only happen rarely)
if (!xmlHtmlPrePattern.test(fileContent.toString())) {
fileContent = pd.xmlmin(fileContent.toString(), false);
if (!xmlHtmlPrePattern.test(fileContent)) {
fileContent = pd.xmlmin(fileContent, false);
}
}
outW.write( makeStringLiteral( fileContent ) );
Expand Down Expand Up @@ -503,11 +525,10 @@ class BundleBuilder {

const CALL_SAP_UI_DEFINE = ["sap", "ui", "define"];

function rewriteDefine(targetBundleFormat, code, moduleName) {
async function rewriteDefine({targetBundleFormat, moduleCode, moduleName, moduleSourceMap}) {
let ast;
const codeStr = code.toString();
try {
ast = parseJS(codeStr, {range: true});
ast = parseJS(moduleCode, {range: true});
} catch (e) {
log.error("error while parsing %s: %s", moduleName, e.message);
return;
Expand All @@ -519,7 +540,6 @@ function rewriteDefine(targetBundleFormat, code, moduleName) {
const changes = [];
const defineCall = ast.body[0].expression;


// Inject module name if missing
if ( defineCall.arguments.length == 0 ||
defineCall.arguments[0].type !== Syntax.Literal ) {
Expand All @@ -537,7 +557,6 @@ function rewriteDefine(targetBundleFormat, code, moduleName) {

changes.push({
index,
count: 0,
value
});
}
Expand All @@ -549,29 +568,81 @@ function rewriteDefine(targetBundleFormat, code, moduleName) {
changes.push({
// asterisk marks the index: sap.ui.*define()
index: defineCall.callee.property.range[0],
count: 0,
value: "pre"
});
}

return applyChanges(codeStr, changes);
if (moduleSourceMap === null) {
log.info(`No input source map available for module ${moduleName}`);
}
return transform(moduleCode, changes, moduleSourceMap);
}

return false;
}

function applyChanges(string, changes) {
// No sorting needed as changes are added in correct order
async function transform(code, changes, inputSourceMap) {
const mappingChanges = [];
const sourceMap = inputSourceMap ? JSON.parse(inputSourceMap) : null;

const array = Array.from(string);
const array = Array.from(code);
// No sorting needed as changes are added in correct (reverse) order
changes.forEach((change) => {
if (sourceMap) {
// Compute line and column for given index to re-align source map with inserted characters
const precedingCode = array.slice(0, change.index);

const line = precedingCode.reduce((lineCount, char) => {
if (char === "\n") {
lineCount++;
}
return lineCount;
}, 0);
const lineStartIndex = precedingCode.lastIndexOf("\n") + 1;
const column = change.index - lineStartIndex;

mappingChanges.push({
line,
column,
columnDiff: change.value.length
});
}

// Apply modification
array.splice(
change.index,
change.count,
0,
change.value
);
});
return array.join("");
}
const transformedCode = array.join("");

if (sourceMap) {
// Source map re-alignment needs to be done from front to back
mappingChanges.reverse();

const mappings = decodeMappings(sourceMap.mappings);
mappingChanges.forEach((mappingChange) => {
const lineMapping = mappings[mappingChange.line];

// Mapping structure:
// [generatedCodeColumn, sourceIndex, sourceCodeLine, sourceCodeColumn, nameIndex]
lineMapping.forEach((mapping) => {
if (mapping[0] > mappingChange.column) {
// All column mappings for the generated code after any change
// need to be moved by the amount of inserted characters
mapping[0] = mapping[0] + mappingChange.columnDiff;
}
});
});
sourceMap.mappings = encodeMappings(mappings);

// No need for file information in source map since the bundled code does not exist in any file anyways
delete sourceMap.file;
}

return {
code: transformedCode,
sourceMap
};
}
module.exports = BundleBuilder;
Loading

0 comments on commit b030373

Please sign in to comment.