Skip to content

Commit

Permalink
[FEATURE] Sourcemap support in the ui5-builder preload generation
Browse files Browse the repository at this point in the history 
- Always go for the predefine rewrite in the library build
- Change the order of the steps to have debug file created early enough
- Sourcemap support for the rewrite and for the preload generation
- Uglifier can take input sourcemap as well (if you have a custom transpile step...)
  • Loading branch information
@nlunets
nlunets committed Jul 4, 2019
1 parent dba90ad commit 4981289
Show file tree
Hide file tree
Showing 8 changed files with 116 additions and 47 deletions.
2 changes: 1 addition & 1 deletion .eslintrc.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ module.exports = {
"es6": true
},
"parserOptions": {
"ecmaVersion": 8
"ecmaVersion": 2018
},
"extends": ["eslint:recommended", "google"],
"plugins": [
Expand Down
88 changes: 63 additions & 25 deletions lib/lbt/bundle/Builder.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,13 @@
// for consistency of write calls, we generally allow template literals
"use strict";

const path = require("path");
const uglify = require("uglify-es");
const {pd} = require("pretty-data");
const esprima = require("esprima");
const escodegen = require("escodegen");
const {Syntax} = esprima;
// const MOZ_SourceMap = require("source-map");
const sourceMap = require("source-map");

const {isMethodCall} = require("../utils/ASTUtils");
const ModuleName = require("../utils/ModuleName");
Expand Down Expand Up @@ -144,6 +145,7 @@ class BundleBuilder {
this.openModule(resolvedModule.name);

this.writeConfiguration(resolvedModule.configuration); // NODE-TODO configuration currently will be undefined
this._sourceMap = new sourceMap.SourceMapGenerator({file: module.name, sourceRoot: "."});

// create all sections in sequence
for ( const section of resolvedModule.sections ) {
Expand All @@ -160,6 +162,7 @@ class BundleBuilder {
return {
name: module.name,
content: this.outW.toString(),
sourceMap: this._sourceMap.toString(),
bundleInfo: bundleInfo
};
}
Expand Down Expand Up @@ -188,11 +191,7 @@ class BundleBuilder {
this.outW.writeln(`if (oError.name != "Restart") { throw oError; }`);
this.outW.writeln(`}`);
}
/* NODE-TODO
if ( writeSourceMap && writeSourceMapAnnotation ) {
outW.ensureNewLine();
outW.write("//# sourceMappingURL=" + moduleName.getBaseName().replaceFirst("\\.js$", ".js.map"));
}*/
this.outW.writeln(`//# sourceMappingURL=${path.basename(resolvedModule.name)}.map`);
}

addSection(section) {
Expand Down Expand Up @@ -287,7 +286,7 @@ class BundleBuilder {
const sequence = section.modules.slice();

this.beforeWriteFunctionPreloadSection(sequence);

this.outW.writeln(`//@ui5-bundle ${section.bundle.name}`);
await this.rewriteAMDModules(sequence, avoidLazyParsing);
if ( sequence.length > 0 ) {
this.targetBundleFormat.beforePreloads(outW, section);
Expand All @@ -301,6 +300,7 @@ class BundleBuilder {
this.beforeWritePreloadModule(module, resource.info, resource);
outW.write(`\t"${module.toString()}":`);
outW.startSegment(module);

await this.writePreloadModule(module, resource.info, resource, avoidLazyParsing);
const compressedSize = outW.endSegment();
log.verbose(" %s (%d,%d)", module,
Expand All @@ -320,11 +320,14 @@ class BundleBuilder {
// this.afterWriteFunctionPreloadSection();
}

compressJS(fileContent, resource) {
compressJS(fileContent, resource, fileMap) {
if ( this.optimize ) {
const result = uglify.minify({
[resource.name]: String(fileContent)
}, {
sourceMap: {
content: fileMap ? fileMap.toString() : "inline"
},
warnings: false, // TODO configure?
compress: false, // TODO configure?
output: {
Expand All @@ -336,13 +339,11 @@ class BundleBuilder {
if ( result.error ) {
throw result.error;
}
// console.log(result.map);
// const map = new MOZ_SourceMap.SourceMapConsumer(result.map);
// map.eachMapping(function (m) { console.log(m); }); // console.log(map);
fileContent = result.code;
fileMap = result.map;
// throw new Error();
}
return fileContent;
return {compressedContent: fileContent, contentMap: fileMap};
}

async compressJSAsync(resource) {
Expand All @@ -355,6 +356,27 @@ class BundleBuilder {
sequence.sort();
}

addSourceMap(contentMap) {
const map = new sourceMap.SourceMapConsumer(contentMap);
map.eachMapping((mapping) => {
if (mapping.source) {
this._sourceMap.addMapping({
generated: {
line: this.outW.lineOffset + mapping.generatedLine,
column: (mapping.generatedLine === 1 ? this.outW.columnOffset : 0) + mapping.generatedColumn
},
original: mapping.originalLine == null ? null : {
line: mapping.originalLine,
column: mapping.originalColumn
},
source: mapping.originalLine != null ?
mapping.source : null,
name: mapping.name
});
}
});
}

async rewriteAMDModules(sequence, avoidLazyParsing) {
if ( this.options.usePredefineCalls ) {
const outW = this.outW;
Expand All @@ -364,14 +386,21 @@ class BundleBuilder {
if ( /\.js$/.test(module) ) {
// console.log("Processing " + module);
const resource = await this.pool.findResourceWithInfo(module);
const sourceMapResource = await this.pool.findResourceWithInfo(`${module}.map`);
const code = await resource.buffer();
const ast = rewriteDefine(this.targetBundleFormat, code, module, avoidLazyParsing);
const sourceMapContent = (await sourceMapResource.buffer()).toString();
let ast = rewriteDefine(this.targetBundleFormat, code, module, avoidLazyParsing, sourceMapContent);
if ( ast ) {
outW.startSegment(module);
outW.ensureNewLine();
const astAsCode = escodegen.generate(ast);
const fileContent = this.compressJS(astAsCode, resource);
outW.write( fileContent );
ast = escodegen.attachComments(ast, ast.comments, ast.tokens);
const astAsCode = escodegen.generate(ast, {comment: true, sourceMap: true, sourceMapWithCode: true});
// Reapply sourcemap on top
astAsCode.map.applySourceMap(new sourceMap.SourceMapConsumer(sourceMapContent), path.basename(module, ".js")+"-dbg.js");
const {compressedContent, contentMap} = {...this.compressJS(astAsCode.code, resource, astAsCode.map.toString())};
this.addSourceMap(contentMap);

outW.write( compressedContent );
outW.ensureNewLine();
const compressedSize = outW.endSegment();
log.verbose(" %s (%d,%d)", module,
Expand Down Expand Up @@ -400,11 +429,15 @@ class BundleBuilder {
const outW = this.outW;

if ( /\.js$/.test(module) && (info == null || !info.requiresTopLevelScope) ) {
const compressedContent = await this.compressJSAsync( resource );
if ( avoidLazyParsing ) {
outW.write(`(`);
}
outW.write(`function(){`);

const compressorOutput = await this.compressJSAsync( resource );
const {compressedContent, contentMap} = {...compressorOutput};

this.addSourceMap(contentMap);
outW.write( compressedContent );
this.exportGlobalNames(info);
outW.ensureNewLine();
Expand Down Expand Up @@ -473,13 +506,15 @@ class BundleBuilder {
const CALL_DEFINE = ["define"];
const CALL_SAP_UI_DEFINE = ["sap", "ui", "define"];

function rewriteDefine(targetBundleFormat, code, moduleName, avoidLazyParsing) {
function _injectModuleNameIfNeeded(defineCall) {
function rewriteDefine(targetBundleFormat, code, moduleName, avoidLazyParsing, sourceMapContent) {
function _injectModuleNameIfNeeded(defineCall, moduleName) {
if ( defineCall.arguments.length == 0
|| defineCall.arguments[0].type !== Syntax.Literal ) {
defineCall.arguments.unshift({
type: Syntax.Literal,
value: ModuleName.toRequireJSName(moduleName)
value: ModuleName.toRequireJSName(moduleName),
loc: null,
range: []
});
}
}
Expand Down Expand Up @@ -511,7 +546,13 @@ function rewriteDefine(targetBundleFormat, code, moduleName, avoidLazyParsing) {

let ast;
try {
ast = esprima.parseScript(code.toString(), {loc: true});
ast = esprima.parseScript(code.toString(), {
comment: true,
loc: true,
source: path.basename(moduleName, ".js")+"-dbg.js",
range: true,
tokens: true
});
} catch (e) {
log.error("error while parsing %s:%s", module, e);
return;
Expand Down Expand Up @@ -559,18 +600,16 @@ function rewriteDefine(targetBundleFormat, code, moduleName, avoidLazyParsing) {
}

// console.log("rewriting %s", module, defineCall);

return ast;
}


if ( isMethodCall(ast.body[0].expression, CALL_SAP_UI_DEFINE) ) {
const defineCall = ast.body[0].expression;

// rewrite sap.ui.define to sap.ui.predefine
if ( defineCall.callee.type === Syntax.MemberExpression
&& defineCall.callee.property.type === Syntax.Identifier
&& defineCall.callee.property.name === "define" ) {
&& defineCall.callee.property.name === "define") {
defineCall.callee.property.name = "predefine";
}

Expand All @@ -579,7 +618,6 @@ function rewriteDefine(targetBundleFormat, code, moduleName, avoidLazyParsing) {
if ( avoidLazyParsing ) {
enforceEagerParsing(defineCall);
}

// console.log("rewriting %s", module, defineCall);

return ast;
Expand Down
10 changes: 10 additions & 0 deletions lib/lbt/bundle/BundleWriter.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,17 @@ const ENDS_WITH_NEW_LINE = /(^|\r\n|\r|\n)[ \t]*$/;
*
* Most methods have been extracted from JSMergeWriter.
*
* columnOffset and lineOffset are used for sourcemap merging as reference to where we are at a given point in time
*
* @author Frank Weigel
* @since 1.27.0
* @private
*/
class BundleWriter {
constructor() {
this.buf = "";
this.lineOffset = 0;
this.columnOffset = 0;
this.segments = [];
this.currentSegment = null;
this.currentSourceIndex = 0;
Expand All @@ -25,6 +29,8 @@ class BundleWriter {
write(...str) {
for ( let i = 0; i < str.length; i++ ) {
this.buf += str[i];
this.lineOffset += str[i].split(NL).length - 1;
this.columnOffset += str[i].length;
}
}

Expand All @@ -33,12 +39,16 @@ class BundleWriter {
this.buf += str[i];
}
this.buf += NL;
this.lineOffset += 1;
this.columnOffset = 0;
}

ensureNewLine() {
// TODO this regexp might be quite expensive (use of $ anchor on long strings)
if ( !ENDS_WITH_NEW_LINE.test(this.buf) ) {
this.buf += NL;
this.lineOffset += 1;
this.columnOffset = 0;
}
}

Expand Down
14 changes: 10 additions & 4 deletions lib/processors/bundlers/moduleBundler.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ module.exports = function({resources, options}) {
const pool = new LocatorResourcePool();
const builder = new BundleBuilder(pool);

const bundleOptions = options.bundleOptions || {optimize: true};
const bundleOptions = options.bundleOptions || {optimize: true, usePredefineCalls: true};

return pool.prepare( resources ).
then( () => builder.createBundle(options.bundleDefinition, bundleOptions) ).
Expand All @@ -68,13 +68,19 @@ module.exports = function({resources, options}) {
bundles = [results];
}

return Promise.all(bundles.map(function({name, content, bundleInfo}) {
return Promise.all(bundles.map(function({name, sourceMap, content, bundleInfo}) {
// console.log("creating bundle as '%s'", "/resources/" + name);
const resource = new EvoResource({
path: "/resources/" + name,
string: content
});
return resource;
}));
const sourceMapResource = new EvoResource({
path: "/resources/" + name + ".map",
string: sourceMap
});


return [resource, sourceMapResource];
})).then((resources) => [].concat(...resources));
});
};
14 changes: 11 additions & 3 deletions lib/processors/uglifier.js
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
const uglify = require("uglify-es");
const copyrightCommentsPattern = /copyright|\(c\)(?:[0-9]+|\s+[0-9A-za-z])|released under|license|\u00a9/i;
const EvoResource = require("@ui5/fs").Resource;

/**
* Minifies the supplied resources.
Expand All @@ -16,6 +17,10 @@ module.exports = function({resources}) {
const result = uglify.minify({
[resource.getPath()]: code
}, {
sourceMap: {
content: "inline",
url: resource.getPath() + ".map"
},
warnings: false,
output: {
comments: copyrightCommentsPattern
Expand All @@ -27,9 +32,12 @@ module.exports = function({resources}) {
`Uglification failed with error: ${result.error.message} in file ${result.error.filename} ` +
`(line ${result.error.line}, col ${result.error.col}, pos ${result.error.pos})`);
}

const sourceMapResource = new EvoResource({
path: resource.getPath() + ".map",
string: result.map
});
resource.setString(result.code);
return resource;
return [sourceMapResource, resource];
});
}));
})).then((resources) => [].concat(...resources));
};
12 changes: 9 additions & 3 deletions lib/tasks/bundlers/generateLibraryPreload.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -155,7 +155,7 @@ module.exports = function({workspace, dependencies, options}) {
readers: [workspace, dependencies]
});

return combo.byGlob("/**/*.{js,json,xml,html,properties,library}").then((resources) => {
return combo.byGlob(["/**/*.{js,json,xml,html,properties,library,js.map}", "!/**/*-dbg.js"]).then((resources) => {
// Find all libraries and create a library-preload.js bundle

let p;
Expand Down Expand Up @@ -235,11 +235,17 @@ module.exports = function({workspace, dependencies, options}) {
bundleDefinition: getBundleDefinition(libraryNamespace)
},
resources
}).then(([bundle]) => {
}).then(([bundle, sourceMapBundle]) => {
const outPromises = [];
if (bundle) {
// console.log(`${libraryNamespace}/library-preload.js bundle created`);
return workspace.write(bundle);
outPromises.push(workspace.write(bundle));
}
if (sourceMapBundle) {
// console.log(`${libraryNamespace}/library-preload.js bundle created`);
outPromises.push(workspace.write(sourceMapBundle));
}
return Promise.all(outPromises);
});
} else {
log.verbose(`Could not determine library namespace from file "${libraryIndicatorPath}" for project ${options.projectName}. Skipping library preload bundling.`);
Expand Down
Loading

0 comments on commit 4981289

Please sign in to comment.