[FEATURE] Generate source maps for bundles (#695)

Resolves SAP/ui5-tooling#472
Supersedes #282
Based on SAP/ui5-tooling#583
JIRA: CPOUI5FOUNDATION-434

Co-authored-by: Matthias Osswald <mat.osswald@sap.com>

lib/lbt/bundle/BundleWriter.js

@@ -11,13 +11,17 @@ const SPACES_OR_TABS_ONLY = /^[ \t]+$/;
  *
  * Most methods have been extracted from JSMergeWriter.
  *
+ * columnOffset and lineOffset are used for sourcemap merging as reference to where we are at a given point in time
+ *
  * @author Frank Weigel
  * @since 1.27.0
  * @private
  */
 class BundleWriter {
 	constructor() {
 		this.buf = "";
+		this.lineOffset = 0;
+		this.columnOffset = 0;
 		this.segments = [];
 		this.currentSegment = null;
 		this.currentSourceIndex = 0;
@@ -28,6 +32,11 @@ class BundleWriter {
 		let writeBuf = "";
 		for ( let i = 0; i < str.length; i++ ) {
 			writeBuf += str[i];
+			if (str[i] != null && str[i].split) {
+				const strSplit = str[i].split(NL);
+				this.lineOffset += strSplit.length - 1;
+				this.columnOffset += strSplit[strSplit.length - 1].length;
+			}
 		}
 		if ( writeBuf.length >= 1 ) {
 			this.buf += writeBuf;
@@ -40,15 +49,23 @@ class BundleWriter {
 	writeln(...str) {
 		for ( let i = 0; i < str.length; i++ ) {
 			this.buf += str[i];
+			if (str[i] != null && str[i].split) {
+				const strSplit = str[i].split(NL);
+				this.lineOffset += strSplit.length - 1;
+			}
 		}
 		this.buf += NL;
 		this.endsWithNewLine = true;
+		this.lineOffset += 1;
+		this.columnOffset = 0;
 	}
 
 	ensureNewLine() {
 		if ( !this.endsWithNewLine ) {
 			this.buf += NL;
 			this.endsWithNewLine = true;
+			this.lineOffset += 1;
+			this.columnOffset = 0;
 		}
 	}
 

lib/lbt/resources/LocatorResource.js

@@ -1,12 +1,12 @@
 const Resource = require("./Resource");
 
-function extractName(path) {
-	return path.slice( "/resources/".length);
-}
+// function extractName(path) {
+// 	return path.slice( "/resources/".length);
+// }
 
 class LocatorResource extends Resource {
-	constructor(pool, resource) {
-		super(pool, extractName(resource.getPath()), null, resource.getStatInfo());
+	constructor(pool, resource, moduleName) {
+		super(pool, moduleName, null, resource.getStatInfo());
 		this.resource = resource;
 	}
 
@@ -17,6 +17,10 @@ class LocatorResource extends Resource {
 	getProject() {
 		return this.resource._project;
 	}
+
+	getPath() {
+		return this.resource.getPath();
+	}
 }
 
 module.exports = LocatorResource;

lib/lbt/resources/LocatorResourcePool.js

@@ -2,12 +2,16 @@ const ResourcePool = require("./ResourcePool");
 const LocatorResource = require("./LocatorResource");
 
 class LocatorResourcePool extends ResourcePool {
-	prepare(resources) {
+	prepare(resources, moduleNameMapping) {
 		resources = resources.filter( (res) => !res.getStatInfo().isDirectory() );
 		return Promise.all(
-			resources.map(
-				(resource) => this.addResource( new LocatorResource(this, resource) )
-			).filter(Boolean)
+			resources.map((resource) => {
+				let moduleName = moduleNameMapping && moduleNameMapping[resource.getPath()];
+				if (!moduleName) {
+					moduleName = resource.getPath().slice("/resources/".length);
+				}
+				this.addResource(new LocatorResource(this, resource, moduleName));
+			}).filter(Boolean)
 		);
 	}
 }

lib/processors/bundlers/moduleBundler.js

@@ -90,6 +90,8 @@ const log = require("@ui5/logger").getLogger("builder:processors:bundlers:module
  * @public
  * @typedef {object} ModuleBundleOptions
  * @property {boolean} [optimize=true] Whether the module bundle gets minified
+ * @property {boolean} [sourceMap] Whether to generate a source map file for the bundle.
+ *   Defaults to true if <code>optimize</code> is set to true
  * @property {boolean} [decorateBootstrapModule=false] If set to 'false', the module won't be decorated
  *   with an optimization marker
  * @property {boolean} [addTryCatchRestartWrapper=false] Whether to wrap bootable module bundles with
@@ -101,18 +103,30 @@ const log = require("@ui5/logger").getLogger("builder:processors:bundlers:module
  */
 
 /**
- * Legacy preload bundler.
+ * Result set
+ *
+ * @public
+ * @typedef {object} ModuleBundlerResult
+ * @property {module:@ui5/fs.Resource} bundle Bundle resource
+ * @property {module:@ui5/fs.Resource} sourceMap Source Map
+ * @memberof module:@ui5/builder.processors
+ */
+
+/**
+ * Legacy module bundler.
  *
  * @public
  * @alias module:@ui5/builder.processors.moduleBundler
  * @param {object} parameters Parameters
  * @param {module:@ui5/fs.Resource[]} parameters.resources Resources
  * @param {object} parameters.options Options
+ * @param {object} [parameters.options.moduleNameMapping]
+ 				Optional mapping of resource paths to module name in order to overwrite the default determination
  * @param {ModuleBundleDefinition} parameters.options.bundleDefinition Module bundle definition
  * @param {ModuleBundleOptions} [parameters.options.bundleOptions] Module bundle options
- * @returns {Promise<module:@ui5/fs.Resource[]>} Promise resolving with module bundle resources
+ * @returns {Promise<module:@ui5/builder.processors.MinifierResult[]>} Promise resolving with module bundle resources
  */
-module.exports = function({resources, options: {bundleDefinition, bundleOptions}}) {
+module.exports = function({resources, options: {bundleDefinition, bundleOptions, moduleNameMapping}}) {
 	// Apply defaults without modifying the passed object
 	bundleOptions = Object.assign({}, {
 		optimize: true,
@@ -134,7 +148,7 @@ module.exports = function({resources, options: {bundleDefinition, bundleOptions}
 		log.verbose(`bundleOptions: ${JSON.stringify(bundleOptions, null, 2)}`);
 	}
 
-	return pool.prepare( resources ).
+	return pool.prepare( resources, moduleNameMapping ).
 		then( () => builder.createBundle(bundleDefinition, bundleOptions) ).
 		then( (results) => {
 			let bundles;
@@ -146,13 +160,20 @@ module.exports = function({resources, options: {bundleDefinition, bundleOptions}
 
 			return Promise.all(bundles.map((bundleObj) => {
 				if ( bundleObj ) {
-					const {name, content} = bundleObj;
+					const {name, content, sourceMap} = bundleObj;
 					// console.log("creating bundle as '%s'", "/resources/" + name);
-					const resource = new EvoResource({
+					const res = {};
+					res.bundle = new EvoResource({
 						path: "/resources/" + name,
 						string: content
 					});
-					return resource;
+					if (sourceMap) {
+						res.sourceMap = new EvoResource({
+							path: "/resources/" + name + ".map",
+							string: sourceMap
+						});
+					}
+					return res;
 				}
 			}));
 		});

lib/processors/minifier.js

@@ -34,10 +34,12 @@ const debugFileRegex = /((?:\.view|\.fragment|\.controller|\.designtime|\.suppor
  * @alias module:@ui5/builder.processors.minifier
  * @param {object} parameters Parameters
  * @param {module:@ui5/fs.Resource[]} parameters.resources List of resources to be processed
+ * @param {boolean} [parameters.addSourceMappingUrl=true]
+ * 				Whether to add a sourceMappingURL reference to the end of the minified resource
  * @returns {Promise<module:@ui5/builder.processors.MinifierResult[]>}
  * 				Promise resolving with object of resource, dbgResource and sourceMap
  */
-module.exports = async function({resources}) {
+module.exports = async function({resources, addSourceMappingUrl = true}) {
 	return Promise.all(resources.map(async (resource) => {
 		const dbgPath = resource.getPath().replace(debugFileRegex, "-dbg$1");
 		const dbgResource = await resource.clone();
@@ -46,6 +48,12 @@ module.exports = async function({resources}) {
 		const filename = path.posix.basename(resource.getPath());
 		const code = await resource.getString();
 		try {
+			const sourceMapOptions = {
+				filename
+			};
+			if (addSourceMappingUrl) {
+				sourceMapOptions.url = filename + ".map";
+			}
 			const dbgFilename = path.posix.basename(dbgPath);
 			const result = await terser.minify({
 				// Use debug-name since this will be referenced in the source map "sources"
@@ -63,10 +71,7 @@ module.exports = async function({resources}) {
 						"sap",
 					]
 				},
-				sourceMap: {
-					filename,
-					url: filename + ".map"
-				}
+				sourceMap: sourceMapOptions
 			});
 			resource.setString(result.code);
 			const sourceMapResource = new Resource({

lib/tasks/bundlers/generateBundle.js

@@ -25,46 +25,104 @@ module.exports = function({
 		readers: [workspace, dependencies]
 	});
 
+	const optimize = !bundleOptions || bundleOptions.optimize !== false;
 	if (taskUtil) {
-		const optimize = !bundleOptions || bundleOptions.optimize !== false;
+		/* Scenarios
+			1. Optimize bundle with minification already done
+				Workspace:
+					* /resources/my/lib/Control.js 				[ui5:HasDebugVariant]
+					* /resources/my/lib/Control.js.map 			[ui5:HasDebugVariant]
+					* /resources/my/lib/Control-dbg.js 			[ui5:IsDebugVariant]
+
+				Bundler input:
+					* /resources/my/lib/Control.js
+					* /resources/my/lib/Control.js.map
+
+				=> Filter out debug resources
+
+			2. Optimize bundle with no minification
+				* /resources/my/lib/Control.js
+
+				=> No action necessary
+
+			3. Debug-bundle with minification already done
+				Workspace:
+					* /resources/my/lib/Control.js 				[ui5:HasDebugVariant]
+					* /resources/my/lib/Control.js.map 			[ui5:HasDebugVariant]
+					* /resources/my/lib/Control-dbg.js 			[ui5:IsDebugVariant]
+
+				Bundler input:
+					* /resources/my/lib/Control-dbg.js
+					* moduleNameMapping: [{"/resources/my/lib/Control-dbg.js": "my/lib/Control.js"}]
+
+				=> Filter out minified-resources (tagged as "HasDebugVariant", incl. source maps) and rename debug-files
+
+			4. Debug-bundle with no minification
+				* /resources/my/lib/Control.js
+
+				=> No action necessary
+
+			5. Bundle with external input (optimize or not), e.g. TS-project
+				Workspace:
+					* /resources/my/lib/Control.ts
+					* /resources/my/lib/Control.js
+					* /resources/my/lib/Control.js.map
+
+				Bundler input:
+					* /resources/my/lib/Control.js
+					* /resources/my/lib/Control.js.map
+		*/
 
 		// Omit -dbg files for optimize bundles and vice versa
 		const filterTag = optimize ?
 			taskUtil.STANDARD_TAGS.IsDebugVariant : taskUtil.STANDARD_TAGS.HasDebugVariant;
 		combo = combo.filter(function(resource) {
 			return !taskUtil.getTag(resource, filterTag);
 		});
+	}
 
-		if (!optimize) {
-			// For "unoptimized" bundles, the non-debug files have already been filtered out
-			// Now rename the debug variants to the same name so that they appear like the original
-			// resource to the bundler
-			combo = combo.transformer(async function(resourcePath, getResource) {
+	return combo.byGlob("/resources/**/*.{js,json,xml,html,properties,library,js.map}").then((resources) => {
+		const moduleNameMapping = {};
+		if (!optimize && taskUtil) {
+			// For "unoptimized" bundles, the non-debug files have already been filtered out above.
+			// Now we need to create a mapping from the debug-variant resource path to the respective module name,
+			// which is basically the non-debug resource path, minus the "/resources/"" prefix.
+			// This mapping overwrites internal logic of the LocatorResourcePool which would otherwise determine
+			// the module name from the resource path, which would contain "-dbg" in this case. That would be
+			// incorrect since debug-variants should still keep the original module name.
+			for (let i = resources.length - 1; i >= 0; i--) {
+				const resourcePath = resources[i].getPath();
 				if (taskUtil.getTag(resourcePath, taskUtil.STANDARD_TAGS.IsDebugVariant)) {
-					const resource = await getResource();
-					const nonDbgPath = ModuleName.getNonDebugName(resource.getPath());
+					const nonDbgPath = ModuleName.getNonDebugName(resourcePath);
 					if (!nonDbgPath) {
-						throw new Error(`Failed to resolve non-debug name for ${resource.getPath()}`);
+						throw new Error(`Failed to resolve non-debug name for ${resourcePath}`);
 					}
-					resource.setPath(nonDbgPath);
+					moduleNameMapping[resourcePath] = nonDbgPath.slice("/resources/".length);
 				}
-			});
+			}
 		}
-	}
-
-	return combo.byGlob("/resources/**/*.{js,json,xml,html,properties,library}").then((resources) => {
 		return moduleBundler({
 			options: {
 				bundleDefinition,
-				bundleOptions
+				bundleOptions,
+				moduleNameMapping
 			},
 			resources
 		}).then((bundles) => {
-			return Promise.all(bundles.map((bundle) => {
+			return Promise.all(bundles.map(({bundle, sourceMap}) => {
 				if (taskUtil) {
 					taskUtil.setTag(bundle, taskUtil.STANDARD_TAGS.IsBundle);
+					if (sourceMap) {
+						// Clear tag that might have been set by the minify task, in cases where
+						// the bundle name is identical to a source file
+						taskUtil.clearTag(sourceMap, taskUtil.STANDARD_TAGS.OmitFromBuildResult);
+					}
+				}
+				const writes = [workspace.write(bundle)];
+				if (sourceMap) {
+					writes.push(workspace.write(sourceMap));
 				}
-				return workspace.write(bundle);
+				return Promise.all(writes);
 			}));
 		});
 	});

lib/tasks/bundlers/generateComponentPreload.js

@@ -41,7 +41,7 @@ module.exports = function({
 	}
 
 	// TODO 3.0: Limit to workspace resources?
-	return combo.byGlob("/resources/**/*.{js,json,xml,html,properties,library}")
+	return combo.byGlob("/resources/**/*.{js,json,xml,html,properties,library,js.map}")
 		.then(async (resources) => {
 			let allNamespaces = [];
 			if (paths) {
@@ -148,12 +148,19 @@ module.exports = function({
 				});
 			}));
 		})
-		.then((processedResources) => {
-			return Promise.all(processedResources.map((resource) => {
+		.then((results) => {
+			const bundles = Array.prototype.concat.apply([], results);
+			return Promise.all(bundles.map(({bundle, sourceMap}) => {
 				if (taskUtil) {
-					taskUtil.setTag(resource[0], taskUtil.STANDARD_TAGS.IsBundle);
+					taskUtil.setTag(bundle, taskUtil.STANDARD_TAGS.IsBundle);
+					// Clear tag that might have been set by the minify task, in cases where
+					// the bundle name is identical to a source file
+					taskUtil.clearTag(sourceMap, taskUtil.STANDARD_TAGS.OmitFromBuildResult);
 				}
-				return workspace.write(resource[0]);
+				return Promise.all([
+					workspace.write(bundle),
+					workspace.write(sourceMap)
+				]);
 			}));
 		});
 };