Feature/sourcemap preprocessor support

package.json

@@ -56,6 +56,7 @@
   },
   "homepage": "https://github.com/sveltejs/svelte#README",
   "devDependencies": {
+    "@ampproject/remapping": "^0.3.0",
     "@rollup/plugin-commonjs": "^11.0.0",
     "@rollup/plugin-json": "^4.0.1",
     "@rollup/plugin-node-resolve": "^6.0.0",
@@ -89,6 +90,7 @@
     "rollup": "^1.27.14",
     "source-map": "^0.7.3",
     "source-map-support": "^0.5.13",
+    "sourcemap-codec": "^1.4.8",
     "tiny-glob": "^0.2.6",
     "tslib": "^1.10.0",
     "typescript": "^3.5.3"

src/compiler/compile/Component.ts

@@ -29,7 +29,9 @@ import add_to_set from './utils/add_to_set';
 import check_graph_for_cycles from './utils/check_graph_for_cycles';
 import { print, x, b } from 'code-red';
 import { is_reserved_keyword } from './utils/reserved_keywords';
+import { apply_preprocessor_sourcemap } from '../utils/string_with_sourcemap';
 import Element from './nodes/Element';
+import { DecodedSourceMap, RawSourceMap } from '@ampproject/remapping/dist/types/types';
 
 interface ComponentOptions {
 	namespace?: string;
@@ -330,6 +332,8 @@ export default class Component {
 			js.map.sourcesContent = [
 				this.source
 			];
+
+			js.map = apply_preprocessor_sourcemap(this.file, js.map, compile_options.sourcemap as (string | RawSourceMap | DecodedSourceMap));
 		}
 
 		return {

src/compiler/compile/index.ts

@@ -11,6 +11,7 @@ const valid_options = [
 	'format',
 	'name',
 	'filename',
+	'sourcemap',
 	'generate',
 	'outputFilename',
 	'cssOutputFilename',

src/compiler/compile/render_dom/index.ts

@@ -7,6 +7,8 @@ import { extract_names, Scope } from '../utils/scope';
 import { invalidate } from './invalidate';
 import Block from './Block';
 import { ClassDeclaration, FunctionExpression, Node, Statement, ObjectExpression, Expression } from 'estree';
+import { apply_preprocessor_sourcemap } from '../../utils/string_with_sourcemap';
+import { RawSourceMap, DecodedSourceMap } from '@ampproject/remapping/dist/types/types';
 
 export default function dom(
 	component: Component,
@@ -30,6 +32,9 @@ export default function dom(
 	}
 
 	const css = component.stylesheet.render(options.filename, !options.customElement);
+
+	css.map = apply_preprocessor_sourcemap(options.filename, css.map, options.sourcemap as string | RawSourceMap | DecodedSourceMap);
+
 	const styles = component.stylesheet.has_styles && options.dev
 		? `${css.code}\n/*# sourceMappingURL=${css.map.toUrl()} */`
 		: css.code;

src/compiler/interfaces.ts

@@ -110,6 +110,7 @@ export interface CompileOptions {
 	filename?: string;
 	generate?: 'dom' | 'ssr' | false;
 
+	sourcemap?: object | string;
 	outputFilename?: string;
 	cssOutputFilename?: string;
 	sveltePath?: string;

src/compiler/preprocess/index.ts

@@ -1,6 +1,11 @@
+import { RawSourceMap, DecodedSourceMap } from '@ampproject/remapping/dist/types/types';
+import { decode as decode_mappings } from 'sourcemap-codec';
+import { getLocator } from 'locate-character';
+import { StringWithSourcemap, sourcemap_add_offset, combine_sourcemaps } from '../utils/string_with_sourcemap';
+
 export interface Processed {
 	code: string;
-	map?: object | string;
+	map?: string | object;  // we are opaque with the type here to avoid dependency on the remapping module for our public types.
 	dependencies?: string[];
 }
 
@@ -37,12 +42,18 @@ function parse_attributes(str: string) {
 interface Replacement {
 	offset: number;
 	length: number;
-	replacement: string;
+	replacement: StringWithSourcemap;
 }
 
-async function replace_async(str: string, re: RegExp, func: (...any) => Promise<string>) {
+async function replace_async(
+	filename: string,
+	source: string,
+	get_location: ReturnType<typeof getLocator>,
+	re: RegExp,
+	func: (...any) => Promise<StringWithSourcemap>
+): Promise<StringWithSourcemap> {
 	const replacements: Array<Promise<Replacement>> = [];
-	str.replace(re, (...args) => {
+	source.replace(re, (...args) => {
 		replacements.push(
 			func(...args).then(
 				res =>
@@ -55,16 +66,55 @@ async function replace_async(str: string, re: RegExp, func: (...any) => Promise<
 		);
 		return '';
 	});
-	let out = '';
+	const out = new StringWithSourcemap();
 	let last_end = 0;
 	for (const { offset, length, replacement } of await Promise.all(
 		replacements
 	)) {
-		out += str.slice(last_end, offset) + replacement;
+		// content = unchanged source characters before the replaced segment
+		const content = StringWithSourcemap.from_source(
+			filename, source.slice(last_end, offset), get_location(last_end));
+		out.concat(content).concat(replacement);
 		last_end = offset + length;
 	}
-	out += str.slice(last_end);
-	return out;
+	// final_content = unchanged source characters after last replaced segment
+	const final_content = StringWithSourcemap.from_source(
+		filename, source.slice(last_end), get_location(last_end));
+	return out.concat(final_content);
+}
+
+/** 
+ * Convert a preprocessor output and its leading prefix and trailing suffix into StringWithSourceMap  
+ */
+function get_replacement(
+	filename: string,
+	offset: number,
+	get_location: ReturnType<typeof getLocator>,
+	original: string,
+	processed: Processed,
+	prefix: string,
+	suffix: string
+): StringWithSourcemap {
+
+	// Convert the unchanged prefix and suffix to StringWithSourcemap
+	const prefix_with_map = StringWithSourcemap.from_source(
+		filename, prefix, get_location(offset));
+	const suffix_with_map = StringWithSourcemap.from_source(
+		filename, suffix, get_location(offset + prefix.length + original.length));
+
+	// Convert the preprocessed code and its sourcemap to a StringWithSourcemap
+	let decoded_map: DecodedSourceMap;
+	if (processed.map) {
+		decoded_map = typeof processed.map === 'string' ? JSON.parse(processed.map) : processed.map;
+		if (typeof(decoded_map.mappings) === 'string') {
+			decoded_map.mappings = decode_mappings(decoded_map.mappings);
+		}
+		sourcemap_add_offset(decoded_map, get_location(offset + prefix.length));
+	}
+	const processed_with_map = StringWithSourcemap.from_processed(processed.code, decoded_map);
+
+	// Surround the processed code with the prefix and suffix, retaining valid sourcemappings
+	return prefix_with_map.concat(processed_with_map).concat(suffix_with_map);
 }
 
 export default async function preprocess(
@@ -76,60 +126,92 @@ export default async function preprocess(
 	const filename = (options && options.filename) || preprocessor.filename; // legacy
 	const dependencies = [];
 
-	const preprocessors = Array.isArray(preprocessor) ? preprocessor : [preprocessor];
+	const preprocessors = preprocessor
+		? Array.isArray(preprocessor) ? preprocessor : [preprocessor]
+		: [];
 
 	const markup = preprocessors.map(p => p.markup).filter(Boolean);
 	const script = preprocessors.map(p => p.script).filter(Boolean);
 	const style = preprocessors.map(p => p.style).filter(Boolean);
 
+	// sourcemap_list is sorted in reverse order from last map (index 0) to first map (index -1)
+	// so we use sourcemap_list.unshift() to add new maps
+	// https://github.com/ampproject/remapping#multiple-transformations-of-a-file
+	const sourcemap_list: Array<DecodedSourceMap | RawSourceMap> = [];
+
+	// TODO keep track: what preprocessor generated what sourcemap? to make debugging easier = detect low-resolution sourcemaps in fn combine_mappings
+
 	for (const fn of markup) {
+
+		// run markup preprocessor
 		const processed = await fn({
 			content: source,
 			filename
 		});
-		if (processed && processed.dependencies) dependencies.push(...processed.dependencies);
-		source = processed ? processed.code : source;
+
+		if (!processed) continue;
+
+		if (processed.dependencies) dependencies.push(...processed.dependencies);
+		source = processed.code;
+		if (processed.map) {
+			sourcemap_list.unshift(
+				typeof(processed.map) === 'string'
+					? JSON.parse(processed.map)
+					: processed.map
+			);
+		}
 	}
 
-	for (const fn of script) {
-		source = await replace_async(
+	async function preprocess_tag_content(tag_name: 'style' | 'script', preprocessor: Preprocessor) {
+		const get_location = getLocator(source);
+		const tag_regex = tag_name == 'style' 
+			? /<!--[^]*?-->|<style(\s[^]*?)?(?:>([^]*?)<\/style>|\/>)/gi
+			: /<!--[^]*?-->|<script(\s[^]*?)?(?:>([^]*?)<\/script>|\/>)/gi;
+
+		const res = await replace_async(
+			filename,
 			source,
-			/<!--[^]*?-->|<script(\s[^]*?)?(?:>([^]*?)<\/script>|\/>)/gi,
-			async (match, attributes = '', content = '') => {
+			get_location,
+			tag_regex,
+			async (match, attributes = '', content = '', offset) => {
+				const no_change = () => StringWithSourcemap.from_source(
+					filename, match, get_location(offset));
 				if (!attributes && !content) {
-					return match;
+					return no_change();
 				}
 				attributes = attributes || '';
-				const processed = await fn({
+				content = content || '';
+
+				// run script preprocessor
+				const processed = await preprocessor({
 					content,
 					attributes: parse_attributes(attributes),
 					filename
 				});
-				if (processed && processed.dependencies) dependencies.push(...processed.dependencies);
-				return processed ? `<script${attributes}>${processed.code}</script>` : match;
+
+				if (!processed) return no_change();
+				if (processed.dependencies) dependencies.push(...processed.dependencies);
+				return get_replacement(filename, offset, get_location, content, processed, `<${tag_name}${attributes}>`, `</${tag_name}>`);
 			}
 		);
+		source = res.string;
+		sourcemap_list.unshift(res.map);
+	}
+
+	for (const fn of script) {
+		await preprocess_tag_content('script', fn);
 	}
 
 	for (const fn of style) {
-		source = await replace_async(
-			source,
-			/<!--[^]*?-->|<style(\s[^]*?)?(?:>([^]*?)<\/style>|\/>)/gi,
-			async (match, attributes = '', content = '') => {
-				if (!attributes && !content) {
-					return match;
-				}
-				const processed: Processed = await fn({
-					content,
-					attributes: parse_attributes(attributes),
-					filename
-				});
-				if (processed && processed.dependencies) dependencies.push(...processed.dependencies);
-				return processed ? `<style${attributes}>${processed.code}</style>` : match;
-			}
-		);
+		await preprocess_tag_content('style', fn);
 	}
 
+	// Combine all the source maps for each preprocessor function into one
+	const map: RawSourceMap = combine_sourcemaps(
+		filename,
+		sourcemap_list
+	);
+
 	return {
 		// TODO return separated output, in future version where svelte.compile supports it:
 		// style: { code: styleCode, map: styleMap },
@@ -138,7 +220,7 @@ export default async function preprocess(
 
 		code: source,
 		dependencies: [...new Set(dependencies)],
-
+		map: (map as object),
 		toString() {
 			return source;
 		}