diff --git a/external/builder/builder.js b/external/builder/builder.js index c336e432b2f82..d3b6ed8628101 100644 --- a/external/builder/builder.js +++ b/external/builder/builder.js @@ -10,16 +10,32 @@ var fs = require('fs'), vm = require('vm'); /** - * A simple preprocessor that is based on the firefox preprocessor - * see (https://developer.mozilla.org/en/Build/Text_Preprocessor). The main - * difference is that this supports a subset of the commands and it supports - * preproccesor commands in html style comments. - * Currently Supported commands: + * A simple preprocessor that is based on the Firefox preprocessor + * (https://dxr.mozilla.org/mozilla-central/source/build/docs/preprocessor.rst). + * The main difference is that this supports a subset of the commands and it + * supports preprocessor commands in HTML-style comments. + * + * Currently supported commands: * - if + * - elif * - else * - endif * - include * - expand + * - error + * + * Every #if must be closed with an #endif. Nested conditions are supported. + * + * Within an #if or #else block, one level of comment tokens is stripped. This + * allows us to write code that can run even without preprocessing. For example: + * + * //#if SOME_RARE_CONDITION + * // // Decrement by one + * // --i; + * //#else + * // // Increment by one. + * ++i; + * //#endif */ function preprocess(inFilename, outFilename, defines) { // TODO make this really read line by line. @@ -37,10 +53,28 @@ function preprocess(inFilename, outFilename, defines) { function(line) { out += line + '\n'; }); + function evaluateCondition(code) { + if (!code || !code.trim()) { + throw new Error('No JavaScript expression given at ' + loc()); + } + try { + return vm.runInNewContext(code, defines, {displayErrors: false}); + } catch (e) { + throw new Error('Could not evaluate "' + code + '" at ' + loc() + '\n' + + e.name + ': ' + e.message); + } + } function include(file) { var realPath = fs.realpathSync(inFilename); var dir = path.dirname(realPath); - preprocess(path.join(dir, file), writeLine, defines); + try { + preprocess(path.join(dir, file), writeLine, defines); + } catch (e) { + if (e.code === 'ENOENT') { + throw new Error('Failed to include "' + file + '" at ' + loc()); + } + throw e; // Some other error + } } function expand(line) { line = line.replace(/__[\w]+__/g, function(variable) { @@ -53,52 +87,92 @@ function preprocess(inFilename, outFilename, defines) { writeLine(line); } - var s, state = 0, stack = []; + // not inside if or else (process lines) + var STATE_NONE = 0; + // inside if, condition false (ignore until #else or #endif) + var STATE_IF_FALSE = 1; + // inside else, #if was false, so #else is true (process lines until #endif) + var STATE_ELSE_TRUE = 2; + // inside if, condition true (process lines until #else or #endif) + var STATE_IF_TRUE = 3; + // inside else, #if was true, so #else is false (ignore lines until #endif) + var STATE_ELSE_FALSE = 4; + + var line; + var state = STATE_NONE; + var stack = []; var control = - /^(?:\/\/|)?$)?/; + /* jshint -W101 */ + /^(?:\/\/|)?$)?/; + /* jshint +W101 */ var lineNumber = 0; - while ((s = readLine()) !== null) { + var loc = function() { + return fs.realpathSync(inFilename) + ':' + lineNumber; + }; + while ((line = readLine()) !== null) { ++lineNumber; - var m = control.exec(s); + var m = control.exec(line); if (m) { switch (m[1]) { case 'if': stack.push(state); - try { - state = vm.runInNewContext(m[2], defines) ? 3 : 1; - } catch (e) { - console.error('Could not evalute line \'' + m[2] + '\' at ' + - fs.realpathSync(inFilename) + ':' + lineNumber); - throw e; + state = evaluateCondition(m[2]) ? STATE_IF_TRUE : STATE_IF_FALSE; + break; + case 'elif': + if (state === STATE_IF_TRUE) { + state = STATE_ELSE_FALSE; + } else if (state === STATE_IF_FALSE) { + state = evaluateCondition(m[2]) ? STATE_IF_TRUE : STATE_IF_FALSE; + } else if (state === STATE_ELSE_TRUE || state === STATE_ELSE_FALSE) { + throw new Error('Found #elif after #else at ' + loc()); + } else { + throw new Error('Found #elif without matching #if at ' + loc()); } break; case 'else': - state = state === 1 ? 3 : 2; + if (state === STATE_IF_TRUE) { + state = STATE_ELSE_FALSE; + } else if (state === STATE_IF_FALSE) { + state = STATE_ELSE_TRUE; + } else { + throw new Error('Found #else without matching #if at ' + loc()); + } break; case 'endif': + if (state === STATE_NONE) { + throw new Error('Found #endif without #if at ' + loc()); + } state = stack.pop(); break; case 'expand': - if (state === 0 || state === 3) { + if (state !== STATE_IF_FALSE && state !== STATE_ELSE_FALSE) { expand(m[2]); } break; case 'include': - if (state === 0 || state === 3) { + if (state !== STATE_IF_FALSE && state !== STATE_ELSE_FALSE) { include(m[2]); } break; + case 'error': + if (state !== STATE_IF_FALSE && state !== STATE_ELSE_FALSE) { + throw new Error('Found #error ' + m[2] + ' at ' + loc()); + } + break; } } else { - if (state === 0) { - writeLine(s); - } else if (state === 3) { - writeLine(s.replace(/^\/\/|^/g, ' ')); + if (state === STATE_NONE) { + writeLine(line); + } else if ((state === STATE_IF_TRUE || state === STATE_ELSE_TRUE) && + stack.indexOf(STATE_IF_FALSE) === -1 && + stack.indexOf(STATE_ELSE_FALSE) === -1) { + writeLine(line.replace(/^\/\/|^$/g, ' ')); } } } - if (state !== 0 || stack.length !== 0) { - throw new Error('Missing endif in preprocessor.'); + if (state !== STATE_NONE || stack.length !== 0) { + throw new Error('Missing #endif in preprocessor for ' + + fs.realpathSync(inFilename)); } if (typeof outFilename !== 'function') { fs.writeFileSync(outFilename, out); @@ -235,21 +309,29 @@ function getWorkerSrcFiles(filePath) { var src = fs.readFileSync(filePath).toString(); var reSrcFiles = /var\s+otherFiles\s*=\s*(\[[^\]]*\])/; var match = reSrcFiles.exec(src); + if (!match) { + throw new Error('Cannot find otherFiles array in ' + filePath); + } + + var files = match[1].replace(/'/g, '"').replace(/^\s*\/\/.*/gm, '') + .replace(/,\s*]$/, ']'); try { - var files = JSON.parse(match[1].replace(/'/g, '"')); - var srcFiles = files.filter(function(name) { - return name.indexOf('external') === -1; - }); - var externalSrcFiles = files.filter(function(name) { - return name.indexOf('external') > -1; - }); - return { - srcFiles: srcFiles, - externalSrcFiles: externalSrcFiles - }; - } catch(e) { - return {}; + files = JSON.parse(files); + } catch (e) { + throw new Error('Failed to parse otherFiles in ' + filePath + ' as JSON, ' + + e); } + + var srcFiles = files.filter(function(name) { + return name.indexOf('external') === -1; + }); + var externalSrcFiles = files.filter(function(name) { + return name.indexOf('external') > -1; + }); + return { + srcFiles: srcFiles, + externalSrcFiles: externalSrcFiles + }; } exports.getWorkerSrcFiles = getWorkerSrcFiles; diff --git a/external/builder/fixtures/confusing-comment-expected.js b/external/builder/fixtures/confusing-comment-expected.js new file mode 100644 index 0000000000000..018f08436a665 --- /dev/null +++ b/external/builder/fixtures/confusing-comment-expected.js @@ -0,0 +1,4 @@ +'use strict'; +var i = 0; +while(i-->0) { +} diff --git a/external/builder/fixtures/confusing-comment.js b/external/builder/fixtures/confusing-comment.js new file mode 100644 index 0000000000000..ac626b90633e0 --- /dev/null +++ b/external/builder/fixtures/confusing-comment.js @@ -0,0 +1,6 @@ +'use strict'; +//#if TRUE +var i = 0; +while(i-->0) { +} +//#endif diff --git a/external/builder/fixtures/elif-expected.js b/external/builder/fixtures/elif-expected.js new file mode 100644 index 0000000000000..c2ef779da866b --- /dev/null +++ b/external/builder/fixtures/elif-expected.js @@ -0,0 +1 @@ +//Error: Found #elif without matching #if at __filename:2 diff --git a/external/builder/fixtures/elif.js b/external/builder/fixtures/elif.js new file mode 100644 index 0000000000000..3900de2bc2983 --- /dev/null +++ b/external/builder/fixtures/elif.js @@ -0,0 +1,4 @@ +'use strict'; +//#elif TRUE +var a; +//#endif diff --git a/external/builder/fixtures/else-expected.js b/external/builder/fixtures/else-expected.js new file mode 100644 index 0000000000000..a623ae5d39033 --- /dev/null +++ b/external/builder/fixtures/else-expected.js @@ -0,0 +1 @@ +//Error: Found #else without matching #if at __filename:2 diff --git a/external/builder/fixtures/else.js b/external/builder/fixtures/else.js new file mode 100644 index 0000000000000..f28b328d8b7ee --- /dev/null +++ b/external/builder/fixtures/else.js @@ -0,0 +1,3 @@ +'use strict'; +//#else +//#endif diff --git a/external/builder/fixtures/error-expected.js b/external/builder/fixtures/error-expected.js new file mode 100644 index 0000000000000..7e061ad1f1946 --- /dev/null +++ b/external/builder/fixtures/error-expected.js @@ -0,0 +1 @@ +//Error: Found #error "Some Error" at __filename:3 diff --git a/external/builder/fixtures/error-false-expected.js b/external/builder/fixtures/error-false-expected.js new file mode 100644 index 0000000000000..0220a0031b0e7 --- /dev/null +++ b/external/builder/fixtures/error-false-expected.js @@ -0,0 +1,2 @@ +'use strict'; +var a; diff --git a/external/builder/fixtures/error-false.js b/external/builder/fixtures/error-false.js new file mode 100644 index 0000000000000..e5b8153dad018 --- /dev/null +++ b/external/builder/fixtures/error-false.js @@ -0,0 +1,5 @@ +'use strict'; +//#if FALSE +//#error "Some Error" +//#endif +var a; diff --git a/external/builder/fixtures/error.js b/external/builder/fixtures/error.js new file mode 100644 index 0000000000000..facd0e86e0dde --- /dev/null +++ b/external/builder/fixtures/error.js @@ -0,0 +1,5 @@ +'use strict'; +//#if TRUE +//#error "Some Error" +//#endif +var b; diff --git a/external/builder/fixtures/expand-expected.html b/external/builder/fixtures/expand-expected.html new file mode 100644 index 0000000000000..89cc081779f00 --- /dev/null +++ b/external/builder/fixtures/expand-expected.html @@ -0,0 +1 @@ +prefixtruesuffix diff --git a/external/builder/fixtures/expand.html b/external/builder/fixtures/expand.html new file mode 100644 index 0000000000000..a2247ea3d4330 --- /dev/null +++ b/external/builder/fixtures/expand.html @@ -0,0 +1 @@ + diff --git a/external/builder/fixtures/if-empty-expected.js b/external/builder/fixtures/if-empty-expected.js new file mode 100644 index 0000000000000..5dd4f66a2135e --- /dev/null +++ b/external/builder/fixtures/if-empty-expected.js @@ -0,0 +1 @@ +//Error: No JavaScript expression given at __filename:2 diff --git a/external/builder/fixtures/if-empty.js b/external/builder/fixtures/if-empty.js new file mode 100644 index 0000000000000..81ad761e5c0d5 --- /dev/null +++ b/external/builder/fixtures/if-empty.js @@ -0,0 +1,6 @@ +'use strict'; +//#if +var a; +//#else +var b; +//#endif diff --git a/external/builder/fixtures/if-false-elif-false-else-expected.js b/external/builder/fixtures/if-false-elif-false-else-expected.js new file mode 100644 index 0000000000000..724f9bd972136 --- /dev/null +++ b/external/builder/fixtures/if-false-elif-false-else-expected.js @@ -0,0 +1,2 @@ +'use strict'; +var c; diff --git a/external/builder/fixtures/if-false-elif-false-else.js b/external/builder/fixtures/if-false-elif-false-else.js new file mode 100644 index 0000000000000..e3f6aca936a4b --- /dev/null +++ b/external/builder/fixtures/if-false-elif-false-else.js @@ -0,0 +1,8 @@ +'use strict'; +//#if FALSE +var a; +//#elif FALSE +var b; +//#else +var c; +//#endif diff --git a/external/builder/fixtures/if-false-elif-true-else-expected.js b/external/builder/fixtures/if-false-elif-true-else-expected.js new file mode 100644 index 0000000000000..fe04f49586958 --- /dev/null +++ b/external/builder/fixtures/if-false-elif-true-else-expected.js @@ -0,0 +1,2 @@ +'use strict'; +var b; diff --git a/external/builder/fixtures/if-false-elif-true-else.js b/external/builder/fixtures/if-false-elif-true-else.js new file mode 100644 index 0000000000000..2a37d2fba4c93 --- /dev/null +++ b/external/builder/fixtures/if-false-elif-true-else.js @@ -0,0 +1,8 @@ +'use strict'; +//#if FALSE +var a; +//#elif TRUE +var b; +//#else +var c; +//#endif diff --git a/external/builder/fixtures/if-false-else-expected.js b/external/builder/fixtures/if-false-else-expected.js new file mode 100644 index 0000000000000..fe04f49586958 --- /dev/null +++ b/external/builder/fixtures/if-false-else-expected.js @@ -0,0 +1,2 @@ +'use strict'; +var b; diff --git a/external/builder/fixtures/if-false-else.js b/external/builder/fixtures/if-false-else.js new file mode 100644 index 0000000000000..831ca99452f25 --- /dev/null +++ b/external/builder/fixtures/if-false-else.js @@ -0,0 +1,6 @@ +'use strict'; +//#if FALSE +var a; +//#else +var b; +//#endif diff --git a/external/builder/fixtures/if-nested-expected.js b/external/builder/fixtures/if-nested-expected.js new file mode 100644 index 0000000000000..7dd77df9ef5a0 --- /dev/null +++ b/external/builder/fixtures/if-nested-expected.js @@ -0,0 +1,6 @@ +'use strict'; +var a; + +var b; + +var d; diff --git a/external/builder/fixtures/if-nested.js b/external/builder/fixtures/if-nested.js new file mode 100644 index 0000000000000..d49180a47aa32 --- /dev/null +++ b/external/builder/fixtures/if-nested.js @@ -0,0 +1,19 @@ +'use strict'; +//#if TRUE +var a; + +//#if TRUE +var b; +//#else +var c; +//#endif + +var d; +//#else +var e; +//#if TRUE +var f; +//#endif + +var g; +//#endif diff --git a/external/builder/fixtures/if-true-else-expected.js b/external/builder/fixtures/if-true-else-expected.js new file mode 100644 index 0000000000000..0220a0031b0e7 --- /dev/null +++ b/external/builder/fixtures/if-true-else-expected.js @@ -0,0 +1,2 @@ +'use strict'; +var a; diff --git a/external/builder/fixtures/if-true-else.js b/external/builder/fixtures/if-true-else.js new file mode 100644 index 0000000000000..c4ed6cb1f7cab --- /dev/null +++ b/external/builder/fixtures/if-true-else.js @@ -0,0 +1,6 @@ +'use strict'; +//#if TRUE +var a; +//#else +var b; +//#endif diff --git a/external/builder/fixtures/if-unclosed-expected.js b/external/builder/fixtures/if-unclosed-expected.js new file mode 100644 index 0000000000000..0ad021009a75b --- /dev/null +++ b/external/builder/fixtures/if-unclosed-expected.js @@ -0,0 +1 @@ +//Error: Missing #endif in preprocessor for __filename diff --git a/external/builder/fixtures/if-unclosed.js b/external/builder/fixtures/if-unclosed.js new file mode 100644 index 0000000000000..b74a4fb55c607 --- /dev/null +++ b/external/builder/fixtures/if-unclosed.js @@ -0,0 +1,3 @@ +'use strict'; +//#if TRUE +var a; diff --git a/external/builder/fixtures/include-expected.html b/external/builder/fixtures/include-expected.html new file mode 100644 index 0000000000000..0c0f019a6a4ae --- /dev/null +++ b/external/builder/fixtures/include-expected.html @@ -0,0 +1,5 @@ + diff --git a/external/builder/fixtures/include-non-existent-expected.html b/external/builder/fixtures/include-non-existent-expected.html new file mode 100644 index 0000000000000..c9a0b5b1c33ed --- /dev/null +++ b/external/builder/fixtures/include-non-existent-expected.html @@ -0,0 +1 @@ +//Error: Failed to include "some file that does not exist" at __filename:2 diff --git a/external/builder/fixtures/include-non-existent.html b/external/builder/fixtures/include-non-existent.html new file mode 100644 index 0000000000000..9b892ab324074 --- /dev/null +++ b/external/builder/fixtures/include-non-existent.html @@ -0,0 +1,2 @@ + + diff --git a/external/builder/fixtures/include.html b/external/builder/fixtures/include.html new file mode 100644 index 0000000000000..44352b514b5b6 --- /dev/null +++ b/external/builder/fixtures/include.html @@ -0,0 +1,5 @@ + diff --git a/external/builder/fixtures/js-comment-expected.js b/external/builder/fixtures/js-comment-expected.js new file mode 100644 index 0000000000000..a0b6ada86c289 --- /dev/null +++ b/external/builder/fixtures/js-comment-expected.js @@ -0,0 +1,4 @@ +'use strict'; + //var a; + var b; +var c; diff --git a/external/builder/fixtures/js-comment.js b/external/builder/fixtures/js-comment.js new file mode 100644 index 0000000000000..361f9e832aebc --- /dev/null +++ b/external/builder/fixtures/js-comment.js @@ -0,0 +1,6 @@ +'use strict'; +//#if TRUE +////var a; +//var b; +var c; +//#endif diff --git a/external/builder/fixtures/undefined-define-expected.js b/external/builder/fixtures/undefined-define-expected.js new file mode 100644 index 0000000000000..67bfbca98b950 --- /dev/null +++ b/external/builder/fixtures/undefined-define-expected.js @@ -0,0 +1,2 @@ +//Error: Could not evaluate "notdefined" at __filename:2 +//ReferenceError: notdefined is not defined diff --git a/external/builder/fixtures/undefined-define.js b/external/builder/fixtures/undefined-define.js new file mode 100644 index 0000000000000..a59f4165e61bf --- /dev/null +++ b/external/builder/fixtures/undefined-define.js @@ -0,0 +1,6 @@ +'use strict'; +//#if notdefined +var a; +//#else +var b; +//#endif diff --git a/external/builder/fixtures/unsupported-ifdef-expected.js b/external/builder/fixtures/unsupported-ifdef-expected.js new file mode 100644 index 0000000000000..7b9907b537196 --- /dev/null +++ b/external/builder/fixtures/unsupported-ifdef-expected.js @@ -0,0 +1 @@ +//Error: Found #endif without #if at __filename:4 diff --git a/external/builder/fixtures/unsupported-ifdef.js b/external/builder/fixtures/unsupported-ifdef.js new file mode 100644 index 0000000000000..b3c1d817567cd --- /dev/null +++ b/external/builder/fixtures/unsupported-ifdef.js @@ -0,0 +1,5 @@ +'use strict'; +//#ifdef TRUE +//ifdef should not be recognized +//#endif +var a; diff --git a/external/builder/test.js b/external/builder/test.js new file mode 100644 index 0000000000000..274c68f603797 --- /dev/null +++ b/external/builder/test.js @@ -0,0 +1,54 @@ +/* -*- Mode: Java; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set shiftwidth=2 tabstop=2 autoindent cindent expandtab: */ +/* jshint node:true */ +/* globals cat, cd, echo, ls */ +'use strict'; + +require('shelljs/make'); + +var builder = require('./builder'); +var fs = require('fs'); +var path = require('path'); + +var errors = 0; + +cd(__dirname); +cd('fixtures'); +ls('*-expected.*').forEach(function(expectationFilename) { + var inFilename = expectationFilename.replace('-expected', ''); + var expectation = cat(expectationFilename).trim() + .replace(/__filename/g, fs.realpathSync(inFilename)); + var outLines = []; + + var outFilename = function(line) { + outLines.push(line); + }; + var defines = { + TRUE: true, + FALSE: false, + }; + var out; + try { + builder.preprocess(inFilename, outFilename, defines); + out = outLines.join('\n').trim(); + } catch (e) { + out = ('Error: ' + e.message).replace(/^/gm, '//'); + } + if (out !== expectation) { + echo('Assertion failed for ' + inFilename); + echo('--------------------------------------------------'); + echo('EXPECTED:'); + echo(expectation); + echo('--------------------------------------------------'); + echo('ACTUAL'); + echo(out); + echo('--------------------------------------------------'); + echo(); + } +}); + +if (errors) { + echo('Found ' + errors + ' expectation failures.'); +} else { + echo('All tests completed without errors.'); +} diff --git a/make.js b/make.js index d82db2e178e5b..b35b68e44c8cc 100644 --- a/make.js +++ b/make.js @@ -482,9 +482,8 @@ target.bundle = function(args) { cd(ROOT_DIR); echo(); echo('### Bundling files into ' + BUILD_TARGET); - var reg = /\n\/\* -\*- Mode(.|\n)*?Mozilla Foundation(.|\n)*?'use strict';/g; - function bundle(filename, dir, SRC_FILES, EXT_SRC_FILES) { + function bundle(filename, outfilename, SRC_FILES, EXT_SRC_FILES) { for (var i = 0, length = excludes.length; i < length; ++i) { var exclude = excludes[i]; var index = SRC_FILES.indexOf(exclude); @@ -500,15 +499,17 @@ target.bundle = function(args) { crlfchecker.checkIfCrlfIsPresent(SRC_FILES); - // Strip out all the vim/license headers. - bundleContent = bundleContent.replace(reg, ''); + // Prepend a newline because stripCommentHeaders only strips comments that + // follow a line feed. The file where bundleContent is inserted already + // contains a license header, so the header of bundleContent can be removed. + bundleContent = stripCommentHeaders('\n' + bundleContent); // Append external files last since we don't want to modify them. bundleContent += cat(EXT_SRC_FILES); // This just preprocesses the empty pdf.js file, we don't actually want to // preprocess everything yet since other build targets use this file. - builder.preprocess(filename, dir, builder.merge(defines, + builder.preprocess(filename, outfilename, builder.merge(defines, {BUNDLE: bundleContent, BUNDLE_VERSION: bundleVersion, BUNDLE_BUILD: bundleBuild})); @@ -600,12 +601,21 @@ target.singlefile = function() { }; +function stripCommentHeaders(content, filename) { + // Strip out all the vim/license headers. + var notEndOfComment = '(?:[^*]|\\*(?!/))+'; + var reg = new RegExp( + '\n/\\* -\\*- Mode' + notEndOfComment + '\\*/\\s*' + + '(?:/\\*' + notEndOfComment + '\\*/\\s*|//(?!#).*\n\\s*)+' + + '\'use strict\';', 'g'); + content = content.replace(reg, ''); + return content; +} + function cleanupJSSource(file) { var content = cat(file); - // Strip out all the vim/license headers. - var reg = /\n\/\* -\*- Mode(.|\n)*?Mozilla Foundation(.|\n)*?'use strict';/g; - content = content.replace(reg, ''); + content = stripCommentHeaders(content, file); content.to(file); }