From 2b8a1d2fb6b57b8c77cf52e36ba66ec36553e7fd Mon Sep 17 00:00:00 2001 From: Tobias Sorn Date: Tue, 16 Jul 2019 11:09:47 +0200 Subject: [PATCH 01/27] [FEATURE] Properties File Escaping Add processor which escapes non ASCII characters in resources. --- lib/processors/stringEscaper.js | 59 +++++++++++++++++++++ package-lock.json | 15 ++++-- package.json | 1 + test/lib/processors/stringEscaper.js | 76 ++++++++++++++++++++++++++++ 4 files changed, 146 insertions(+), 5 deletions(-) create mode 100644 lib/processors/stringEscaper.js create mode 100644 test/lib/processors/stringEscaper.js diff --git a/lib/processors/stringEscaper.js b/lib/processors/stringEscaper.js new file mode 100644 index 000000000..51459d64b --- /dev/null +++ b/lib/processors/stringEscaper.js @@ -0,0 +1,59 @@ +const escapeUnicode = require("escape-unicode"); + +/** + * @see https://ascii.cl/ + * @type {number} + */ +const NUMBER_OF_ASCII_CHARACTERS = 127; + +// use memoization for escapeUnicode function for performance +const memoizeEscapeUnicodeMap = {}; +const memoizeEscapeUnicode = function(sChar) { + if (memoizeEscapeUnicodeMap[sChar]) { + return memoizeEscapeUnicodeMap[sChar]; + } + memoizeEscapeUnicodeMap[sChar] = escapeUnicode(sChar); + return memoizeEscapeUnicodeMap[sChar]; +}; + +/** + * Escapes non ASCII characters with unicode characters. + * + * @see https://ascii.cl/ + * @see https://tools.ietf.org/html/rfc5137#section-6.1 + * + * + * @param {string} string input string with non ascii characters, e.g. L♥VE + * @returns {{string: (string), modified: boolean}} output string with all non ascii characters being escaped by unicode sequence, e.g. L\u2665VE + */ +const escapeNonAscii = function(string) { + let result = ""; + let modified = false; + for (let i = 0; i < string.length; i++) { + const char = string[i]; + // check for non ascii characters + if (string.charCodeAt(i) > NUMBER_OF_ASCII_CHARACTERS) { + result += memoizeEscapeUnicode(char); + modified = true; + } else { + result += char; + } + } + return { + modified, + string: result + }; +}; + +module.exports = function({resources}) { + return Promise.all(resources.map((resource) => { + return resource.getString().then((resourceString) => { + const escaped = escapeNonAscii(resourceString); + // only modify the resource's string if it was changed + if (escaped.modified) { + resource.setString(escaped.string); + } + return resource; + }); + })); +}; diff --git a/package-lock.json b/package-lock.json index a9bb42830..1b7bf02da 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2255,6 +2255,11 @@ "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=" }, + "escape-unicode": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/escape-unicode/-/escape-unicode-0.2.0.tgz", + "integrity": "sha512-7jMQuKb8nm0h/9HYLfu4NCLFwoUsd5XO6OZ1z86PbKcMf8zDK1m7nFR0iA2CCShq4TSValaLIveE8T1UBxgALQ==" + }, "escodegen": { "version": "1.11.1", "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-1.11.1.tgz", @@ -4402,7 +4407,7 @@ }, "find-up": { "version": "3.0.0", - "resolved": "https://registry.npmjs.org/find-up/-/find-up-3.0.0.tgz", + "resolved": false, "integrity": "sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg==", "dev": true, "requires": { @@ -4411,7 +4416,7 @@ }, "locate-path": { "version": "3.0.0", - "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-3.0.0.tgz", + "resolved": false, "integrity": "sha512-7AO748wWnIhNqAuaty2ZWHkQHRSNfPVIsPIfwEOWO22AmaoVrWavlOcMR5nzTLNYvp36X220/maaRsrec1G65A==", "dev": true, "requires": { @@ -4431,7 +4436,7 @@ }, "p-locate": { "version": "3.0.0", - "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-3.0.0.tgz", + "resolved": false, "integrity": "sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ==", "dev": true, "requires": { @@ -4440,13 +4445,13 @@ }, "path-exists": { "version": "3.0.0", - "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz", + "resolved": false, "integrity": "sha1-zg6+ql94yxiSXqfYENe1mwEP1RU=", "dev": true }, "resolve-from": { "version": "4.0.0", - "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", + "resolved": false, "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", "dev": true }, diff --git a/package.json b/package.json index 8fefaf0ed..852e98cba 100644 --- a/package.json +++ b/package.json @@ -99,6 +99,7 @@ "@ui5/fs": "^1.1.2", "@ui5/logger": "^1.0.1", "cheerio": "^0.22.0", + "escape-unicode": "^0.2.0", "escodegen": "^1.11.1", "escope": "^3.6.0", "esprima": "^4.0.1", diff --git a/test/lib/processors/stringEscaper.js b/test/lib/processors/stringEscaper.js new file mode 100644 index 000000000..4208e03ca --- /dev/null +++ b/test/lib/processors/stringEscaper.js @@ -0,0 +1,76 @@ +const test = require("ava"); + +const stringEscaper = require("../../../lib/processors/stringEscaper"); + +/** + * Executes string escaping. Returns undefined if nothing was escaped. + * + * @param {string} input string + * @returns {Promise} escaped string if non-ascii characters present, undefined otherwise + */ +const escape = async function(input) { + let result = undefined; + const resource = { + getString: () => Promise.resolve(input), + setString: (actual) => { + result = actual; + } + }; + await stringEscaper({resources: [resource]}); + return result; +}; + +test("Replace symbol characters", async (t) => { + t.plan(1); + + const input = `L♥VE is everywhere`; + const expected = `L\\u2665VE is everywhere`; + const output = await escape(input); + t.deepEqual(output, expected, "Correct file content should be set"); +}); + +test("Replace chinese characters", async (t) => { + t.plan(1); + + const input = `These are 人物 characters`; + const expected = "These are \\u4eba\\u7269 characters"; + const output = await escape(input); + t.deepEqual(output, expected, "Correct file content should be set"); +}); + +test("Replace umlaut characters", async (t) => { + t.plan(1); + + const input = `Achso Ähem`; + const expected = "Achso \\u00c4hem"; + const output = await escape(input); + t.deepEqual(output, expected, "Correct file content should be set"); +}); + +test("Replace constructed characters", async (t) => { + t.plan(1); + + const input = `Oh ẛ̣ that's ẛ̣ yes`; + const expected = "Oh \\u1e9b\\u0323 that's \\u1e9b\\u0323 yes"; + const output = await escape(input); + t.deepEqual(output, expected, "Correct file content should be set"); +}); + + +test("Replace multiple times same character", async (t) => { + t.plan(1); + + const input = `♥H L♥VE AND HARM♥NY ♥MG`; + const expected = "\\u2665H L\\u2665VE AND HARM\\u2665NY \\u2665MG"; + const output = await escape(input); + t.deepEqual(output, expected, "Correct file content should be set"); +}); + +test("No Replace of characters", async (t) => { + t.plan(1); + + const input = `ONE LOVE`; + const expected = undefined; + const output = await escape(input); + t.deepEqual(output, expected, "Correct file content should not be modified"); +}); From d4f1e62d307ca4165c87c484e00ce6ca4f214a40 Mon Sep 17 00:00:00 2001 From: Tobias Sorn Date: Tue, 16 Jul 2019 14:26:34 +0200 Subject: [PATCH 02/27] [FEATURE] Properties File Escaping Add task escapeNonAsciiCharacters which uses stringEscaper to escape non ascii characters. Made it generic using a pattern to re-use it for potential other files. --- lib/tasks/escapeNonAsciiCharacters.js | 43 ++++++++ lib/tasks/taskRepository.js | 1 + test/lib/tasks/escapeNonAsciiCharacters.js | 116 +++++++++++++++++++++ 3 files changed, 160 insertions(+) create mode 100644 lib/tasks/escapeNonAsciiCharacters.js create mode 100644 test/lib/tasks/escapeNonAsciiCharacters.js diff --git a/lib/tasks/escapeNonAsciiCharacters.js b/lib/tasks/escapeNonAsciiCharacters.js new file mode 100644 index 000000000..def0819e0 --- /dev/null +++ b/lib/tasks/escapeNonAsciiCharacters.js @@ -0,0 +1,43 @@ +const stringEscaper = require("../processors/stringEscaper"); + +/** + * Encodings + * + * @type {{UTF_8: string, ISO_8859_1: string}} + */ +const ENCODING_TYPES = { + UTF_8: "UTF-8", + ISO_8859_1: "ISO-8859-1" +}; + +/** + * Task to escape non ascii characters in properties files resources. + * + * @public + * @alias module:@ui5/builder.tasks.escapePropertiesFiles + * @param {Object} parameters Parameters + * @param {module:@ui5/fs.DuplexCollection} parameters.workspace DuplexCollection to read and write files + * @param {Object} parameters.options Options + * @param {string} parameters.options.pattern Pattern to locate the files to be processed + * @param {string} [parameters.options.sourceEncoding] source file encoding: "UTF-8" or "ISO-8859-1". Defaults to "ISO-8859-1" + * @returns {Promise} Promise resolving with undefined once data has been written + */ +module.exports = function({workspace, options}) { + if (options.sourceEncoding && !Object.values(ENCODING_TYPES).includes(options.sourceEncoding)) { + return Promise.reject(new Error(`Invalid encoding specified: '${options.sourceEncoding}'. Must be one of ${Object.values(ENCODING_TYPES)}`)); + } + if (options.sourceEncoding && options.sourceEncoding === ENCODING_TYPES.UTF_8) { + return Promise.resolve(); + } + return workspace.byGlob(options.pattern) + .then((allResources) => { + return stringEscaper({ + resources: allResources + }); + }) + .then((processedResources) => { + return Promise.all(processedResources.map((resource) => { + return workspace.write(resource); + })); + }); +}; diff --git a/lib/tasks/taskRepository.js b/lib/tasks/taskRepository.js index b8a4dbb06..b45560f48 100644 --- a/lib/tasks/taskRepository.js +++ b/lib/tasks/taskRepository.js @@ -2,6 +2,7 @@ const tasks = { replaceCopyright: require("./replaceCopyright"), replaceVersion: require("./replaceVersion"), createDebugFiles: require("./createDebugFiles"), + escapeNonAsciiCharacters: require("./escapeNonAsciiCharacters"), executeJsdocSdkTransformation: require("./jsdoc/executeJsdocSdkTransformation"), generateApiIndex: require("./jsdoc/generateApiIndex"), generateJsdoc: require("./jsdoc/generateJsdoc"), diff --git a/test/lib/tasks/escapeNonAsciiCharacters.js b/test/lib/tasks/escapeNonAsciiCharacters.js new file mode 100644 index 000000000..28b08b877 --- /dev/null +++ b/test/lib/tasks/escapeNonAsciiCharacters.js @@ -0,0 +1,116 @@ +const test = require("ava"); + +const ui5Builder = require("../../../"); +const tasks = ui5Builder.builder.tasks; +const ui5Fs = require("@ui5/fs"); +const resourceFactory = ui5Fs.resourceFactory; +const DuplexCollection = ui5Fs.DuplexCollection; + +test("integration: escape non ascii characters", (t) => { + const reader = resourceFactory.createAdapter({ + virBasePath: "/" + }); + const writer = resourceFactory.createAdapter({ + virBasePath: "/" + }); + const workspace = new DuplexCollection({reader, writer}); + + const content = `welcome=Willkommen {0}. Bitte geben Sie einen neuen Kontakt ein: +lastname=Nachname: +firstname=Vorname: +street=Straße: +zip=PLZ: +city=Ort:`; + + const expected = `welcome=Willkommen {0}. Bitte geben Sie einen neuen Kontakt ein: +lastname=Nachname: +firstname=Vorname: +street=Stra\\u00dfe: +zip=PLZ: +city=Ort:`; + + const resource = resourceFactory.createResource({ + path: "/i18n.properties", + string: content + }); + + return workspace.write(resource).then(() => { + return tasks.escapePropertiesFiles({ + workspace, + options: { + pattern: "/**/*.properties" + } + }).then(() => { + return writer.byPath("/i18n.properties").then((resource) => { + if (!resource) { + t.fail("Could not find /i18n.properties in target"); + } else { + return resource.getString(); + } + }); + }).then((result) => { + return t.deepEqual(result, expected); + }); + }); +}); + +test("integration: escape non ascii characters source encoding being UTF-8", (t) => { + const reader = resourceFactory.createAdapter({ + virBasePath: "/" + }); + const writer = resourceFactory.createAdapter({ + virBasePath: "/" + }); + const workspace = new DuplexCollection({reader, writer}); + + const content = `welcome=Willkommen {0}. Bitte geben Sie einen neuen Kontakt ein: +lastname=Nachname: +firstname=Vorname: +street=Straße: +zip=PLZ: +city=Ort:`; + + const expected = `welcome=Willkommen {0}. Bitte geben Sie einen neuen Kontakt ein: +lastname=Nachname: +firstname=Vorname: +street=Straße: +zip=PLZ: +city=Ort:`; + + const resource = resourceFactory.createResource({ + path: "/i18n.properties", + string: content + }); + + return workspace.write(resource).then(() => { + return tasks.escapePropertiesFiles({ + workspace, + options: { + sourceEncoding: "UTF-8", + pattern: "/**/*.properties" + } + }).then(() => { + return writer.byPath("/i18n.properties").then((resource) => { + if (!resource) { + t.fail("Could not find /i18n.properties in target"); + } else { + return resource.getString(); + } + }); + }).then((result) => { + return t.deepEqual(result, expected); + }); + }); +}); + +test("integration: escape non ascii characters source encoding being UTF-16", (t) => { + return tasks.escapePropertiesFiles({ + workspace: undefined, + options: { + sourceEncoding: "UTF-16", + pattern: "/**/*.properties" + } + }).catch((error) => { + return t.deepEqual(error.message, "Invalid encoding specified: 'UTF-16'. Must be one of UTF-8,ISO-8859-1"); + }); +}); From d1ffb977e5ced45318c8eb11caf78392cf26a100 Mon Sep 17 00:00:00 2001 From: Tobias Sorn Date: Tue, 16 Jul 2019 14:31:04 +0200 Subject: [PATCH 03/27] [FEATURE] Properties File Escaping Fix test --- test/lib/tasks/escapeNonAsciiCharacters.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/lib/tasks/escapeNonAsciiCharacters.js b/test/lib/tasks/escapeNonAsciiCharacters.js index 28b08b877..3e108fa42 100644 --- a/test/lib/tasks/escapeNonAsciiCharacters.js +++ b/test/lib/tasks/escapeNonAsciiCharacters.js @@ -35,7 +35,7 @@ city=Ort:`; }); return workspace.write(resource).then(() => { - return tasks.escapePropertiesFiles({ + return tasks.escapeNonAsciiCharacters({ workspace, options: { pattern: "/**/*.properties" @@ -83,7 +83,7 @@ city=Ort:`; }); return workspace.write(resource).then(() => { - return tasks.escapePropertiesFiles({ + return tasks.escapeNonAsciiCharacters({ workspace, options: { sourceEncoding: "UTF-8", @@ -104,7 +104,7 @@ city=Ort:`; }); test("integration: escape non ascii characters source encoding being UTF-16", (t) => { - return tasks.escapePropertiesFiles({ + return tasks.escapeNonAsciiCharacters({ workspace: undefined, options: { sourceEncoding: "UTF-16", From 355bac340fcfdd6fdfda9e16c480235c099a3a51 Mon Sep 17 00:00:00 2001 From: Tobias Sorn Date: Tue, 16 Jul 2019 16:19:41 +0200 Subject: [PATCH 04/27] [FEATURE] Properties File Escaping Add task "escapeNonAsciiCharacters" to ApplicationBuilder and LibraryBuilder. Add non-ascii characters to i18n test files. --- lib/types/application/ApplicationBuilder.js | 11 +++++++++++ lib/types/library/LibraryBuilder.js | 11 +++++++++++ .../build/application.b/dest/i18n/i18n.properties | 3 ++- .../build/application.b/dest/i18n/i18n_de.properties | 3 ++- .../dest/manifest-bundle/i18n/i18n.properties | 3 ++- .../dest/manifest-bundle/i18n/i18n_de.properties | 3 ++- .../build/application.b/standalone/i18n.properties | 3 ++- .../application.b/standalone/i18n/i18n.properties | 3 ++- .../application.b/standalone/i18n/i18n_de.properties | 3 ++- .../application.b/standalone/i18n/l10n.properties | 3 ++- .../standalone/resources/sap-ui-custom.js | 8 ++++---- test/fixtures/application.b/webapp/i18n.properties | 3 ++- .../application.b/webapp/i18n/i18n.properties | 3 ++- .../application.b/webapp/i18n/i18n_de.properties | 3 ++- .../application.b/webapp/i18n/l10n.properties | 3 ++- test/lib/types/application/ApplicationBuilder.js | 4 ++++ 16 files changed, 54 insertions(+), 16 deletions(-) diff --git a/lib/types/application/ApplicationBuilder.js b/lib/types/application/ApplicationBuilder.js index 0f7a393da..0ffd517c4 100644 --- a/lib/types/application/ApplicationBuilder.js +++ b/lib/types/application/ApplicationBuilder.js @@ -8,6 +8,7 @@ const tasks = { // can't require index.js due to circular dependency generateStandaloneAppBundle: require("../../tasks/bundlers/generateStandaloneAppBundle"), generateBundle: require("../../tasks/bundlers/generateBundle"), generateCachebusterInfo: require("../../tasks/generateCachebusterInfo"), + escapeNonAsciiCharacters: require("../../tasks/escapeNonAsciiCharacters"), buildThemes: require("../../tasks/buildThemes"), createDebugFiles: require("../../tasks/createDebugFiles"), generateVersionInfo: require("../../tasks/generateVersionInfo"), @@ -26,6 +27,16 @@ class ApplicationBuilder extends AbstractBuilder { "Also see: https://github.com/SAP/ui5-builder#application"); } + this.addTask("escapeNonAsciiCharacters", () => { + return tasks.escapeNonAsciiCharacters({ + workspace: resourceCollections.workspace, + options: { + sourceEncoding: project.resources && project.resources.propertiesFileEncoding, + pattern: "/**/*.properties" + } + }); + }); + this.addTask("replaceCopyright", () => { return tasks.replaceCopyright({ workspace: resourceCollections.workspace, diff --git a/lib/types/library/LibraryBuilder.js b/lib/types/library/LibraryBuilder.js index 806278ee8..a8f6c616e 100644 --- a/lib/types/library/LibraryBuilder.js +++ b/lib/types/library/LibraryBuilder.js @@ -6,6 +6,7 @@ const tasks = { // can't require index.js due to circular dependency generateLibraryPreload: require("../../tasks/bundlers/generateLibraryPreload"), generateManifestBundle: require("../../tasks/bundlers/generateManifestBundle"), generateStandaloneAppBundle: require("../../tasks/bundlers/generateStandaloneAppBundle"), + escapeNonAsciiCharacters: require("../../tasks/escapeNonAsciiCharacters"), buildThemes: require("../../tasks/buildThemes"), createDebugFiles: require("../../tasks/createDebugFiles"), generateJsdoc: require("../../tasks/jsdoc/generateJsdoc"), @@ -25,6 +26,16 @@ class LibraryBuilder extends AbstractBuilder { "Also see: https://github.com/SAP/ui5-builder#library"); } + this.addTask("escapeNonAsciiCharacters", () => { + return tasks.escapeNonAsciiCharacters({ + workspace: resourceCollections.workspace, + options: { + sourceEncoding: project.resources && project.resources.propertiesFileEncoding, + pattern: "/**/*.properties" + } + }); + }); + this.addTask("replaceCopyright", () => { const replaceCopyright = tasks.replaceCopyright; return replaceCopyright({ diff --git a/test/expected/build/application.b/dest/i18n/i18n.properties b/test/expected/build/application.b/dest/i18n/i18n.properties index 575fb20d0..17772dbf5 100644 --- a/test/expected/build/application.b/dest/i18n/i18n.properties +++ b/test/expected/build/application.b/dest/i18n/i18n.properties @@ -1 +1,2 @@ -title=app-i18n \ No newline at end of file +title=app-i18n +fame=Stra\u00dfe \ No newline at end of file diff --git a/test/expected/build/application.b/dest/i18n/i18n_de.properties b/test/expected/build/application.b/dest/i18n/i18n_de.properties index f58858038..d2c23da4b 100644 --- a/test/expected/build/application.b/dest/i18n/i18n_de.properties +++ b/test/expected/build/application.b/dest/i18n/i18n_de.properties @@ -1 +1,2 @@ -title=app-i18n_de \ No newline at end of file +title=app-i18n_de +fame=Stra\u00dfe \ No newline at end of file diff --git a/test/expected/build/application.b/dest/manifest-bundle/i18n/i18n.properties b/test/expected/build/application.b/dest/manifest-bundle/i18n/i18n.properties index 575fb20d0..49f4756f9 100644 --- a/test/expected/build/application.b/dest/manifest-bundle/i18n/i18n.properties +++ b/test/expected/build/application.b/dest/manifest-bundle/i18n/i18n.properties @@ -1 +1,2 @@ -title=app-i18n \ No newline at end of file +title=app-i18n +fame=Straße \ No newline at end of file diff --git a/test/expected/build/application.b/dest/manifest-bundle/i18n/i18n_de.properties b/test/expected/build/application.b/dest/manifest-bundle/i18n/i18n_de.properties index f58858038..a81193360 100644 --- a/test/expected/build/application.b/dest/manifest-bundle/i18n/i18n_de.properties +++ b/test/expected/build/application.b/dest/manifest-bundle/i18n/i18n_de.properties @@ -1 +1,2 @@ -title=app-i18n_de \ No newline at end of file +title=app-i18n_de +fame=Straße \ No newline at end of file diff --git a/test/expected/build/application.b/standalone/i18n.properties b/test/expected/build/application.b/standalone/i18n.properties index 88b84f602..d8d43be34 100644 --- a/test/expected/build/application.b/standalone/i18n.properties +++ b/test/expected/build/application.b/standalone/i18n.properties @@ -1 +1,2 @@ -title=app-i18n-wrong \ No newline at end of file +title=app-i18n-wrong +fame=Straße \ No newline at end of file diff --git a/test/expected/build/application.b/standalone/i18n/i18n.properties b/test/expected/build/application.b/standalone/i18n/i18n.properties index 575fb20d0..49f4756f9 100644 --- a/test/expected/build/application.b/standalone/i18n/i18n.properties +++ b/test/expected/build/application.b/standalone/i18n/i18n.properties @@ -1 +1,2 @@ -title=app-i18n \ No newline at end of file +title=app-i18n +fame=Straße \ No newline at end of file diff --git a/test/expected/build/application.b/standalone/i18n/i18n_de.properties b/test/expected/build/application.b/standalone/i18n/i18n_de.properties index f58858038..a81193360 100644 --- a/test/expected/build/application.b/standalone/i18n/i18n_de.properties +++ b/test/expected/build/application.b/standalone/i18n/i18n_de.properties @@ -1 +1,2 @@ -title=app-i18n_de \ No newline at end of file +title=app-i18n_de +fame=Straße \ No newline at end of file diff --git a/test/expected/build/application.b/standalone/i18n/l10n.properties b/test/expected/build/application.b/standalone/i18n/l10n.properties index 88b84f602..d8d43be34 100644 --- a/test/expected/build/application.b/standalone/i18n/l10n.properties +++ b/test/expected/build/application.b/standalone/i18n/l10n.properties @@ -1 +1,2 @@ -title=app-i18n-wrong \ No newline at end of file +title=app-i18n-wrong +fame=Straße \ No newline at end of file diff --git a/test/expected/build/application.b/standalone/resources/sap-ui-custom.js b/test/expected/build/application.b/standalone/resources/sap-ui-custom.js index 5da6683a2..906641067 100644 --- a/test/expected/build/application.b/standalone/resources/sap-ui-custom.js +++ b/test/expected/build/application.b/standalone/resources/sap-ui-custom.js @@ -5,10 +5,10 @@ jQuery.sap.registerPreloadedModules({ "id1/embedded/i18n/i18n_de.properties":'title=embedded-i18n_de', "id1/embedded/i18n_fr.properties":'title=embedded-i18n_fr-wrong', "id1/embedded/manifest.json":'{"_version":"1.1.0","sap.app":{"_version":"1.1.0","id":"id1.embedded","type":"component","applicationVersion":{"version":"1.2.2"},"embeddedBy":"../","title":"{{title}}"}}', - "id1/i18n.properties":'title=app-i18n-wrong', - "id1/i18n/i18n.properties":'title=app-i18n', - "id1/i18n/i18n_de.properties":'title=app-i18n_de', - "id1/i18n/l10n.properties":'title=app-i18n-wrong', + "id1/i18n.properties":'title=app-i18n-wrong\nfame=Straße', + "id1/i18n/i18n.properties":'title=app-i18n\nfame=Straße', + "id1/i18n/i18n_de.properties":'title=app-i18n_de\nfame=Straße', + "id1/i18n/l10n.properties":'title=app-i18n-wrong\nfame=Straße', "id1/manifest.json":'{"_version":"1.1.0","sap.app":{"_version":"1.1.0","id":"id1","type":"application","applicationVersion":{"version":"1.2.2"},"embeds":["embedded"],"title":"{{title}}"}}', "sap/ui/core/Core.js":function(){ } diff --git a/test/fixtures/application.b/webapp/i18n.properties b/test/fixtures/application.b/webapp/i18n.properties index 88b84f602..d8d43be34 100644 --- a/test/fixtures/application.b/webapp/i18n.properties +++ b/test/fixtures/application.b/webapp/i18n.properties @@ -1 +1,2 @@ -title=app-i18n-wrong \ No newline at end of file +title=app-i18n-wrong +fame=Straße \ No newline at end of file diff --git a/test/fixtures/application.b/webapp/i18n/i18n.properties b/test/fixtures/application.b/webapp/i18n/i18n.properties index 575fb20d0..49f4756f9 100644 --- a/test/fixtures/application.b/webapp/i18n/i18n.properties +++ b/test/fixtures/application.b/webapp/i18n/i18n.properties @@ -1 +1,2 @@ -title=app-i18n \ No newline at end of file +title=app-i18n +fame=Straße \ No newline at end of file diff --git a/test/fixtures/application.b/webapp/i18n/i18n_de.properties b/test/fixtures/application.b/webapp/i18n/i18n_de.properties index f58858038..a81193360 100644 --- a/test/fixtures/application.b/webapp/i18n/i18n_de.properties +++ b/test/fixtures/application.b/webapp/i18n/i18n_de.properties @@ -1 +1,2 @@ -title=app-i18n_de \ No newline at end of file +title=app-i18n_de +fame=Straße \ No newline at end of file diff --git a/test/fixtures/application.b/webapp/i18n/l10n.properties b/test/fixtures/application.b/webapp/i18n/l10n.properties index 88b84f602..d8d43be34 100644 --- a/test/fixtures/application.b/webapp/i18n/l10n.properties +++ b/test/fixtures/application.b/webapp/i18n/l10n.properties @@ -1 +1,2 @@ -title=app-i18n-wrong \ No newline at end of file +title=app-i18n-wrong +fame=Straße \ No newline at end of file diff --git a/test/lib/types/application/ApplicationBuilder.js b/test/lib/types/application/ApplicationBuilder.js index 57bf109d8..f05d56552 100644 --- a/test/lib/types/application/ApplicationBuilder.js +++ b/test/lib/types/application/ApplicationBuilder.js @@ -57,6 +57,7 @@ test("Instantiation", (t) => { const appBuilder = new ApplicationBuilder({parentLogger, project}); t.truthy(appBuilder); t.deepEqual(appBuilder.taskExecutionOrder, [ + "escapeNonAsciiCharacters", "replaceCopyright", "replaceVersion", "generateFlexChangesBundle", @@ -79,6 +80,7 @@ test("Instantiation without component preload project configuration", (t) => { const appBuilder = new ApplicationBuilder({parentLogger, project}); t.truthy(appBuilder); t.deepEqual(appBuilder.taskExecutionOrder, [ + "escapeNonAsciiCharacters", "replaceCopyright", "replaceVersion", "generateFlexChangesBundle", @@ -102,6 +104,7 @@ test("Instantiation without project namespace", (t) => { const appBuilder = new ApplicationBuilder({parentLogger, project}); t.truthy(appBuilder); t.deepEqual(appBuilder.taskExecutionOrder, [ + "escapeNonAsciiCharacters", "replaceCopyright", "replaceVersion", "generateFlexChangesBundle", @@ -124,6 +127,7 @@ test("Instantiation with custom tasks", (t) => { const appBuilder = new ApplicationBuilder({parentLogger, project}); t.truthy(appBuilder); t.deepEqual(appBuilder.taskExecutionOrder, [ + "escapeNonAsciiCharacters", "replaceCopyright", "uglify--1", "replaceVersion", From 6504c6efced1d365e39b213ab6e5b94e40234e99 Mon Sep 17 00:00:00 2001 From: Tobias Sorn Date: Thu, 18 Jul 2019 10:10:53 +0200 Subject: [PATCH 05/27] [FEATURE] Properties File Escaping Change name of constant and adjust its jsdoc for clarification. --- lib/processors/stringEscaper.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/processors/stringEscaper.js b/lib/processors/stringEscaper.js index 51459d64b..506edbae9 100644 --- a/lib/processors/stringEscaper.js +++ b/lib/processors/stringEscaper.js @@ -2,9 +2,11 @@ const escapeUnicode = require("escape-unicode"); /** * @see https://ascii.cl/ + * ascii contains 128 characters. + * its char codes reach from 0 to 127. * @type {number} */ -const NUMBER_OF_ASCII_CHARACTERS = 127; +const CHAR_CODE_OF_LAST_ASCII_CHARACTER = 127; // use memoization for escapeUnicode function for performance const memoizeEscapeUnicodeMap = {}; @@ -31,8 +33,8 @@ const escapeNonAscii = function(string) { let modified = false; for (let i = 0; i < string.length; i++) { const char = string[i]; - // check for non ascii characters - if (string.charCodeAt(i) > NUMBER_OF_ASCII_CHARACTERS) { + // check for non ascii characters (characters which have a char code greater than the ascii character code range) + if (string.charCodeAt(i) > CHAR_CODE_OF_LAST_ASCII_CHARACTER) { result += memoizeEscapeUnicode(char); modified = true; } else { From 4bfc662e8a93140963163df3552f1e75d8318254 Mon Sep 17 00:00:00 2001 From: Thorsten Hochreuter Date: Fri, 19 Jul 2019 16:52:34 +0200 Subject: [PATCH 06/27] Refactor implementation - Always escape chars - Read files in source-encoding --- lib/lbt/bundle/AutoSplitter.js | 19 +++++++- lib/lbt/bundle/Builder.js | 16 +++++- lib/processors/bundlers/moduleBundler.js | 4 ++ lib/processors/stringEscaper.js | 54 +++++++++++++++------ lib/tasks/escapeNonAsciiCharacters.js | 41 +++++----------- lib/types/application/ApplicationBuilder.js | 2 +- package-lock.json | 10 ++-- test/lib/lbt/bundle/AutoSplitter.js | 33 ++++++++++--- test/lib/processors/stringEscaper.js | 34 +++++++------ test/lib/tasks/escapeNonAsciiCharacters.js | 21 ++++---- 10 files changed, 146 insertions(+), 88 deletions(-) diff --git a/lib/lbt/bundle/AutoSplitter.js b/lib/lbt/bundle/AutoSplitter.js index da99f14c6..bc8ec29c3 100644 --- a/lib/lbt/bundle/AutoSplitter.js +++ b/lib/lbt/bundle/AutoSplitter.js @@ -2,6 +2,9 @@ const uglify = require("uglify-es"); const {pd} = require("pretty-data"); + +const stringEscaper = require("../../processors/stringEscaper"); + const ModuleName = require("../utils/ModuleName"); const {SectionType} = require("./BundleDefinition"); const log = require("@ui5/logger").getLogger("lbt:bundle:AutoSplitter"); @@ -217,7 +220,6 @@ class AutoSplitter { // trace.debug("analyzed %s:%d%n", module, mw.getTargetLength()); return fileContent.length; } else if ( /\.properties$/.test(module) ) { - const fileContent = await resource.buffer(); /* NODE-TODO minimize *.properties Properties props = new Properties(); props.load(in); @@ -226,7 +228,20 @@ class AutoSplitter { props.store(out, ""); return out.toString().length(); */ - return fileContent.toString("latin1").length; + + // input encoding is read from project configuration, default is ISO-8859-1 + const encoding = resource.getProject() && resource.getProject().resources.propertiesFileEncoding; + + await stringEscaper({ + resources: [resource.resource], + options: { + encoding: encoding || "ISO-8859-1" + } + }); + + const fileContent = await resource.buffer(); + + return fileContent.toString().length; } else if ( this.optimizeXMLViews && /\.view.xml$/.test(module) ) { // needs to be activated when it gets activated in JSMergedModuleBuilderExt let fileContent = await resource.buffer(); diff --git a/lib/lbt/bundle/Builder.js b/lib/lbt/bundle/Builder.js index 0dd366b6e..309a9ca6c 100644 --- a/lib/lbt/bundle/Builder.js +++ b/lib/lbt/bundle/Builder.js @@ -9,6 +9,8 @@ const escodegen = require("escodegen"); const {Syntax} = esprima; // const MOZ_SourceMap = require("source-map"); +const stringEscaper = require("../../processors/stringEscaper"); + const {isMethodCall} = require("../utils/ASTUtils"); const ModuleName = require("../utils/ModuleName"); const UI5ClientConstants = require("../UI5ClientConstants"); @@ -442,9 +444,19 @@ class BundleBuilder { } outW.write( makeStringLiteral( fileContent ) ); } else if ( /\.properties$/.test(module) ) { - // same as for other text files, but input encoding is ISO_8859_1 + // same as for other text files, input encoding is read from project configuration, default is ISO-8859-1 + const encoding = resource.getProject() && resource.getProject().resources.propertiesFileEncoding; + + await stringEscaper({ + resources: [resource.resource], + options: { + encoding: encoding || "ISO-8859-1" + } + }); + const fileContent = await resource.buffer(); - outW.write( makeStringLiteral( fileContent.toString("latin1") ) ); + + outW.write( makeStringLiteral( fileContent.toString() ) ); } else { log.error("don't know how to embed module " + module); // TODO throw? } diff --git a/lib/processors/bundlers/moduleBundler.js b/lib/processors/bundlers/moduleBundler.js index aef847673..a6bc5da4e 100644 --- a/lib/processors/bundlers/moduleBundler.js +++ b/lib/processors/bundlers/moduleBundler.js @@ -16,6 +16,10 @@ class LocatorResource extends Resource { buffer() { return this.resource.getBuffer(); } + + getProject() { + return this.resource._project; + } } class LocatorResourcePool extends ResourcePool { diff --git a/lib/processors/stringEscaper.js b/lib/processors/stringEscaper.js index 506edbae9..18b56fd75 100644 --- a/lib/processors/stringEscaper.js +++ b/lib/processors/stringEscaper.js @@ -1,7 +1,7 @@ const escapeUnicode = require("escape-unicode"); /** - * @see https://ascii.cl/ + * @see https://en.wikipedia.org/wiki/ASCII * ascii contains 128 characters. * its char codes reach from 0 to 127. * @type {number} @@ -19,21 +19,23 @@ const memoizeEscapeUnicode = function(sChar) { }; /** - * Escapes non ASCII characters with unicode characters. + * Escapes non ASCII characters with unicode escape sequences. * - * @see https://ascii.cl/ + * @see https://en.wikipedia.org/wiki/ASCII * @see https://tools.ietf.org/html/rfc5137#section-6.1 * * * @param {string} string input string with non ascii characters, e.g. L♥VE - * @returns {{string: (string), modified: boolean}} output string with all non ascii characters being escaped by unicode sequence, e.g. L\u2665VE + * @returns {{string: (string), modified: boolean}} output string with all non ascii + * characters being escaped by unicode sequence, e.g. L\u2665VE */ const escapeNonAscii = function(string) { let result = ""; let modified = false; for (let i = 0; i < string.length; i++) { const char = string[i]; - // check for non ascii characters (characters which have a char code greater than the ascii character code range) + // check for non ascii characters (characters which have a char code + // greater than the ascii character code range) if (string.charCodeAt(i) > CHAR_CODE_OF_LAST_ASCII_CHARACTER) { result += memoizeEscapeUnicode(char); modified = true; @@ -47,15 +49,35 @@ const escapeNonAscii = function(string) { }; }; -module.exports = function({resources}) { - return Promise.all(resources.map((resource) => { - return resource.getString().then((resourceString) => { - const escaped = escapeNonAscii(resourceString); - // only modify the resource's string if it was changed - if (escaped.modified) { - resource.setString(escaped.string); - } - return resource; - }); - })); +/** + * Escapes non ASCII characters with unicode escape sequences. + * + * @public + * @alias module:@ui5/builder.processors.stringEscaper + * @param {Object} parameters Parameters + * @param {module:@ui5/fs.Resource[]} parameters.resources List of resources to be processed + * @param {Object} [parameters.options] Options + * @param {string} [parameters.options.encoding="utf8"] resource file encoding based on supported + * {@link https://nodejs.org/api/buffer.html#buffer_buffers_and_character_encodings Node.js character encodings}; + * ISO-8859-1 is mapped to latin1. + * @returns {Promise} Promise resolving with the processed resources + */ +module.exports = async function({resources, options={}}) { + let encoding = options.encoding || "utf8"; + + if (encoding.toUpperCase() == "ISO-8859-1") { + encoding = "latin1"; + } + + async function processResource(resource) { + const resourceString = (await resource.getBuffer()).toString(encoding); + const escaped = escapeNonAscii(resourceString); + // only modify the resource's string if it was changed + if (escaped.modified) { + resource.setString(escaped.string); + } + return resource; + } + + return Promise.all(resources.map(processResource)); }; diff --git a/lib/tasks/escapeNonAsciiCharacters.js b/lib/tasks/escapeNonAsciiCharacters.js index def0819e0..392f91240 100644 --- a/lib/tasks/escapeNonAsciiCharacters.js +++ b/lib/tasks/escapeNonAsciiCharacters.js @@ -1,15 +1,5 @@ const stringEscaper = require("../processors/stringEscaper"); -/** - * Encodings - * - * @type {{UTF_8: string, ISO_8859_1: string}} - */ -const ENCODING_TYPES = { - UTF_8: "UTF-8", - ISO_8859_1: "ISO-8859-1" -}; - /** * Task to escape non ascii characters in properties files resources. * @@ -19,25 +9,18 @@ const ENCODING_TYPES = { * @param {module:@ui5/fs.DuplexCollection} parameters.workspace DuplexCollection to read and write files * @param {Object} parameters.options Options * @param {string} parameters.options.pattern Pattern to locate the files to be processed - * @param {string} [parameters.options.sourceEncoding] source file encoding: "UTF-8" or "ISO-8859-1". Defaults to "ISO-8859-1" + * @param {string} [parameters.options.sourceEncoding] source file encoding * @returns {Promise} Promise resolving with undefined once data has been written */ -module.exports = function({workspace, options}) { - if (options.sourceEncoding && !Object.values(ENCODING_TYPES).includes(options.sourceEncoding)) { - return Promise.reject(new Error(`Invalid encoding specified: '${options.sourceEncoding}'. Must be one of ${Object.values(ENCODING_TYPES)}`)); - } - if (options.sourceEncoding && options.sourceEncoding === ENCODING_TYPES.UTF_8) { - return Promise.resolve(); - } - return workspace.byGlob(options.pattern) - .then((allResources) => { - return stringEscaper({ - resources: allResources - }); - }) - .then((processedResources) => { - return Promise.all(processedResources.map((resource) => { - return workspace.write(resource); - })); - }); +module.exports = async function({workspace, options}) { + const allResources = await workspace.byGlob(options.pattern); + + const processedResources = await stringEscaper({ + resources: allResources, + options: { + encoding: options.encoding + } + }); + + await Promise.all(processedResources.map((resource) => workspace.write(resource))); }; diff --git a/lib/types/application/ApplicationBuilder.js b/lib/types/application/ApplicationBuilder.js index 0ffd517c4..ad921f4aa 100644 --- a/lib/types/application/ApplicationBuilder.js +++ b/lib/types/application/ApplicationBuilder.js @@ -31,7 +31,7 @@ class ApplicationBuilder extends AbstractBuilder { return tasks.escapeNonAsciiCharacters({ workspace: resourceCollections.workspace, options: { - sourceEncoding: project.resources && project.resources.propertiesFileEncoding, + encoding: project.resources && project.resources.propertiesFileEncoding, pattern: "/**/*.properties" } }); diff --git a/package-lock.json b/package-lock.json index 1b7bf02da..4ec9425eb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4407,7 +4407,7 @@ }, "find-up": { "version": "3.0.0", - "resolved": false, + "resolved": "https://registry.npmjs.org/find-up/-/find-up-3.0.0.tgz", "integrity": "sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg==", "dev": true, "requires": { @@ -4416,7 +4416,7 @@ }, "locate-path": { "version": "3.0.0", - "resolved": false, + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-3.0.0.tgz", "integrity": "sha512-7AO748wWnIhNqAuaty2ZWHkQHRSNfPVIsPIfwEOWO22AmaoVrWavlOcMR5nzTLNYvp36X220/maaRsrec1G65A==", "dev": true, "requires": { @@ -4436,7 +4436,7 @@ }, "p-locate": { "version": "3.0.0", - "resolved": false, + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-3.0.0.tgz", "integrity": "sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ==", "dev": true, "requires": { @@ -4445,13 +4445,13 @@ }, "path-exists": { "version": "3.0.0", - "resolved": false, + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz", "integrity": "sha1-zg6+ql94yxiSXqfYENe1mwEP1RU=", "dev": true }, "resolve-from": { "version": "4.0.0", - "resolved": false, + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", "dev": true }, diff --git a/test/lib/lbt/bundle/AutoSplitter.js b/test/lib/lbt/bundle/AutoSplitter.js index b17747613..62db2616b 100644 --- a/test/lib/lbt/bundle/AutoSplitter.js +++ b/test/lib/lbt/bundle/AutoSplitter.js @@ -16,7 +16,14 @@ function createMockPool(dependencies) { if (name === "c.js") { info.compressedSize = 512; } - return {info, buffer: async () => name.padStart(2048, "*")}; + return { + info, + buffer: async () => Buffer.from(name.padStart(2048, "*")), + getProject: () => undefined, + resource: { + getBuffer: () => Buffer.from(name.padStart(2048, "*")) + } + }; }, resources: [{ name: "a.js" @@ -152,7 +159,8 @@ test("_calcMinSize: js resource", async (t) => { size: 333, compressedSize: 333 }, - buffer: async () => "var test = 5;" + buffer: async () => "var test = 5;", + getProject: () => undefined }; } }; @@ -170,7 +178,8 @@ test.serial("_calcMinSize: uglify js resource", async (t) => { size: 333, compressedSize: 333 }, - buffer: async () => "var test = 5;" + buffer: async () => "var test = 5;", + getProject: () => undefined }; } }; @@ -184,7 +193,11 @@ test("_calcMinSize: properties resource", async (t) => { const pool = { findResourceWithInfo: function() { return { - buffer: async () => "1234" + buffer: async () => Buffer.from("1234"), + resource: { + getBuffer: () => Buffer.from("1234") + }, + getProject: () => undefined }; } }; @@ -196,7 +209,8 @@ test("_calcMinSize: xml view resource", async (t) => { const pool = { findResourceWithInfo: function() { return { - buffer: async () => "12345" + buffer: async () => "12345", + getProject: () => undefined }; } }; @@ -209,7 +223,8 @@ test("_calcMinSize: xml view resource without optimizeXMLViews", async (t) => { const pool = { findResourceWithInfo: function() { return { - buffer: async () => "123456" + buffer: async () => "123456", + getProject: () => undefined }; } }; @@ -222,7 +237,8 @@ test.serial("_calcMinSize: optimize xml view resource", async (t) => { const pool = { findResourceWithInfo: function() { return { - buffer: async () => "xxx" + buffer: async () => "xxx", + getProject: () => undefined }; } }; @@ -238,7 +254,8 @@ test.serial("_calcMinSize: optimize xml view resource and pre tag", async (t) => const pool = { findResourceWithInfo: function() { return { - buffer: async () => "
asd
" + buffer: async () => "
asd
", + getProject: () => undefined }; } }; diff --git a/test/lib/processors/stringEscaper.js b/test/lib/processors/stringEscaper.js index 4208e03ca..ae42afa85 100644 --- a/test/lib/processors/stringEscaper.js +++ b/test/lib/processors/stringEscaper.js @@ -11,13 +11,17 @@ const stringEscaper = require("../../../lib/processors/stringEscaper"); const escape = async function(input) { let result = undefined; const resource = { - getString: () => Promise.resolve(input), setString: (actual) => { result = actual; + }, + getString: async () => { + return result; + }, + getBuffer: async () => { + return Buffer.from(input, "utf8"); } }; - await stringEscaper({resources: [resource]}); - return result; + return stringEscaper({resources: [resource]}); }; test("Replace symbol characters", async (t) => { @@ -25,8 +29,8 @@ test("Replace symbol characters", async (t) => { const input = `L♥VE is everywhere`; const expected = `L\\u2665VE is everywhere`; - const output = await escape(input); - t.deepEqual(output, expected, "Correct file content should be set"); + const [resource] = await escape(input); + t.deepEqual(await resource.getString(), expected, "Correct file content should be set"); }); test("Replace chinese characters", async (t) => { @@ -34,8 +38,8 @@ test("Replace chinese characters", async (t) => { const input = `These are 人物 characters`; const expected = "These are \\u4eba\\u7269 characters"; - const output = await escape(input); - t.deepEqual(output, expected, "Correct file content should be set"); + const [resource] = await escape(input); + t.deepEqual(await resource.getString(), expected, "Correct file content should be set"); }); test("Replace umlaut characters", async (t) => { @@ -43,8 +47,8 @@ test("Replace umlaut characters", async (t) => { const input = `Achso Ähem`; const expected = "Achso \\u00c4hem"; - const output = await escape(input); - t.deepEqual(output, expected, "Correct file content should be set"); + const [resource] = await escape(input); + t.deepEqual(await resource.getString(), expected, "Correct file content should be set"); }); test("Replace constructed characters", async (t) => { @@ -52,8 +56,8 @@ test("Replace constructed characters", async (t) => { const input = `Oh ẛ̣ that's ẛ̣ yes`; const expected = "Oh \\u1e9b\\u0323 that's \\u1e9b\\u0323 yes"; - const output = await escape(input); - t.deepEqual(output, expected, "Correct file content should be set"); + const [resource] = await escape(input); + t.deepEqual(await resource.getString(), expected, "Correct file content should be set"); }); @@ -62,8 +66,8 @@ test("Replace multiple times same character", async (t) => { const input = `♥H L♥VE AND HARM♥NY ♥MG`; const expected = "\\u2665H L\\u2665VE AND HARM\\u2665NY \\u2665MG"; - const output = await escape(input); - t.deepEqual(output, expected, "Correct file content should be set"); + const [resource] = await escape(input); + t.deepEqual(await resource.getString(), expected, "Correct file content should be set"); }); test("No Replace of characters", async (t) => { @@ -71,6 +75,6 @@ test("No Replace of characters", async (t) => { const input = `ONE LOVE`; const expected = undefined; - const output = await escape(input); - t.deepEqual(output, expected, "Correct file content should not be modified"); + const [resource] = await escape(input); + t.deepEqual(await resource.getString(), expected, "Correct file content should be set"); }); diff --git a/test/lib/tasks/escapeNonAsciiCharacters.js b/test/lib/tasks/escapeNonAsciiCharacters.js index 3e108fa42..b787621ed 100644 --- a/test/lib/tasks/escapeNonAsciiCharacters.js +++ b/test/lib/tasks/escapeNonAsciiCharacters.js @@ -6,7 +6,7 @@ const ui5Fs = require("@ui5/fs"); const resourceFactory = ui5Fs.resourceFactory; const DuplexCollection = ui5Fs.DuplexCollection; -test("integration: escape non ascii characters", (t) => { +test("integration: escape non ascii characters (utf8, default)", (t) => { const reader = resourceFactory.createAdapter({ virBasePath: "/" }); @@ -18,14 +18,14 @@ test("integration: escape non ascii characters", (t) => { const content = `welcome=Willkommen {0}. Bitte geben Sie einen neuen Kontakt ein: lastname=Nachname: firstname=Vorname: -street=Straße: +street=Straße:♥ zip=PLZ: city=Ort:`; const expected = `welcome=Willkommen {0}. Bitte geben Sie einen neuen Kontakt ein: lastname=Nachname: firstname=Vorname: -street=Stra\\u00dfe: +street=Stra\\u00dfe:\\u2665 zip=PLZ: city=Ort:`; @@ -54,7 +54,7 @@ city=Ort:`; }); }); -test("integration: escape non ascii characters source encoding being UTF-8", (t) => { +test("integration: escape non ascii characters source encoding being (ISO-8859-1)", (t) => { const reader = resourceFactory.createAdapter({ virBasePath: "/" }); @@ -63,30 +63,31 @@ test("integration: escape non ascii characters source encoding being UTF-8", (t) }); const workspace = new DuplexCollection({reader, writer}); - const content = `welcome=Willkommen {0}. Bitte geben Sie einen neuen Kontakt ein: + // create buffer in ISO encoding + const content = Buffer.from(`welcome=Willkommen {0}. Bitte geben Sie einen neuen Kontakt ein: lastname=Nachname: firstname=Vorname: street=Straße: zip=PLZ: -city=Ort:`; +city=Ort:`, "latin1"); const expected = `welcome=Willkommen {0}. Bitte geben Sie einen neuen Kontakt ein: lastname=Nachname: firstname=Vorname: -street=Straße: +street=Stra\\u00dfe: zip=PLZ: city=Ort:`; const resource = resourceFactory.createResource({ path: "/i18n.properties", - string: content + buffer: content }); return workspace.write(resource).then(() => { return tasks.escapeNonAsciiCharacters({ workspace, options: { - sourceEncoding: "UTF-8", + encoding: "ISO-8859-1", pattern: "/**/*.properties" } }).then(() => { @@ -103,7 +104,7 @@ city=Ort:`; }); }); -test("integration: escape non ascii characters source encoding being UTF-16", (t) => { +test.skip("integration: escape non ascii characters source encoding being UTF-16", (t) => { return tasks.escapeNonAsciiCharacters({ workspace: undefined, options: { From e0331afce6fa7a69878b6e2ca14b06db89968719 Mon Sep 17 00:00:00 2001 From: Tobias Sorn Date: Sat, 20 Jul 2019 11:45:51 +0200 Subject: [PATCH 07/27] [FEATURE] Properties File Escaping Fix expected outcome in test --- .../application.b/standalone/resources/sap-ui-custom.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/expected/build/application.b/standalone/resources/sap-ui-custom.js b/test/expected/build/application.b/standalone/resources/sap-ui-custom.js index 906641067..2f1bb3225 100644 --- a/test/expected/build/application.b/standalone/resources/sap-ui-custom.js +++ b/test/expected/build/application.b/standalone/resources/sap-ui-custom.js @@ -5,10 +5,10 @@ jQuery.sap.registerPreloadedModules({ "id1/embedded/i18n/i18n_de.properties":'title=embedded-i18n_de', "id1/embedded/i18n_fr.properties":'title=embedded-i18n_fr-wrong', "id1/embedded/manifest.json":'{"_version":"1.1.0","sap.app":{"_version":"1.1.0","id":"id1.embedded","type":"component","applicationVersion":{"version":"1.2.2"},"embeddedBy":"../","title":"{{title}}"}}', - "id1/i18n.properties":'title=app-i18n-wrong\nfame=Straße', - "id1/i18n/i18n.properties":'title=app-i18n\nfame=Straße', - "id1/i18n/i18n_de.properties":'title=app-i18n_de\nfame=Straße', - "id1/i18n/l10n.properties":'title=app-i18n-wrong\nfame=Straße', + "id1/i18n.properties":'title=app-i18n-wrong\nfame=Stra\\u00dfe', + "id1/i18n/i18n.properties":'title=app-i18n\nfame=Stra\\u00dfe', + "id1/i18n/i18n_de.properties":'title=app-i18n_de\nfame=Stra\\u00dfe', + "id1/i18n/l10n.properties":'title=app-i18n-wrong\nfame=Stra\\u00dfe', "id1/manifest.json":'{"_version":"1.1.0","sap.app":{"_version":"1.1.0","id":"id1","type":"application","applicationVersion":{"version":"1.2.2"},"embeds":["embedded"],"title":"{{title}}"}}', "sap/ui/core/Core.js":function(){ } From ddb37510293e8db5649f6e69e8e626dc77fa5c6d Mon Sep 17 00:00:00 2001 From: Tobias Sorn Date: Mon, 22 Jul 2019 08:09:34 +0200 Subject: [PATCH 08/27] [FEATURE] Properties File Escaping Adjust encoding parameter --- lib/tasks/escapeNonAsciiCharacters.js | 4 ++-- lib/types/library/LibraryBuilder.js | 2 +- test/lib/tasks/escapeNonAsciiCharacters.js | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/tasks/escapeNonAsciiCharacters.js b/lib/tasks/escapeNonAsciiCharacters.js index 392f91240..2696b6377 100644 --- a/lib/tasks/escapeNonAsciiCharacters.js +++ b/lib/tasks/escapeNonAsciiCharacters.js @@ -8,8 +8,8 @@ const stringEscaper = require("../processors/stringEscaper"); * @param {Object} parameters Parameters * @param {module:@ui5/fs.DuplexCollection} parameters.workspace DuplexCollection to read and write files * @param {Object} parameters.options Options - * @param {string} parameters.options.pattern Pattern to locate the files to be processed - * @param {string} [parameters.options.sourceEncoding] source file encoding + * @param {string} parameters.options.pattern Glob pattern to locate the files to be processed + * @param {string} [parameters.options.encoding] source file encoding, e.g. "utf8", "latin1", ... (default is "utf8") * @returns {Promise} Promise resolving with undefined once data has been written */ module.exports = async function({workspace, options}) { diff --git a/lib/types/library/LibraryBuilder.js b/lib/types/library/LibraryBuilder.js index a8f6c616e..07e6c1e75 100644 --- a/lib/types/library/LibraryBuilder.js +++ b/lib/types/library/LibraryBuilder.js @@ -30,7 +30,7 @@ class LibraryBuilder extends AbstractBuilder { return tasks.escapeNonAsciiCharacters({ workspace: resourceCollections.workspace, options: { - sourceEncoding: project.resources && project.resources.propertiesFileEncoding, + encoding: project.resources && project.resources.propertiesFileEncoding, pattern: "/**/*.properties" } }); diff --git a/test/lib/tasks/escapeNonAsciiCharacters.js b/test/lib/tasks/escapeNonAsciiCharacters.js index b787621ed..61d8c14d1 100644 --- a/test/lib/tasks/escapeNonAsciiCharacters.js +++ b/test/lib/tasks/escapeNonAsciiCharacters.js @@ -108,7 +108,7 @@ test.skip("integration: escape non ascii characters source encoding being UTF-16 return tasks.escapeNonAsciiCharacters({ workspace: undefined, options: { - sourceEncoding: "UTF-16", + encoding: "utf16le", pattern: "/**/*.properties" } }).catch((error) => { From 006cac51df02eacd7ae83307f82a170f3d17d97b Mon Sep 17 00:00:00 2001 From: Tobias Sorn Date: Mon, 22 Jul 2019 08:33:51 +0200 Subject: [PATCH 09/27] [FEATURE] Properties File Escaping Expose stringEscaper processor and escapeNonAsciiCharacters task via index.js to be able to consume it from server module. --- index.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/index.js b/index.js index 73f11d149..ae7d488b7 100644 --- a/index.js +++ b/index.js @@ -19,6 +19,7 @@ module.exports = { bootstrapHtmlTransformer: require("./lib/processors/bootstrapHtmlTransformer"), debugFileCreator: require("./lib/processors/debugFileCreator"), resourceCopier: require("./lib/processors/resourceCopier"), + stringEscaper: require("./lib/processors/stringEscaper"), stringReplacer: require("./lib/processors/stringReplacer"), themeBuilder: require("./lib/processors/themeBuilder"), uglifier: require("./lib/processors/uglifier"), @@ -43,6 +44,7 @@ module.exports = { generateApiIndex: require("./lib/tasks/jsdoc/generateApiIndex"), generateJsdoc: require("./lib/tasks/jsdoc/generateJsdoc"), generateVersionInfo: require("./lib/tasks/generateVersionInfo"), + escapeNonAsciiCharacters: require("./lib/tasks/escapeNonAsciiCharacters"), replaceCopyright: require("./lib/tasks/replaceCopyright"), replaceVersion: require("./lib/tasks/replaceVersion"), transformBootstrapHtml: require("./lib/tasks/transformBootstrapHtml"), From 336e5d179deb34c7efdc460f6725e325fddc8cf1 Mon Sep 17 00:00:00 2001 From: Tobias Sorn Date: Mon, 22 Jul 2019 09:45:56 +0200 Subject: [PATCH 10/27] [FEATURE] Properties File Escaping Add default to formatter. Adjust tests. --- lib/processors/stringEscaper.js | 2 +- lib/types/application/ApplicationFormatter.js | 5 +++++ lib/types/library/LibraryFormatter.js | 5 +++++ test/lib/processors/stringEscaper.js | 13 +++++++++++-- test/lib/types/library/LibraryFormatter.js | 3 ++- 5 files changed, 24 insertions(+), 4 deletions(-) diff --git a/lib/processors/stringEscaper.js b/lib/processors/stringEscaper.js index 18b56fd75..ab895bb4f 100644 --- a/lib/processors/stringEscaper.js +++ b/lib/processors/stringEscaper.js @@ -65,7 +65,7 @@ const escapeNonAscii = function(string) { module.exports = async function({resources, options={}}) { let encoding = options.encoding || "utf8"; - if (encoding.toUpperCase() == "ISO-8859-1") { + if (encoding.toUpperCase() === "ISO-8859-1") { encoding = "latin1"; } diff --git a/lib/types/application/ApplicationFormatter.js b/lib/types/application/ApplicationFormatter.js index b099bcffb..d42d0c9c3 100644 --- a/lib/types/application/ApplicationFormatter.js +++ b/lib/types/application/ApplicationFormatter.js @@ -20,6 +20,11 @@ class ApplicationFormatter extends AbstractUi5Formatter { "/": project.resources.configuration.paths.webapp }; + // set default encoding + if (!project.resources.propertiesFileEncoding) { + project.resources.propertiesFileEncoding = "ISO-8859-1"; + } + try { project.metadata.namespace = await this.getNamespace(); } catch (err) { diff --git a/lib/types/library/LibraryFormatter.js b/lib/types/library/LibraryFormatter.js index 652fb8323..592c6ea3d 100644 --- a/lib/types/library/LibraryFormatter.js +++ b/lib/types/library/LibraryFormatter.js @@ -23,6 +23,11 @@ class LibraryFormatter extends AbstractUi5Formatter { "/resources/": project.resources.configuration.paths.src }; + // set default encoding + if (!project.resources.propertiesFileEncoding) { + project.resources.propertiesFileEncoding = "ISO-8859-1"; + } + if (project.resources.configuration.paths.test) { // Directory 'test' is somewhat optional for libraries project.resources.pathMappings["/test-resources/"] = project.resources.configuration.paths.test; diff --git a/test/lib/processors/stringEscaper.js b/test/lib/processors/stringEscaper.js index ae42afa85..f445b204e 100644 --- a/test/lib/processors/stringEscaper.js +++ b/test/lib/processors/stringEscaper.js @@ -6,9 +6,10 @@ const stringEscaper = require("../../../lib/processors/stringEscaper"); * Executes string escaping. Returns undefined if nothing was escaped. * * @param {string} input string + * @param {Object} [options] * @returns {Promise} escaped string if non-ascii characters present, undefined otherwise */ -const escape = async function(input) { +const escape = async function(input, options) { let result = undefined; const resource = { setString: (actual) => { @@ -21,7 +22,7 @@ const escape = async function(input) { return Buffer.from(input, "utf8"); } }; - return stringEscaper({resources: [resource]}); + return stringEscaper({resources: [resource], options}); }; test("Replace symbol characters", async (t) => { @@ -78,3 +79,11 @@ test("No Replace of characters", async (t) => { const [resource] = await escape(input); t.deepEqual(await resource.getString(), expected, "Correct file content should be set"); }); + +test("Invalid encoding", async (t) => { + t.plan(2); + + const input = `ONE LOVE`; + const error = await t.throwsAsync(escape(input, {encoding: "asd"})); + t.is(error.message, "Unknown encoding: asd"); +}); diff --git a/test/lib/types/library/LibraryFormatter.js b/test/lib/types/library/LibraryFormatter.js index 70ec04309..83d1dd20f 100644 --- a/test/lib/types/library/LibraryFormatter.js +++ b/test/lib/types/library/LibraryFormatter.js @@ -151,7 +151,8 @@ test("format: formats correctly", async (t) => { pathMappings: { "/resources/": "src", "/test-resources/": "test" - } + }, + propertiesFileEncoding: "ISO-8859-1" } }, "Project got formatted correctly"); }); From d05d833d654aeaccaa6189de139ba95fab2f065c Mon Sep 17 00:00:00 2001 From: Tobias Sorn Date: Mon, 22 Jul 2019 15:53:39 +0200 Subject: [PATCH 11/27] [FEATURE] Properties File Escaping Rename stringEscaper to nonAsciiEscaper since it only escapes non ascii characters. Made encoding a required parameter for the task. Added static method to nonAsciiEscaper processor since node.js encoding does not support "ISO-8859-1" only knows it as "latin1" though. --- index.js | 2 +- lib/lbt/bundle/AutoSplitter.js | 9 ++--- lib/lbt/bundle/Builder.js | 6 ++-- .../{stringEscaper.js => nonAsciiEscaper.js} | 35 ++++++++++++++----- lib/tasks/escapeNonAsciiCharacters.js | 12 ++++--- test/lib/builder/builder.js | 21 +++++------ .../{stringEscaper.js => nonAsciiEscaper.js} | 16 +++++++-- test/lib/tasks/escapeNonAsciiCharacters.js | 1 + test/lib/tasks/generateCachebusterInfo.js | 4 +-- 9 files changed, 71 insertions(+), 35 deletions(-) rename lib/processors/{stringEscaper.js => nonAsciiEscaper.js} (72%) rename test/lib/processors/{stringEscaper.js => nonAsciiEscaper.js} (83%) diff --git a/index.js b/index.js index ae7d488b7..77f967d03 100644 --- a/index.js +++ b/index.js @@ -19,7 +19,7 @@ module.exports = { bootstrapHtmlTransformer: require("./lib/processors/bootstrapHtmlTransformer"), debugFileCreator: require("./lib/processors/debugFileCreator"), resourceCopier: require("./lib/processors/resourceCopier"), - stringEscaper: require("./lib/processors/stringEscaper"), + nonAsciiEscaper: require("./lib/processors/nonAsciiEscaper"), stringReplacer: require("./lib/processors/stringReplacer"), themeBuilder: require("./lib/processors/themeBuilder"), uglifier: require("./lib/processors/uglifier"), diff --git a/lib/lbt/bundle/AutoSplitter.js b/lib/lbt/bundle/AutoSplitter.js index bc8ec29c3..dc2fa831a 100644 --- a/lib/lbt/bundle/AutoSplitter.js +++ b/lib/lbt/bundle/AutoSplitter.js @@ -3,7 +3,7 @@ const uglify = require("uglify-es"); const {pd} = require("pretty-data"); -const stringEscaper = require("../../processors/stringEscaper"); +const nonAsciiEscaper = require("../../processors/nonAsciiEscaper"); const ModuleName = require("../utils/ModuleName"); const {SectionType} = require("./BundleDefinition"); @@ -229,13 +229,14 @@ class AutoSplitter { return out.toString().length(); */ - // input encoding is read from project configuration, default is ISO-8859-1 + // input encoding is read from project configuration, default is "ISO-8859-1" + // since AutoSplitter is also used when serving non-project resources (e.g. dependencies) *.properties files should be escaped const encoding = resource.getProject() && resource.getProject().resources.propertiesFileEncoding; - await stringEscaper({ + await nonAsciiEscaper({ resources: [resource.resource], options: { - encoding: encoding || "ISO-8859-1" + encoding: nonAsciiEscaper.getEncodingFromNiceName(encoding || "ISO-8859-1") } }); diff --git a/lib/lbt/bundle/Builder.js b/lib/lbt/bundle/Builder.js index 309a9ca6c..93894debd 100644 --- a/lib/lbt/bundle/Builder.js +++ b/lib/lbt/bundle/Builder.js @@ -9,7 +9,7 @@ const escodegen = require("escodegen"); const {Syntax} = esprima; // const MOZ_SourceMap = require("source-map"); -const stringEscaper = require("../../processors/stringEscaper"); +const nonAsciiEscaper = require("../../processors/nonAsciiEscaper"); const {isMethodCall} = require("../utils/ASTUtils"); const ModuleName = require("../utils/ModuleName"); @@ -447,10 +447,10 @@ class BundleBuilder { // same as for other text files, input encoding is read from project configuration, default is ISO-8859-1 const encoding = resource.getProject() && resource.getProject().resources.propertiesFileEncoding; - await stringEscaper({ + await nonAsciiEscaper({ resources: [resource.resource], options: { - encoding: encoding || "ISO-8859-1" + encoding: nonAsciiEscaper.getEncodingFromNiceName(encoding || "ISO-8859-1") } }); diff --git a/lib/processors/stringEscaper.js b/lib/processors/nonAsciiEscaper.js similarity index 72% rename from lib/processors/stringEscaper.js rename to lib/processors/nonAsciiEscaper.js index ab895bb4f..c18d2c41e 100644 --- a/lib/processors/stringEscaper.js +++ b/lib/processors/nonAsciiEscaper.js @@ -52,22 +52,22 @@ const escapeNonAscii = function(string) { /** * Escapes non ASCII characters with unicode escape sequences. * + * @example + * const encoding = nonAsciiEscaper.getEncodingFromNiceName("ISO-8859-1"); + * nonAsciiEscaper({resources, options: {encoding}}); + * + * * @public - * @alias module:@ui5/builder.processors.stringEscaper + * @alias module:@ui5/builder.processors.nonAsciiEscaper * @param {Object} parameters Parameters * @param {module:@ui5/fs.Resource[]} parameters.resources List of resources to be processed * @param {Object} [parameters.options] Options - * @param {string} [parameters.options.encoding="utf8"] resource file encoding based on supported + * @param {string} [parameters.options.encoding="utf8"] resource file encoding (node.js based encodings). Use #getEncodingFromNiceName to get the encoding string * {@link https://nodejs.org/api/buffer.html#buffer_buffers_and_character_encodings Node.js character encodings}; - * ISO-8859-1 is mapped to latin1. * @returns {Promise} Promise resolving with the processed resources */ -module.exports = async function({resources, options={}}) { - let encoding = options.encoding || "utf8"; - - if (encoding.toUpperCase() === "ISO-8859-1") { - encoding = "latin1"; - } +module.exports = async function nonAsciiEscaper({resources, options={}}) { + const encoding = options.encoding || "utf8"; async function processResource(resource) { const resourceString = (await resource.getBuffer()).toString(encoding); @@ -81,3 +81,20 @@ module.exports = async function({resources, options={}}) { return Promise.all(resources.map(processResource)); }; + +const encodingMap = { + "UTF-8": "utf8", + "ISO-8859-1": "latin1", + +}; + +/** + * @param {string} encoding encoding labels: "UTF-8" and "ISO-8859-1" + * @returns {string} node.js character encoding string, e.g. utf8 and latin1 + */ +module.exports.getEncodingFromNiceName = function(encoding) { + if (!encodingMap[encoding]) { + throw new Error(`Encoding "${encoding}" is not supported. Only ${Object.keys(encodingMap).join(",")} are allowed values` ); + } + return encodingMap[encoding]; +}; diff --git a/lib/tasks/escapeNonAsciiCharacters.js b/lib/tasks/escapeNonAsciiCharacters.js index 2696b6377..e62e2ea5f 100644 --- a/lib/tasks/escapeNonAsciiCharacters.js +++ b/lib/tasks/escapeNonAsciiCharacters.js @@ -1,4 +1,4 @@ -const stringEscaper = require("../processors/stringEscaper"); +const nonAsciiEscaper = require("../processors/nonAsciiEscaper"); /** * Task to escape non ascii characters in properties files resources. @@ -9,16 +9,20 @@ const stringEscaper = require("../processors/stringEscaper"); * @param {module:@ui5/fs.DuplexCollection} parameters.workspace DuplexCollection to read and write files * @param {Object} parameters.options Options * @param {string} parameters.options.pattern Glob pattern to locate the files to be processed - * @param {string} [parameters.options.encoding] source file encoding, e.g. "utf8", "latin1", ... (default is "utf8") + * @param {string} parameters.options.encoding source file encoding either "UTF-8" or "ISO-8859-1" * @returns {Promise} Promise resolving with undefined once data has been written */ module.exports = async function({workspace, options}) { + if (!options.encoding) { + throw new Error("No Encoding specified"); + } + const allResources = await workspace.byGlob(options.pattern); - const processedResources = await stringEscaper({ + const processedResources = await nonAsciiEscaper({ resources: allResources, options: { - encoding: options.encoding + encoding: nonAsciiEscaper.getEncodingFromNiceName(options.encoding) } }); diff --git a/test/lib/builder/builder.js b/test/lib/builder/builder.js index cc4af0b93..9ae0c5a1e 100644 --- a/test/lib/builder/builder.js +++ b/test/lib/builder/builder.js @@ -84,7 +84,7 @@ test("Build application.a", (t) => { return builder.build({ tree: applicationATree, destPath, - excludedTasks: ["generateComponentPreload", "generateStandaloneAppBundle", "generateVersionInfo"] + excludedTasks: ["escapeNonAsciiCharacters", "generateComponentPreload", "generateStandaloneAppBundle", "generateVersionInfo"] }).then(() => { return findFiles(expectedPath); }).then((expectedFiles) => { @@ -136,7 +136,7 @@ test("Build application.g", (t) => { return builder.build({ tree: applicationGTree, destPath, - excludedTasks: ["generateStandaloneAppBundle", "generateVersionInfo"] + excludedTasks: ["escapeNonAsciiCharacters", "generateStandaloneAppBundle", "generateVersionInfo"] }).then(() => { return findFiles(expectedPath); }).then((expectedFiles) => { @@ -156,7 +156,7 @@ test("Build application.g with component preload paths", (t) => { return builder.build({ tree: applicationGTreeComponentPreloadPaths, destPath, - excludedTasks: ["generateStandaloneAppBundle", "generateVersionInfo"] + excludedTasks: ["escapeNonAsciiCharacters", "generateStandaloneAppBundle", "generateVersionInfo"] }).then(() => { return findFiles(expectedPath); }).then((expectedFiles) => { @@ -197,7 +197,7 @@ test("Build application.h", (t) => { return builder.build({ tree: applicationHTree, destPath, - excludedTasks: ["createDebugFiles", "generateComponentPreload", + excludedTasks: ["escapeNonAsciiCharacters", "createDebugFiles", "generateComponentPreload", "generateStandaloneAppBundle", "generateVersionInfo"] }).then(() => { return findFiles(expectedPath); @@ -218,7 +218,7 @@ test("Build application.i", (t) => { return builder.build({ tree: applicationITree, destPath, - excludedTasks: ["createDebugFiles", "generateStandaloneAppBundle", "generateVersionInfo"] + excludedTasks: ["escapeNonAsciiCharacters", "createDebugFiles", "generateStandaloneAppBundle", "generateVersionInfo"] }).then(() => { return findFiles(expectedPath); }).then((expectedFiles) => { @@ -238,7 +238,7 @@ test("Build library.d with copyright from .library file", (t) => { return builder.build({ tree: libraryDTree, destPath, - excludedTasks: ["generateLibraryPreload"] + excludedTasks: ["escapeNonAsciiCharacters", "generateLibraryPreload"] }).then(() => { return findFiles(expectedPath); }).then((expectedFiles) => { @@ -259,7 +259,7 @@ test("Build library.e with copyright from settings of ui5.yaml", (t) => { return builder.build({ tree: libraryETree, destPath, - excludedTasks: ["generateLibraryPreload"] + excludedTasks: ["escapeNonAsciiCharacters", "generateLibraryPreload"] }).then(() => { return findFiles(expectedPath); }).then((expectedFiles) => { @@ -280,7 +280,7 @@ test("Build library.h with custom bundles and component-preloads", (t) => { return builder.build({ tree: libraryHTree, destPath, - excludedTasks: ["createDebugFiles", "generateLibraryPreload"] + excludedTasks: ["escapeNonAsciiCharacters", "createDebugFiles", "generateLibraryPreload"] }).then(() => { return findFiles(expectedPath); }).then((expectedFiles) => { @@ -301,7 +301,7 @@ test("Build library.i with manifest info taken from .library and library.js", (t return builder.build({ tree: libraryITree, destPath, - excludedTasks: ["createDebugFiles", "generateLibraryPreload", "uglify"] + excludedTasks: ["escapeNonAsciiCharacters", "createDebugFiles", "generateLibraryPreload", "uglify"] }).then(() => { return findFiles(expectedPath); }).then((expectedFiles) => { @@ -342,7 +342,8 @@ test("Build theme.j even without an library", (t) => { const expectedPath = "./test/expected/build/theme.j/dest"; return builder.build({ tree: themeJTree, - destPath + destPath, + excludedTasks: ["escapeNonAsciiCharacters"] }).then(() => { return findFiles(expectedPath); }).then((expectedFiles) => { diff --git a/test/lib/processors/stringEscaper.js b/test/lib/processors/nonAsciiEscaper.js similarity index 83% rename from test/lib/processors/stringEscaper.js rename to test/lib/processors/nonAsciiEscaper.js index f445b204e..0b24da4cd 100644 --- a/test/lib/processors/stringEscaper.js +++ b/test/lib/processors/nonAsciiEscaper.js @@ -1,6 +1,6 @@ const test = require("ava"); -const stringEscaper = require("../../../lib/processors/stringEscaper"); +const nonAsciiEscaper = require("../../../lib/processors/nonAsciiEscaper"); /** * Executes string escaping. Returns undefined if nothing was escaped. @@ -22,7 +22,7 @@ const escape = async function(input, options) { return Buffer.from(input, "utf8"); } }; - return stringEscaper({resources: [resource], options}); + return nonAsciiEscaper({resources: [resource], options}); }; test("Replace symbol characters", async (t) => { @@ -87,3 +87,15 @@ test("Invalid encoding", async (t) => { const error = await t.throwsAsync(escape(input, {encoding: "asd"})); t.is(error.message, "Unknown encoding: asd"); }); + + +test("getEncodingFromNiceName", (t) => { + t.is("utf8", nonAsciiEscaper.getEncodingFromNiceName("UTF-8")); +}); + +test("getEncodingFromNiceName invalid", (t) => { + const error = t.throws(function() { + nonAsciiEscaper.getEncodingFromNiceName("asd"); + }); + t.is(error.message, `Encoding "asd" is not supported. Only UTF-8,ISO-8859-1 are allowed values`); +}); diff --git a/test/lib/tasks/escapeNonAsciiCharacters.js b/test/lib/tasks/escapeNonAsciiCharacters.js index 61d8c14d1..043b6a6ae 100644 --- a/test/lib/tasks/escapeNonAsciiCharacters.js +++ b/test/lib/tasks/escapeNonAsciiCharacters.js @@ -38,6 +38,7 @@ city=Ort:`; return tasks.escapeNonAsciiCharacters({ workspace, options: { + encoding: "UTF-8", pattern: "/**/*.properties" } }).then(() => { diff --git a/test/lib/tasks/generateCachebusterInfo.js b/test/lib/tasks/generateCachebusterInfo.js index 17ddaa9bb..52a403ced 100644 --- a/test/lib/tasks/generateCachebusterInfo.js +++ b/test/lib/tasks/generateCachebusterInfo.js @@ -27,7 +27,7 @@ const findFiles = (folder) => { test("integration: Build application.g with manifestBundler", (t) => { const destPath = path.join("test", "tmp", "build", "application.g", "cachebuster"); const expectedPath = path.join("test", "expected", "build", "application.g", "cachebuster"); - const excludedTasks = ["generateVersionInfo"]; + const excludedTasks = ["escapeNonAsciiCharacters", "generateVersionInfo"]; const includedTasks = ["generateCachebusterInfo"]; return builder.build({ @@ -60,7 +60,7 @@ test("integration: Build application.g with manifestBundler", (t) => { test("integration: Build application.g with manifestBundler and cachebuster using hashes", (t) => { const destPath = path.join("test", "tmp", "build", "application.g", "cachebuster_hash"); const expectedPath = path.join("test", "expected", "build", "application.g", "cachebuster"); - const excludedTasks = ["generateVersionInfo"]; + const excludedTasks = ["escapeNonAsciiCharacters", "generateVersionInfo"]; const includedTasks = ["generateCachebusterInfo"]; return builder.build({ From 7ba1022d90b228d78c616596cb6ef00638944ac7 Mon Sep 17 00:00:00 2001 From: Tobias Sorn Date: Mon, 22 Jul 2019 16:43:11 +0200 Subject: [PATCH 12/27] [FEATURE] Properties File Escaping renamed variable --- lib/lbt/bundle/AutoSplitter.js | 6 +++--- lib/processors/nonAsciiEscaper.js | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/lib/lbt/bundle/AutoSplitter.js b/lib/lbt/bundle/AutoSplitter.js index dc2fa831a..91916e98c 100644 --- a/lib/lbt/bundle/AutoSplitter.js +++ b/lib/lbt/bundle/AutoSplitter.js @@ -231,12 +231,12 @@ class AutoSplitter { // input encoding is read from project configuration, default is "ISO-8859-1" // since AutoSplitter is also used when serving non-project resources (e.g. dependencies) *.properties files should be escaped - const encoding = resource.getProject() && resource.getProject().resources.propertiesFileEncoding; - + const propertiesFileEncoding = resource.getProject() && resource.getProject().resources.propertiesFileEncoding; + const encoding = nonAsciiEscaper.getEncodingFromNiceName(propertiesFileEncoding || "ISO-8859-1"); await nonAsciiEscaper({ resources: [resource.resource], options: { - encoding: nonAsciiEscaper.getEncodingFromNiceName(encoding || "ISO-8859-1") + encoding } }); diff --git a/lib/processors/nonAsciiEscaper.js b/lib/processors/nonAsciiEscaper.js index c18d2c41e..7a7e9db7a 100644 --- a/lib/processors/nonAsciiEscaper.js +++ b/lib/processors/nonAsciiEscaper.js @@ -85,7 +85,6 @@ module.exports = async function nonAsciiEscaper({resources, options={}}) { const encodingMap = { "UTF-8": "utf8", "ISO-8859-1": "latin1", - }; /** From a6d2c6a334d88724228742b20724717e32d4302f Mon Sep 17 00:00:00 2001 From: Tobias Sorn Date: Tue, 23 Jul 2019 13:38:32 +0200 Subject: [PATCH 13/27] [FEATURE] Properties File Escaping Adjust name of helper method to get encoding name. Add documentation for helper method. --- lib/lbt/bundle/AutoSplitter.js | 2 +- lib/lbt/bundle/Builder.js | 2 +- lib/processors/nonAsciiEscaper.js | 10 +++++++--- lib/tasks/escapeNonAsciiCharacters.js | 2 +- test/lib/processors/nonAsciiEscaper.js | 8 ++++---- 5 files changed, 14 insertions(+), 10 deletions(-) diff --git a/lib/lbt/bundle/AutoSplitter.js b/lib/lbt/bundle/AutoSplitter.js index 91916e98c..5199ea5af 100644 --- a/lib/lbt/bundle/AutoSplitter.js +++ b/lib/lbt/bundle/AutoSplitter.js @@ -232,7 +232,7 @@ class AutoSplitter { // input encoding is read from project configuration, default is "ISO-8859-1" // since AutoSplitter is also used when serving non-project resources (e.g. dependencies) *.properties files should be escaped const propertiesFileEncoding = resource.getProject() && resource.getProject().resources.propertiesFileEncoding; - const encoding = nonAsciiEscaper.getEncodingFromNiceName(propertiesFileEncoding || "ISO-8859-1"); + const encoding = nonAsciiEscaper.getEncodingFromAlias(propertiesFileEncoding || "ISO-8859-1"); await nonAsciiEscaper({ resources: [resource.resource], options: { diff --git a/lib/lbt/bundle/Builder.js b/lib/lbt/bundle/Builder.js index 93894debd..e0464ced9 100644 --- a/lib/lbt/bundle/Builder.js +++ b/lib/lbt/bundle/Builder.js @@ -450,7 +450,7 @@ class BundleBuilder { await nonAsciiEscaper({ resources: [resource.resource], options: { - encoding: nonAsciiEscaper.getEncodingFromNiceName(encoding || "ISO-8859-1") + encoding: nonAsciiEscaper.getEncodingFromAlias(encoding || "ISO-8859-1") } }); diff --git a/lib/processors/nonAsciiEscaper.js b/lib/processors/nonAsciiEscaper.js index 7a7e9db7a..c6ff59486 100644 --- a/lib/processors/nonAsciiEscaper.js +++ b/lib/processors/nonAsciiEscaper.js @@ -53,7 +53,7 @@ const escapeNonAscii = function(string) { * Escapes non ASCII characters with unicode escape sequences. * * @example - * const encoding = nonAsciiEscaper.getEncodingFromNiceName("ISO-8859-1"); + * const encoding = nonAsciiEscaper.getEncodingFromAlias("ISO-8859-1"); * nonAsciiEscaper({resources, options: {encoding}}); * * @@ -62,7 +62,7 @@ const escapeNonAscii = function(string) { * @param {Object} parameters Parameters * @param {module:@ui5/fs.Resource[]} parameters.resources List of resources to be processed * @param {Object} [parameters.options] Options - * @param {string} [parameters.options.encoding="utf8"] resource file encoding (node.js based encodings). Use #getEncodingFromNiceName to get the encoding string + * @param {string} [parameters.options.encoding="utf8"] resource file encoding (node.js based encodings). Use #getEncodingFromAlias to get the encoding string * {@link https://nodejs.org/api/buffer.html#buffer_buffers_and_character_encodings Node.js character encodings}; * @returns {Promise} Promise resolving with the processed resources */ @@ -88,10 +88,14 @@ const encodingMap = { }; /** + * Provides a mapping from user-friendly encoding name (alias) such as "UTF-8" and "ISO-8859-1" to node + * specific encoding name such as "utf8" or "latin1". Simplifies usage of nonAsciiEscaper encoding + * option such that it can be used standalone without the respective task (e.g. in Splitter, Bundler and related projects). + * * @param {string} encoding encoding labels: "UTF-8" and "ISO-8859-1" * @returns {string} node.js character encoding string, e.g. utf8 and latin1 */ -module.exports.getEncodingFromNiceName = function(encoding) { +module.exports.getEncodingFromAlias = function(encoding) { if (!encodingMap[encoding]) { throw new Error(`Encoding "${encoding}" is not supported. Only ${Object.keys(encodingMap).join(",")} are allowed values` ); } diff --git a/lib/tasks/escapeNonAsciiCharacters.js b/lib/tasks/escapeNonAsciiCharacters.js index e62e2ea5f..9b1b35dc2 100644 --- a/lib/tasks/escapeNonAsciiCharacters.js +++ b/lib/tasks/escapeNonAsciiCharacters.js @@ -22,7 +22,7 @@ module.exports = async function({workspace, options}) { const processedResources = await nonAsciiEscaper({ resources: allResources, options: { - encoding: nonAsciiEscaper.getEncodingFromNiceName(options.encoding) + encoding: nonAsciiEscaper.getEncodingFromAlias(options.encoding) } }); diff --git a/test/lib/processors/nonAsciiEscaper.js b/test/lib/processors/nonAsciiEscaper.js index 0b24da4cd..693db1b55 100644 --- a/test/lib/processors/nonAsciiEscaper.js +++ b/test/lib/processors/nonAsciiEscaper.js @@ -89,13 +89,13 @@ test("Invalid encoding", async (t) => { }); -test("getEncodingFromNiceName", (t) => { - t.is("utf8", nonAsciiEscaper.getEncodingFromNiceName("UTF-8")); +test("getEncodingFromAlias", (t) => { + t.is("utf8", nonAsciiEscaper.getEncodingFromAlias("UTF-8")); }); -test("getEncodingFromNiceName invalid", (t) => { +test("getEncodingFromAlias invalid", (t) => { const error = t.throws(function() { - nonAsciiEscaper.getEncodingFromNiceName("asd"); + nonAsciiEscaper.getEncodingFromAlias("asd"); }); t.is(error.message, `Encoding "asd" is not supported. Only UTF-8,ISO-8859-1 are allowed values`); }); From bf84357ef914bfef6a2699d7ffdc5bc5d8ee8168 Mon Sep 17 00:00:00 2001 From: Tobias Sorn Date: Tue, 23 Jul 2019 14:35:38 +0200 Subject: [PATCH 14/27] [FEATURE] Properties File Escaping Set default in formatters for encoding value. Add validation to formatters for encoding value. --- lib/types/application/ApplicationFormatter.js | 14 +++++++++----- lib/types/library/LibraryFormatter.js | 14 +++++++++----- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/lib/types/application/ApplicationFormatter.js b/lib/types/application/ApplicationFormatter.js index d42d0c9c3..651162c0e 100644 --- a/lib/types/application/ApplicationFormatter.js +++ b/lib/types/application/ApplicationFormatter.js @@ -20,11 +20,6 @@ class ApplicationFormatter extends AbstractUi5Formatter { "/": project.resources.configuration.paths.webapp }; - // set default encoding - if (!project.resources.propertiesFileEncoding) { - project.resources.propertiesFileEncoding = "ISO-8859-1"; - } - try { project.metadata.namespace = await this.getNamespace(); } catch (err) { @@ -138,6 +133,15 @@ class ApplicationFormatter extends AbstractUi5Formatter { project.resources.configuration.paths.webapp = "webapp"; } + // default encoding to "ISO-8859-1" if not specified + if (!project.resources.propertiesFileEncoding) { + project.resources.propertiesFileEncoding = "ISO-8859-1"; + } + if (!["ISO-8859-1", "UTF-8"].includes(project.resources.propertiesFileEncoding)) { + throw new Error(`Invalid properties file encoding specified for project ${project.id}: ` + + `encoding provided: ${project.resources.propertiesFileEncoding}. Must be either "ISO-8859-1" or "UTF-8".`); + } + const absolutePath = path.join(project.path, project.resources.configuration.paths.webapp); return this.dirExists(absolutePath).then((bExists) => { if (!bExists) { diff --git a/lib/types/library/LibraryFormatter.js b/lib/types/library/LibraryFormatter.js index 592c6ea3d..e26c1792b 100644 --- a/lib/types/library/LibraryFormatter.js +++ b/lib/types/library/LibraryFormatter.js @@ -23,11 +23,6 @@ class LibraryFormatter extends AbstractUi5Formatter { "/resources/": project.resources.configuration.paths.src }; - // set default encoding - if (!project.resources.propertiesFileEncoding) { - project.resources.propertiesFileEncoding = "ISO-8859-1"; - } - if (project.resources.configuration.paths.test) { // Directory 'test' is somewhat optional for libraries project.resources.pathMappings["/test-resources/"] = project.resources.configuration.paths.test; @@ -345,6 +340,15 @@ class LibraryFormatter extends AbstractUi5Formatter { project.resources.configuration.paths.test = "test"; } + // default encoding to "ISO-8859-1" if not specified + if (!project.resources.propertiesFileEncoding) { + project.resources.propertiesFileEncoding = "ISO-8859-1"; + } + if (!["ISO-8859-1", "UTF-8"].includes(project.resources.propertiesFileEncoding)) { + throw new Error(`Invalid properties file encoding specified for project ${project.id}: ` + + `encoding provided: ${project.resources.propertiesFileEncoding}. Must be either "ISO-8859-1" or "UTF-8".`); + } + const absoluteSrcPath = path.join(project.path, project.resources.configuration.paths.src); const absoluteTestPath = path.join(project.path, project.resources.configuration.paths.test); return Promise.all([ From a11dad92fe62f7e23c0e948fd35d59f7375c7cc1 Mon Sep 17 00:00:00 2001 From: Tobias Sorn Date: Tue, 23 Jul 2019 15:27:06 +0200 Subject: [PATCH 15/27] [FEATURE] Properties File Escaping Only apply encoding in AutoSplitter and Builder if encoding value is specified. --- lib/lbt/bundle/AutoSplitter.js | 19 ++++++++++-------- lib/lbt/bundle/Builder.js | 20 +++++++++++-------- .../bundlers/generateStandaloneAppBundle.js | 3 ++- test/lib/types/library/LibraryFormatter.js | 6 +++--- 4 files changed, 28 insertions(+), 20 deletions(-) diff --git a/lib/lbt/bundle/AutoSplitter.js b/lib/lbt/bundle/AutoSplitter.js index 5199ea5af..703dbbef0 100644 --- a/lib/lbt/bundle/AutoSplitter.js +++ b/lib/lbt/bundle/AutoSplitter.js @@ -230,15 +230,18 @@ class AutoSplitter { */ // input encoding is read from project configuration, default is "ISO-8859-1" - // since AutoSplitter is also used when serving non-project resources (e.g. dependencies) *.properties files should be escaped + // since AutoSplitter is also used when splitting non-project resources (e.g. dependencies) + // *.properties files should be escaped if encoding option is specified const propertiesFileEncoding = resource.getProject() && resource.getProject().resources.propertiesFileEncoding; - const encoding = nonAsciiEscaper.getEncodingFromAlias(propertiesFileEncoding || "ISO-8859-1"); - await nonAsciiEscaper({ - resources: [resource.resource], - options: { - encoding - } - }); + if (propertiesFileEncoding) { + const encoding = nonAsciiEscaper.getEncodingFromAlias(propertiesFileEncoding); + await nonAsciiEscaper({ + resources: [resource.resource], + options: { + encoding + } + }); + } const fileContent = await resource.buffer(); diff --git a/lib/lbt/bundle/Builder.js b/lib/lbt/bundle/Builder.js index e0464ced9..34b47de53 100644 --- a/lib/lbt/bundle/Builder.js +++ b/lib/lbt/bundle/Builder.js @@ -445,14 +445,18 @@ class BundleBuilder { outW.write( makeStringLiteral( fileContent ) ); } else if ( /\.properties$/.test(module) ) { // same as for other text files, input encoding is read from project configuration, default is ISO-8859-1 - const encoding = resource.getProject() && resource.getProject().resources.propertiesFileEncoding; - - await nonAsciiEscaper({ - resources: [resource.resource], - options: { - encoding: nonAsciiEscaper.getEncodingFromAlias(encoding || "ISO-8859-1") - } - }); + // since Builder is also used when building non-project resources (e.g. dependencies) + // *.properties files should be escaped if encoding option is specified + const propertiesFileEncoding = resource.getProject() && resource.getProject().resources.propertiesFileEncoding; + if (propertiesFileEncoding) { + const encoding = nonAsciiEscaper.getEncodingFromAlias(propertiesFileEncoding); + await nonAsciiEscaper({ + resources: [resource.resource], + options: { + encoding + } + }); + } const fileContent = await resource.buffer(); diff --git a/test/lib/tasks/bundlers/generateStandaloneAppBundle.js b/test/lib/tasks/bundlers/generateStandaloneAppBundle.js index cb0a975ee..d4265a4bb 100644 --- a/test/lib/tasks/bundlers/generateStandaloneAppBundle.js +++ b/test/lib/tasks/bundlers/generateStandaloneAppBundle.js @@ -333,6 +333,7 @@ const applicationBTree = { }, "pathMappings": { "/": "webapp" - } + }, + "propertiesFileEncoding": "ISO-8859-1" } }; diff --git a/test/lib/types/library/LibraryFormatter.js b/test/lib/types/library/LibraryFormatter.js index 83d1dd20f..3edd6d91a 100644 --- a/test/lib/types/library/LibraryFormatter.js +++ b/test/lib/types/library/LibraryFormatter.js @@ -109,7 +109,8 @@ test("validate: test directory does not exist", async (t) => { await libraryFormatter.validate(myProject); // Missing test directory is not an error - t.deepEqual(myProject.resources.configuration.paths.test, null, "Project test path configuration is set to nul"); + t.deepEqual(myProject.resources.configuration.paths.test, null, "Project test path configuration is set to null"); + t.deepEqual(myProject.resources.propertiesFileEncoding, "ISO-8859-1", "Project resources propertiesFileEncoding is set to ISO-8859-1"); }); test("format: copyright already configured", async (t) => { @@ -151,8 +152,7 @@ test("format: formats correctly", async (t) => { pathMappings: { "/resources/": "src", "/test-resources/": "test" - }, - propertiesFileEncoding: "ISO-8859-1" + } } }, "Project got formatted correctly"); }); From 09ca13879674eae93858bf9fff045b4c512d17bb Mon Sep 17 00:00:00 2001 From: Tobias Sorn Date: Tue, 23 Jul 2019 16:13:46 +0200 Subject: [PATCH 16/27] Add formatter tests for invalid encoding --- test/lib/types/application/ApplicationFormatter.js | 10 ++++++++++ test/lib/types/library/LibraryFormatter.js | 10 ++++++++++ 2 files changed, 20 insertions(+) diff --git a/test/lib/types/application/ApplicationFormatter.js b/test/lib/types/application/ApplicationFormatter.js index b65095a5c..c817c89ec 100644 --- a/test/lib/types/application/ApplicationFormatter.js +++ b/test/lib/types/application/ApplicationFormatter.js @@ -96,6 +96,16 @@ test("validate: empty resources", async (t) => { t.deepEqual(myProject.resources.configuration.paths.webapp, "webapp", "default webapp directory is set"); }); +test("validate: test invalid encoding", async (t) => { + const myProject = clone(applicationBTree); + myProject.resources.propertiesFileEncoding = "test"; + const applicationFormatter = new ApplicationFormatter({project: myProject}); + + const error = await t.throwsAsync(applicationFormatter.validate(myProject)); + t.is(error.message, `Invalid properties file encoding specified for project application.b: encoding provided: test. Must be either "ISO-8859-1" or "UTF-8".`, + "Missing source directory caused error"); +}); + function createMockProject() { return { resources: { diff --git a/test/lib/types/library/LibraryFormatter.js b/test/lib/types/library/LibraryFormatter.js index 3edd6d91a..7ba358a1b 100644 --- a/test/lib/types/library/LibraryFormatter.js +++ b/test/lib/types/library/LibraryFormatter.js @@ -113,6 +113,16 @@ test("validate: test directory does not exist", async (t) => { t.deepEqual(myProject.resources.propertiesFileEncoding, "ISO-8859-1", "Project resources propertiesFileEncoding is set to ISO-8859-1"); }); +test("validate: test invalid encoding", async (t) => { + const myProject = clone(libraryETree); + myProject.resources.propertiesFileEncoding = "test"; + const libraryFormatter = new LibraryFormatter({project: myProject}); + + const error = await t.throwsAsync(libraryFormatter.validate(myProject)); + t.is(error.message, `Invalid properties file encoding specified for project library.e.id: encoding provided: test. Must be either "ISO-8859-1" or "UTF-8".`, + "Missing source directory caused error"); +}); + test("format: copyright already configured", async (t) => { const myProject = clone(libraryETree); const libraryFormatter = new LibraryFormatter({project: myProject}); From 71778d236f682043ed27afd00e7b03330dcde869 Mon Sep 17 00:00:00 2001 From: Tobias Sorn Date: Tue, 23 Jul 2019 16:24:55 +0200 Subject: [PATCH 17/27] Task escapeNonAsciiCharacters: add tests for validation --- test/lib/tasks/escapeNonAsciiCharacters.js | 38 ++++++++++++++++++---- 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/test/lib/tasks/escapeNonAsciiCharacters.js b/test/lib/tasks/escapeNonAsciiCharacters.js index 043b6a6ae..3482ed1d4 100644 --- a/test/lib/tasks/escapeNonAsciiCharacters.js +++ b/test/lib/tasks/escapeNonAsciiCharacters.js @@ -105,14 +105,40 @@ city=Ort:`; }); }); -test.skip("integration: escape non ascii characters source encoding being UTF-16", (t) => { - return tasks.escapeNonAsciiCharacters({ - workspace: undefined, +test("integration: escape non ascii characters source encoding being empty", async (t) => { + const reader = resourceFactory.createAdapter({ + virBasePath: "/" + }); + const writer = resourceFactory.createAdapter({ + virBasePath: "/" + }); + const workspace = new DuplexCollection({reader, writer}); + + const error = await t.throwsAsync(tasks.escapeNonAsciiCharacters({ + workspace, options: { - encoding: "utf16le", + encoding: "", pattern: "/**/*.properties" } - }).catch((error) => { - return t.deepEqual(error.message, "Invalid encoding specified: 'UTF-16'. Must be one of UTF-8,ISO-8859-1"); + })); + return t.is(error.message, "No Encoding specified"); +}); + +test("integration: escape non ascii characters source encoding being UTF-16", async (t) => { + const reader = resourceFactory.createAdapter({ + virBasePath: "/" + }); + const writer = resourceFactory.createAdapter({ + virBasePath: "/" }); + const workspace = new DuplexCollection({reader, writer}); + + const error = await t.throwsAsync(tasks.escapeNonAsciiCharacters({ + workspace, + options: { + encoding: "utf16le", + pattern: "/**/*.properties" + } + })); + return t.is(error.message, `Encoding "utf16le" is not supported. Only UTF-8,ISO-8859-1 are allowed values`); }); From bb34d1bbc06f7983a1ff6ce3ab59bbb373de57c1 Mon Sep 17 00:00:00 2001 From: Tobias Sorn Date: Wed, 24 Jul 2019 11:10:14 +0200 Subject: [PATCH 18/27] Put propertiesFileEncoding option underneath configuration property in ui5.yml. It makes more sense to group it under configuration. --- lib/lbt/bundle/AutoSplitter.js | 5 ++++- lib/lbt/bundle/Builder.js | 5 ++++- lib/types/application/ApplicationBuilder.js | 5 ++++- lib/types/application/ApplicationFormatter.js | 8 ++++---- lib/types/library/LibraryBuilder.js | 5 ++++- lib/types/library/LibraryFormatter.js | 8 ++++---- test/lib/tasks/bundlers/generateStandaloneAppBundle.js | 6 +++--- test/lib/types/application/ApplicationFormatter.js | 2 +- test/lib/types/library/LibraryFormatter.js | 4 ++-- 9 files changed, 30 insertions(+), 18 deletions(-) diff --git a/lib/lbt/bundle/AutoSplitter.js b/lib/lbt/bundle/AutoSplitter.js index 703dbbef0..d5b6b5263 100644 --- a/lib/lbt/bundle/AutoSplitter.js +++ b/lib/lbt/bundle/AutoSplitter.js @@ -232,7 +232,10 @@ class AutoSplitter { // input encoding is read from project configuration, default is "ISO-8859-1" // since AutoSplitter is also used when splitting non-project resources (e.g. dependencies) // *.properties files should be escaped if encoding option is specified - const propertiesFileEncoding = resource.getProject() && resource.getProject().resources.propertiesFileEncoding; + const propertiesFileEncoding = resource.getProject() + && resource.getProject().resources + && resource.getProject().resources.configuration + && resource.getProject().resources.configuration.propertiesFileEncoding; if (propertiesFileEncoding) { const encoding = nonAsciiEscaper.getEncodingFromAlias(propertiesFileEncoding); await nonAsciiEscaper({ diff --git a/lib/lbt/bundle/Builder.js b/lib/lbt/bundle/Builder.js index 34b47de53..31a75467f 100644 --- a/lib/lbt/bundle/Builder.js +++ b/lib/lbt/bundle/Builder.js @@ -447,7 +447,10 @@ class BundleBuilder { // same as for other text files, input encoding is read from project configuration, default is ISO-8859-1 // since Builder is also used when building non-project resources (e.g. dependencies) // *.properties files should be escaped if encoding option is specified - const propertiesFileEncoding = resource.getProject() && resource.getProject().resources.propertiesFileEncoding; + const propertiesFileEncoding = resource.getProject() + && resource.getProject().resources + && resource.getProject().resources.configuration + && resource.getProject().resources.configuration.propertiesFileEncoding; if (propertiesFileEncoding) { const encoding = nonAsciiEscaper.getEncodingFromAlias(propertiesFileEncoding); await nonAsciiEscaper({ diff --git a/lib/types/application/ApplicationBuilder.js b/lib/types/application/ApplicationBuilder.js index ad921f4aa..ea3e62437 100644 --- a/lib/types/application/ApplicationBuilder.js +++ b/lib/types/application/ApplicationBuilder.js @@ -28,10 +28,13 @@ class ApplicationBuilder extends AbstractBuilder { } this.addTask("escapeNonAsciiCharacters", () => { + const propertiesFileEncoding = project.resources + && project.resources.configuration + && project.resources.configuration.propertiesFileEncoding; return tasks.escapeNonAsciiCharacters({ workspace: resourceCollections.workspace, options: { - encoding: project.resources && project.resources.propertiesFileEncoding, + encoding: propertiesFileEncoding, pattern: "/**/*.properties" } }); diff --git a/lib/types/application/ApplicationFormatter.js b/lib/types/application/ApplicationFormatter.js index 651162c0e..8e5fc8a10 100644 --- a/lib/types/application/ApplicationFormatter.js +++ b/lib/types/application/ApplicationFormatter.js @@ -134,12 +134,12 @@ class ApplicationFormatter extends AbstractUi5Formatter { } // default encoding to "ISO-8859-1" if not specified - if (!project.resources.propertiesFileEncoding) { - project.resources.propertiesFileEncoding = "ISO-8859-1"; + if (!project.resources.configuration.propertiesFileEncoding) { + project.resources.configuration.propertiesFileEncoding = "ISO-8859-1"; } - if (!["ISO-8859-1", "UTF-8"].includes(project.resources.propertiesFileEncoding)) { + if (!["ISO-8859-1", "UTF-8"].includes(project.resources.configuration.propertiesFileEncoding)) { throw new Error(`Invalid properties file encoding specified for project ${project.id}: ` + - `encoding provided: ${project.resources.propertiesFileEncoding}. Must be either "ISO-8859-1" or "UTF-8".`); + `encoding provided: ${project.resources.configuration.propertiesFileEncoding}. Must be either "ISO-8859-1" or "UTF-8".`); } const absolutePath = path.join(project.path, project.resources.configuration.paths.webapp); diff --git a/lib/types/library/LibraryBuilder.js b/lib/types/library/LibraryBuilder.js index 07e6c1e75..489fdf595 100644 --- a/lib/types/library/LibraryBuilder.js +++ b/lib/types/library/LibraryBuilder.js @@ -27,10 +27,13 @@ class LibraryBuilder extends AbstractBuilder { } this.addTask("escapeNonAsciiCharacters", () => { + const propertiesFileEncoding = project.resources + && project.resources.configuration + && project.resources.configuration.propertiesFileEncoding; return tasks.escapeNonAsciiCharacters({ workspace: resourceCollections.workspace, options: { - encoding: project.resources && project.resources.propertiesFileEncoding, + encoding: propertiesFileEncoding, pattern: "/**/*.properties" } }); diff --git a/lib/types/library/LibraryFormatter.js b/lib/types/library/LibraryFormatter.js index e26c1792b..9091f32f2 100644 --- a/lib/types/library/LibraryFormatter.js +++ b/lib/types/library/LibraryFormatter.js @@ -341,12 +341,12 @@ class LibraryFormatter extends AbstractUi5Formatter { } // default encoding to "ISO-8859-1" if not specified - if (!project.resources.propertiesFileEncoding) { - project.resources.propertiesFileEncoding = "ISO-8859-1"; + if (!project.resources.configuration.propertiesFileEncoding) { + project.resources.configuration.propertiesFileEncoding = "ISO-8859-1"; } - if (!["ISO-8859-1", "UTF-8"].includes(project.resources.propertiesFileEncoding)) { + if (!["ISO-8859-1", "UTF-8"].includes(project.resources.configuration.propertiesFileEncoding)) { throw new Error(`Invalid properties file encoding specified for project ${project.id}: ` + - `encoding provided: ${project.resources.propertiesFileEncoding}. Must be either "ISO-8859-1" or "UTF-8".`); + `encoding provided: ${project.resources.configuration.propertiesFileEncoding}. Must be either "ISO-8859-1" or "UTF-8".`); } const absoluteSrcPath = path.join(project.path, project.resources.configuration.paths.src); diff --git a/test/lib/tasks/bundlers/generateStandaloneAppBundle.js b/test/lib/tasks/bundlers/generateStandaloneAppBundle.js index d4265a4bb..1d9e8b240 100644 --- a/test/lib/tasks/bundlers/generateStandaloneAppBundle.js +++ b/test/lib/tasks/bundlers/generateStandaloneAppBundle.js @@ -329,11 +329,11 @@ const applicationBTree = { "configuration": { "paths": { "webapp": "webapp" - } + }, + "propertiesFileEncoding": "ISO-8859-1" }, "pathMappings": { "/": "webapp" - }, - "propertiesFileEncoding": "ISO-8859-1" + } } }; diff --git a/test/lib/types/application/ApplicationFormatter.js b/test/lib/types/application/ApplicationFormatter.js index c817c89ec..b12f81ee7 100644 --- a/test/lib/types/application/ApplicationFormatter.js +++ b/test/lib/types/application/ApplicationFormatter.js @@ -98,7 +98,7 @@ test("validate: empty resources", async (t) => { test("validate: test invalid encoding", async (t) => { const myProject = clone(applicationBTree); - myProject.resources.propertiesFileEncoding = "test"; + myProject.resources.configuration.propertiesFileEncoding = "test"; const applicationFormatter = new ApplicationFormatter({project: myProject}); const error = await t.throwsAsync(applicationFormatter.validate(myProject)); diff --git a/test/lib/types/library/LibraryFormatter.js b/test/lib/types/library/LibraryFormatter.js index 7ba358a1b..6b82d341d 100644 --- a/test/lib/types/library/LibraryFormatter.js +++ b/test/lib/types/library/LibraryFormatter.js @@ -110,12 +110,12 @@ test("validate: test directory does not exist", async (t) => { await libraryFormatter.validate(myProject); // Missing test directory is not an error t.deepEqual(myProject.resources.configuration.paths.test, null, "Project test path configuration is set to null"); - t.deepEqual(myProject.resources.propertiesFileEncoding, "ISO-8859-1", "Project resources propertiesFileEncoding is set to ISO-8859-1"); + t.deepEqual(myProject.resources.configuration.propertiesFileEncoding, "ISO-8859-1", "Project resources propertiesFileEncoding is set to ISO-8859-1"); }); test("validate: test invalid encoding", async (t) => { const myProject = clone(libraryETree); - myProject.resources.propertiesFileEncoding = "test"; + myProject.resources.configuration.propertiesFileEncoding = "test"; const libraryFormatter = new LibraryFormatter({project: myProject}); const error = await t.throwsAsync(libraryFormatter.validate(myProject)); From f0be15ce9f78ac2ac0d7e615cf5ac7ff798066d7 Mon Sep 17 00:00:00 2001 From: Tobias Sorn Date: Wed, 24 Jul 2019 13:48:37 +0200 Subject: [PATCH 19/27] escapeNonAsciiCharacters task: Add meaningful error message Include prefix and option name in escapeNonAsciiCharacters task --- lib/tasks/escapeNonAsciiCharacters.js | 2 +- test/lib/tasks/escapeNonAsciiCharacters.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/tasks/escapeNonAsciiCharacters.js b/lib/tasks/escapeNonAsciiCharacters.js index 9b1b35dc2..b755e1d37 100644 --- a/lib/tasks/escapeNonAsciiCharacters.js +++ b/lib/tasks/escapeNonAsciiCharacters.js @@ -14,7 +14,7 @@ const nonAsciiEscaper = require("../processors/nonAsciiEscaper"); */ module.exports = async function({workspace, options}) { if (!options.encoding) { - throw new Error("No Encoding specified"); + throw new Error("[escapeNonAsciiCharacters] Mandatory option 'encoding' not provided"); } const allResources = await workspace.byGlob(options.pattern); diff --git a/test/lib/tasks/escapeNonAsciiCharacters.js b/test/lib/tasks/escapeNonAsciiCharacters.js index 3482ed1d4..80953a061 100644 --- a/test/lib/tasks/escapeNonAsciiCharacters.js +++ b/test/lib/tasks/escapeNonAsciiCharacters.js @@ -121,7 +121,7 @@ test("integration: escape non ascii characters source encoding being empty", asy pattern: "/**/*.properties" } })); - return t.is(error.message, "No Encoding specified"); + return t.is(error.message, "[escapeNonAsciiCharacters] Mandatory option 'encoding' not provided"); }); test("integration: escape non ascii characters source encoding being UTF-16", async (t) => { From a6309736501ea7e2d45021a9ac0b04acd6dbde2b Mon Sep 17 00:00:00 2001 From: Tobias Sorn Date: Wed, 24 Jul 2019 16:42:52 +0200 Subject: [PATCH 20/27] improve tests --- lib/processors/nonAsciiEscaper.js | 2 +- .../dest/manifest-bundle/i18n/i18n.properties | 2 +- .../manifest-bundle/i18n/i18n_de.properties | 2 +- test/lib/builder/builder.js | 51 +++++++++++-------- test/lib/lbt/bundle/AutoSplitter.js | 30 +++++++++-- test/lib/processors/nonAsciiEscaper.js | 33 ++++++------ .../tasks/bundlers/generateManifestBundle.js | 5 +- test/lib/tasks/escapeNonAsciiCharacters.js | 2 +- 8 files changed, 81 insertions(+), 46 deletions(-) diff --git a/lib/processors/nonAsciiEscaper.js b/lib/processors/nonAsciiEscaper.js index c6ff59486..66b025d1d 100644 --- a/lib/processors/nonAsciiEscaper.js +++ b/lib/processors/nonAsciiEscaper.js @@ -97,7 +97,7 @@ const encodingMap = { */ module.exports.getEncodingFromAlias = function(encoding) { if (!encodingMap[encoding]) { - throw new Error(`Encoding "${encoding}" is not supported. Only ${Object.keys(encodingMap).join(",")} are allowed values` ); + throw new Error(`Encoding "${encoding}" is not supported. Only ${Object.keys(encodingMap).join(", ")} are allowed values` ); } return encodingMap[encoding]; }; diff --git a/test/expected/build/application.b/dest/manifest-bundle/i18n/i18n.properties b/test/expected/build/application.b/dest/manifest-bundle/i18n/i18n.properties index 49f4756f9..17772dbf5 100644 --- a/test/expected/build/application.b/dest/manifest-bundle/i18n/i18n.properties +++ b/test/expected/build/application.b/dest/manifest-bundle/i18n/i18n.properties @@ -1,2 +1,2 @@ title=app-i18n -fame=Straße \ No newline at end of file +fame=Stra\u00dfe \ No newline at end of file diff --git a/test/expected/build/application.b/dest/manifest-bundle/i18n/i18n_de.properties b/test/expected/build/application.b/dest/manifest-bundle/i18n/i18n_de.properties index a81193360..d2c23da4b 100644 --- a/test/expected/build/application.b/dest/manifest-bundle/i18n/i18n_de.properties +++ b/test/expected/build/application.b/dest/manifest-bundle/i18n/i18n_de.properties @@ -1,2 +1,2 @@ title=app-i18n_de -fame=Straße \ No newline at end of file +fame=Stra\u00dfe \ No newline at end of file diff --git a/test/lib/builder/builder.js b/test/lib/builder/builder.js index 9ae0c5a1e..eeb898c36 100644 --- a/test/lib/builder/builder.js +++ b/test/lib/builder/builder.js @@ -84,7 +84,7 @@ test("Build application.a", (t) => { return builder.build({ tree: applicationATree, destPath, - excludedTasks: ["escapeNonAsciiCharacters", "generateComponentPreload", "generateStandaloneAppBundle", "generateVersionInfo"] + excludedTasks: ["generateComponentPreload", "generateStandaloneAppBundle", "generateVersionInfo"] }).then(() => { return findFiles(expectedPath); }).then((expectedFiles) => { @@ -136,7 +136,7 @@ test("Build application.g", (t) => { return builder.build({ tree: applicationGTree, destPath, - excludedTasks: ["escapeNonAsciiCharacters", "generateStandaloneAppBundle", "generateVersionInfo"] + excludedTasks: ["generateStandaloneAppBundle", "generateVersionInfo"] }).then(() => { return findFiles(expectedPath); }).then((expectedFiles) => { @@ -156,7 +156,7 @@ test("Build application.g with component preload paths", (t) => { return builder.build({ tree: applicationGTreeComponentPreloadPaths, destPath, - excludedTasks: ["escapeNonAsciiCharacters", "generateStandaloneAppBundle", "generateVersionInfo"] + excludedTasks: ["generateStandaloneAppBundle", "generateVersionInfo"] }).then(() => { return findFiles(expectedPath); }).then((expectedFiles) => { @@ -197,7 +197,7 @@ test("Build application.h", (t) => { return builder.build({ tree: applicationHTree, destPath, - excludedTasks: ["escapeNonAsciiCharacters", "createDebugFiles", "generateComponentPreload", + excludedTasks: ["createDebugFiles", "generateComponentPreload", "generateStandaloneAppBundle", "generateVersionInfo"] }).then(() => { return findFiles(expectedPath); @@ -218,7 +218,7 @@ test("Build application.i", (t) => { return builder.build({ tree: applicationITree, destPath, - excludedTasks: ["escapeNonAsciiCharacters", "createDebugFiles", "generateStandaloneAppBundle", "generateVersionInfo"] + excludedTasks: ["createDebugFiles", "generateStandaloneAppBundle", "generateVersionInfo"] }).then(() => { return findFiles(expectedPath); }).then((expectedFiles) => { @@ -238,7 +238,7 @@ test("Build library.d with copyright from .library file", (t) => { return builder.build({ tree: libraryDTree, destPath, - excludedTasks: ["escapeNonAsciiCharacters", "generateLibraryPreload"] + excludedTasks: ["generateLibraryPreload"] }).then(() => { return findFiles(expectedPath); }).then((expectedFiles) => { @@ -259,7 +259,7 @@ test("Build library.e with copyright from settings of ui5.yaml", (t) => { return builder.build({ tree: libraryETree, destPath, - excludedTasks: ["escapeNonAsciiCharacters", "generateLibraryPreload"] + excludedTasks: ["generateLibraryPreload"] }).then(() => { return findFiles(expectedPath); }).then((expectedFiles) => { @@ -280,7 +280,7 @@ test("Build library.h with custom bundles and component-preloads", (t) => { return builder.build({ tree: libraryHTree, destPath, - excludedTasks: ["escapeNonAsciiCharacters", "createDebugFiles", "generateLibraryPreload"] + excludedTasks: ["createDebugFiles", "generateLibraryPreload"] }).then(() => { return findFiles(expectedPath); }).then((expectedFiles) => { @@ -301,7 +301,7 @@ test("Build library.i with manifest info taken from .library and library.js", (t return builder.build({ tree: libraryITree, destPath, - excludedTasks: ["escapeNonAsciiCharacters", "createDebugFiles", "generateLibraryPreload", "uglify"] + excludedTasks: ["createDebugFiles", "generateLibraryPreload", "uglify"] }).then(() => { return findFiles(expectedPath); }).then((expectedFiles) => { @@ -534,7 +534,8 @@ const applicationATree = { "configuration": { "paths": { "webapp": "webapp" - } + }, + "propertiesFileEncoding": "ISO-8859-1" }, "pathMappings": { "/": "webapp" @@ -557,7 +558,8 @@ const applicationATreeBadType = { "configuration": { "paths": { "webapp": "webapp" - } + }, + "propertiesFileEncoding": "ISO-8859-1" }, "pathMappings": { "/": "webapp" @@ -582,7 +584,8 @@ const applicationGTree = { "configuration": { "paths": { "webapp": "webapp" - } + }, + "propertiesFileEncoding": "ISO-8859-1" }, "pathMappings": { "/": "webapp" @@ -616,7 +619,8 @@ const applicationGTreeWithExcludes = { "configuration": { "paths": { "webapp": "webapp" - } + }, + "propertiesFileEncoding": "ISO-8859-1" }, "pathMappings": { "/": "webapp" @@ -651,7 +655,8 @@ const applicationGTreeComponentPreloadPaths = { "configuration": { "paths": { "webapp": "webapp" - } + }, + "propertiesFileEncoding": "ISO-8859-1" }, "pathMappings": { "/": "webapp" @@ -682,7 +687,8 @@ const applicationHTree = { "configuration": { "paths": { "webapp": "webapp" - } + }, + "propertiesFileEncoding": "ISO-8859-1" }, "pathMappings": { "/": "webapp" @@ -742,7 +748,8 @@ const applicationITree = { "configuration": { "paths": { "webapp": "webapp" - } + }, + "propertiesFileEncoding": "ISO-8859-1" }, "pathMappings": { "/": "webapp" @@ -796,7 +803,8 @@ const libraryDTree = { "paths": { "src": "main/src", "test": "main/test" - } + }, + "propertiesFileEncoding": "ISO-8859-1" }, "pathMappings": { "/resources/": "main/src", @@ -848,7 +856,8 @@ const libraryETree = { "paths": { "src": "src", "test": "test" - } + }, + "propertiesFileEncoding": "ISO-8859-1" }, "pathMappings": { "/resources/": "src", @@ -900,7 +909,8 @@ const libraryHTree = { "paths": { "src": "main/src", "test": "main/test" - } + }, + "propertiesFileEncoding": "ISO-8859-1" }, "pathMappings": { "/resources/": "main/src", @@ -985,7 +995,8 @@ const libraryITree = { "paths": { "src": "main/src", "test": "main/test" - } + }, + "propertiesFileEncoding": "ISO-8859-1" }, "pathMappings": { "/resources/": "main/src" diff --git a/test/lib/lbt/bundle/AutoSplitter.js b/test/lib/lbt/bundle/AutoSplitter.js index 62db2616b..0f29cd2f1 100644 --- a/test/lib/lbt/bundle/AutoSplitter.js +++ b/test/lib/lbt/bundle/AutoSplitter.js @@ -19,7 +19,15 @@ function createMockPool(dependencies) { return { info, buffer: async () => Buffer.from(name.padStart(2048, "*")), - getProject: () => undefined, + getProject: () => { + return { + "resources": { + "configuration": { + "propertiesFileEncoding": "ISO-8859-1" + } + } + }; + }, resource: { getBuffer: () => Buffer.from(name.padStart(2048, "*")) } @@ -192,17 +200,29 @@ test.serial("_calcMinSize: uglify js resource", async (t) => { test("_calcMinSize: properties resource", async (t) => { const pool = { findResourceWithInfo: function() { + let content = "1234ß"; return { - buffer: async () => Buffer.from("1234"), + buffer: async () => Buffer.from(content), resource: { - getBuffer: () => Buffer.from("1234") + setString: (string) => { + content = string; + }, + getBuffer: () => Buffer.from(content, "latin1") }, - getProject: () => undefined + getProject: () => { + return { + "resources": { + "configuration": { + "propertiesFileEncoding": "ISO-8859-1" + } + } + }; + } }; } }; const autpSplitter = new AutoSplitter(pool); - t.deepEqual(await autpSplitter._calcMinSize("mymodule.properties"), 4); + t.deepEqual(await autpSplitter._calcMinSize("mymodule.properties"), 10, "length of 1234\\u00df"); }); test("_calcMinSize: xml view resource", async (t) => { diff --git a/test/lib/processors/nonAsciiEscaper.js b/test/lib/processors/nonAsciiEscaper.js index 693db1b55..a499cfbcd 100644 --- a/test/lib/processors/nonAsciiEscaper.js +++ b/test/lib/processors/nonAsciiEscaper.js @@ -1,30 +1,33 @@ const test = require("ava"); const nonAsciiEscaper = require("../../../lib/processors/nonAsciiEscaper"); +const Resource = require("@ui5/fs").Resource; /** * Executes string escaping. Returns undefined if nothing was escaped. * * @param {string} input string * @param {Object} [options] + * @param {string} encoding character encoding used, e.g. utf8, latin1, .. * @returns {Promise} escaped string if non-ascii characters present, undefined otherwise */ -const escape = async function(input, options) { - let result = undefined; - const resource = { - setString: (actual) => { - result = actual; - }, - getString: async () => { - return result; - }, - getBuffer: async () => { - return Buffer.from(input, "utf8"); - } - }; +const escape = async function(input, options={}, encoding="utf8") { + const resource = new Resource({ + path: "my.properties", + buffer: Buffer.from(input, encoding) + }); return nonAsciiEscaper({resources: [resource], options}); }; +test("Replace ISO-8859-1 characters", async (t) => { + t.plan(1); + + const input = `Viele Grüße`; + const expected = `Viele Gr\\ufffd\\ufffde`; + const [resource] = await escape(input, {}, "latin1"); + t.deepEqual(await resource.getString(), expected, "Correct file content should be set"); +}); + test("Replace symbol characters", async (t) => { t.plan(1); @@ -75,7 +78,7 @@ test("No Replace of characters", async (t) => { t.plan(1); const input = `ONE LOVE`; - const expected = undefined; + const expected = `ONE LOVE`; const [resource] = await escape(input); t.deepEqual(await resource.getString(), expected, "Correct file content should be set"); }); @@ -97,5 +100,5 @@ test("getEncodingFromAlias invalid", (t) => { const error = t.throws(function() { nonAsciiEscaper.getEncodingFromAlias("asd"); }); - t.is(error.message, `Encoding "asd" is not supported. Only UTF-8,ISO-8859-1 are allowed values`); + t.is(error.message, `Encoding "asd" is not supported. Only UTF-8, ISO-8859-1 are allowed values`); }); diff --git a/test/lib/tasks/bundlers/generateManifestBundle.js b/test/lib/tasks/bundlers/generateManifestBundle.js index 3cb5e9b5b..5f79bcd0c 100644 --- a/test/lib/tasks/bundlers/generateManifestBundle.js +++ b/test/lib/tasks/bundlers/generateManifestBundle.js @@ -118,7 +118,7 @@ test("integration: Build application.b with manifestBundler", (t) => { const destBundle = path.resolve(path.join(destPath, "manifest-bundle")); const expectedPath = path.join("test", "expected", "build", "application.b", "dest", "manifest-bundle"); const excludedTasks = ["*"]; - const includedTasks = ["generateManifestBundle"]; + const includedTasks = ["escapeNonAsciiCharacters", "generateManifestBundle"]; return builder.build({ tree: applicationBTree, @@ -189,7 +189,8 @@ const applicationBTree = { "configuration": { "paths": { "webapp": "webapp" - } + }, + "propertiesFileEncoding": "ISO-8859-1" }, "pathMappings": { "/": "webapp" diff --git a/test/lib/tasks/escapeNonAsciiCharacters.js b/test/lib/tasks/escapeNonAsciiCharacters.js index 80953a061..9c16de7a4 100644 --- a/test/lib/tasks/escapeNonAsciiCharacters.js +++ b/test/lib/tasks/escapeNonAsciiCharacters.js @@ -140,5 +140,5 @@ test("integration: escape non ascii characters source encoding being UTF-16", as pattern: "/**/*.properties" } })); - return t.is(error.message, `Encoding "utf16le" is not supported. Only UTF-8,ISO-8859-1 are allowed values`); + return t.is(error.message, `Encoding "utf16le" is not supported. Only UTF-8, ISO-8859-1 are allowed values`); }); From e0028f05969cc65d7b1a910592494a9e81c9d3fe Mon Sep 17 00:00:00 2001 From: Thorsten Hochreuter Date: Thu, 25 Jul 2019 09:40:25 +0200 Subject: [PATCH 21/27] Refactor escaping in LBT --- lib/lbt/bundle/AutoSplitter.js | 24 ++++----------------- lib/lbt/bundle/Builder.js | 24 ++++----------------- lib/lbt/utils/escapePropertiesFiles.js | 29 ++++++++++++++++++++++++++ 3 files changed, 37 insertions(+), 40 deletions(-) create mode 100644 lib/lbt/utils/escapePropertiesFiles.js diff --git a/lib/lbt/bundle/AutoSplitter.js b/lib/lbt/bundle/AutoSplitter.js index d5b6b5263..e24d2e8ef 100644 --- a/lib/lbt/bundle/AutoSplitter.js +++ b/lib/lbt/bundle/AutoSplitter.js @@ -3,10 +3,9 @@ const uglify = require("uglify-es"); const {pd} = require("pretty-data"); -const nonAsciiEscaper = require("../../processors/nonAsciiEscaper"); - const ModuleName = require("../utils/ModuleName"); const {SectionType} = require("./BundleDefinition"); +const escapePropertiesFiles = require("../utils/escapePropertiesFiles"); const log = require("@ui5/logger").getLogger("lbt:bundle:AutoSplitter"); const copyrightCommentsPattern = /copyright|\(c\)(?:[0-9]+|\s+[0-9A-za-z])|released under|license|\u00a9/i; @@ -229,26 +228,11 @@ class AutoSplitter { return out.toString().length(); */ - // input encoding is read from project configuration, default is "ISO-8859-1" - // since AutoSplitter is also used when splitting non-project resources (e.g. dependencies) + // Since AutoSplitter is also used when splitting non-project resources (e.g. dependencies) // *.properties files should be escaped if encoding option is specified - const propertiesFileEncoding = resource.getProject() - && resource.getProject().resources - && resource.getProject().resources.configuration - && resource.getProject().resources.configuration.propertiesFileEncoding; - if (propertiesFileEncoding) { - const encoding = nonAsciiEscaper.getEncodingFromAlias(propertiesFileEncoding); - await nonAsciiEscaper({ - resources: [resource.resource], - options: { - encoding - } - }); - } + const fileContent = await escapePropertiesFiles(resource); - const fileContent = await resource.buffer(); - - return fileContent.toString().length; + return fileContent.length; } else if ( this.optimizeXMLViews && /\.view.xml$/.test(module) ) { // needs to be activated when it gets activated in JSMergedModuleBuilderExt let fileContent = await resource.buffer(); diff --git a/lib/lbt/bundle/Builder.js b/lib/lbt/bundle/Builder.js index 31a75467f..211cc5769 100644 --- a/lib/lbt/bundle/Builder.js +++ b/lib/lbt/bundle/Builder.js @@ -9,11 +9,10 @@ const escodegen = require("escodegen"); const {Syntax} = esprima; // const MOZ_SourceMap = require("source-map"); -const nonAsciiEscaper = require("../../processors/nonAsciiEscaper"); - const {isMethodCall} = require("../utils/ASTUtils"); const ModuleName = require("../utils/ModuleName"); const UI5ClientConstants = require("../UI5ClientConstants"); +const escapePropertiesFiles = require("../utils/escapePropertiesFiles"); const BundleResolver = require("./Resolver"); const BundleSplitter = require("./AutoSplitter"); @@ -444,26 +443,11 @@ class BundleBuilder { } outW.write( makeStringLiteral( fileContent ) ); } else if ( /\.properties$/.test(module) ) { - // same as for other text files, input encoding is read from project configuration, default is ISO-8859-1 - // since Builder is also used when building non-project resources (e.g. dependencies) + // Since the Builder is also used when building non-project resources (e.g. dependencies) // *.properties files should be escaped if encoding option is specified - const propertiesFileEncoding = resource.getProject() - && resource.getProject().resources - && resource.getProject().resources.configuration - && resource.getProject().resources.configuration.propertiesFileEncoding; - if (propertiesFileEncoding) { - const encoding = nonAsciiEscaper.getEncodingFromAlias(propertiesFileEncoding); - await nonAsciiEscaper({ - resources: [resource.resource], - options: { - encoding - } - }); - } + const fileContent = await escapePropertiesFiles(resource); - const fileContent = await resource.buffer(); - - outW.write( makeStringLiteral( fileContent.toString() ) ); + outW.write( makeStringLiteral( fileContent ) ); } else { log.error("don't know how to embed module " + module); // TODO throw? } diff --git a/lib/lbt/utils/escapePropertiesFiles.js b/lib/lbt/utils/escapePropertiesFiles.js new file mode 100644 index 000000000..871fa177e --- /dev/null +++ b/lib/lbt/utils/escapePropertiesFiles.js @@ -0,0 +1,29 @@ +const nonAsciiEscaper = require("../../processors/nonAsciiEscaper"); + +/** + * Can be used to escape *.properties files. + * + * Input encoding is read from project configuration. + * In case the resource belongs to no project (e.g. bundler is used standalone) the default is "ISO-8859-1". + * + * @private + * @param {Resource} resource the resource for which the content will be escaped + * @returns {Promise} resolves with the escaped string content of the given Resource + */ +module.exports = async function(resource) { + const propertiesFileEncoding = resource.getProject() + && resource.getProject().resources + && resource.getProject().resources.configuration + && resource.getProject().resources.configuration.propertiesFileEncoding; + const encoding = nonAsciiEscaper.getEncodingFromAlias(propertiesFileEncoding || "ISO-8859-1"); + await nonAsciiEscaper({ + resources: [resource.resource], + options: { + encoding + } + }); + + const fileContent = await resource.buffer(); + + return fileContent.toString(encoding); +}; From 4353a51992fd1e3f694d70ffab3fbffe1de27448 Mon Sep 17 00:00:00 2001 From: Thorsten Hochreuter Date: Thu, 25 Jul 2019 10:31:15 +0200 Subject: [PATCH 22/27] Fixed tests for Autosplitter and nonAsciiEscaper --- test/lib/lbt/bundle/AutoSplitter.js | 4 ++-- test/lib/processors/nonAsciiEscaper.js | 29 +++++++++++++++++++++++--- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/test/lib/lbt/bundle/AutoSplitter.js b/test/lib/lbt/bundle/AutoSplitter.js index 0f29cd2f1..15fec1a24 100644 --- a/test/lib/lbt/bundle/AutoSplitter.js +++ b/test/lib/lbt/bundle/AutoSplitter.js @@ -29,7 +29,7 @@ function createMockPool(dependencies) { }; }, resource: { - getBuffer: () => Buffer.from(name.padStart(2048, "*")) + getBuffer: async () => Buffer.from(name.padStart(2048, "*")) } }; }, @@ -207,7 +207,7 @@ test("_calcMinSize: properties resource", async (t) => { setString: (string) => { content = string; }, - getBuffer: () => Buffer.from(content, "latin1") + getBuffer: async () => Buffer.from(content, "latin1") }, getProject: () => { return { diff --git a/test/lib/processors/nonAsciiEscaper.js b/test/lib/processors/nonAsciiEscaper.js index a499cfbcd..4b10c0c1b 100644 --- a/test/lib/processors/nonAsciiEscaper.js +++ b/test/lib/processors/nonAsciiEscaper.js @@ -19,12 +19,35 @@ const escape = async function(input, options={}, encoding="utf8") { return nonAsciiEscaper({resources: [resource], options}); }; -test("Replace ISO-8859-1 characters", async (t) => { +test("Escape characters (ISO-8859-1)", async (t) => { t.plan(1); const input = `Viele Grüße`; - const expected = `Viele Gr\\ufffd\\ufffde`; - const [resource] = await escape(input, {}, "latin1"); + const expected = `Viele Gr\\u00fc\\u00dfe`; + const [resource] = await escape(input, { + encoding: "latin1" // escape encoding + }, "latin1"); // resource encoding + t.deepEqual(await resource.getString(), expected, "Correct file content should be set"); +}); + +test("Escape characters (UTF-8, explicit parameter)", async (t) => { + t.plan(1); + + const input = `These are 人物 characters`; + const expected = "These are \\u4eba\\u7269 characters"; + const [resource] = await escape(input, { + encoding: "utf8" // escape encoding + }, "utf8"); // resource encoding + t.deepEqual(await resource.getString(), expected, "Correct file content should be set"); +}); + +test("Escape characters (UTF-8, default)", async (t) => { + t.plan(1); + + const input = `Viele Grüße`; + const expected = `Viele Gr\\u00fc\\u00dfe`; + // default encoding is utf8 + const [resource] = await escape(input); t.deepEqual(await resource.getString(), expected, "Correct file content should be set"); }); From 5fd3da329d78a428bc1f425f2459f3dcdbbe5ebf Mon Sep 17 00:00:00 2001 From: Tobias Sorn Date: Fri, 26 Jul 2019 11:29:40 +0200 Subject: [PATCH 23/27] Change property name for encoding propertiesFileEncoding -> propertiesSourceFileEncoding --- lib/lbt/utils/escapePropertiesFiles.js | 6 ++--- lib/types/application/ApplicationBuilder.js | 6 ++--- lib/types/application/ApplicationFormatter.js | 8 +++---- lib/types/library/LibraryBuilder.js | 6 ++--- lib/types/library/LibraryFormatter.js | 8 +++---- test/lib/builder/builder.js | 22 +++++++++---------- test/lib/lbt/bundle/AutoSplitter.js | 4 ++-- .../tasks/bundlers/generateManifestBundle.js | 2 +- .../bundlers/generateStandaloneAppBundle.js | 2 +- .../types/application/ApplicationFormatter.js | 2 +- test/lib/types/library/LibraryFormatter.js | 4 ++-- 11 files changed, 35 insertions(+), 35 deletions(-) diff --git a/lib/lbt/utils/escapePropertiesFiles.js b/lib/lbt/utils/escapePropertiesFiles.js index 871fa177e..82855ebd2 100644 --- a/lib/lbt/utils/escapePropertiesFiles.js +++ b/lib/lbt/utils/escapePropertiesFiles.js @@ -11,11 +11,11 @@ const nonAsciiEscaper = require("../../processors/nonAsciiEscaper"); * @returns {Promise} resolves with the escaped string content of the given Resource */ module.exports = async function(resource) { - const propertiesFileEncoding = resource.getProject() + const propertiesSourceFileEncoding = resource.getProject() && resource.getProject().resources && resource.getProject().resources.configuration - && resource.getProject().resources.configuration.propertiesFileEncoding; - const encoding = nonAsciiEscaper.getEncodingFromAlias(propertiesFileEncoding || "ISO-8859-1"); + && resource.getProject().resources.configuration.propertiesSourceFileEncoding; + const encoding = nonAsciiEscaper.getEncodingFromAlias(propertiesSourceFileEncoding || "ISO-8859-1"); await nonAsciiEscaper({ resources: [resource.resource], options: { diff --git a/lib/types/application/ApplicationBuilder.js b/lib/types/application/ApplicationBuilder.js index ea3e62437..dac819d16 100644 --- a/lib/types/application/ApplicationBuilder.js +++ b/lib/types/application/ApplicationBuilder.js @@ -28,13 +28,13 @@ class ApplicationBuilder extends AbstractBuilder { } this.addTask("escapeNonAsciiCharacters", () => { - const propertiesFileEncoding = project.resources + const propertiesSourceFileEncoding = project.resources && project.resources.configuration - && project.resources.configuration.propertiesFileEncoding; + && project.resources.configuration.propertiesSourceFileEncoding; return tasks.escapeNonAsciiCharacters({ workspace: resourceCollections.workspace, options: { - encoding: propertiesFileEncoding, + encoding: propertiesSourceFileEncoding, pattern: "/**/*.properties" } }); diff --git a/lib/types/application/ApplicationFormatter.js b/lib/types/application/ApplicationFormatter.js index 8e5fc8a10..49e0b79a7 100644 --- a/lib/types/application/ApplicationFormatter.js +++ b/lib/types/application/ApplicationFormatter.js @@ -134,12 +134,12 @@ class ApplicationFormatter extends AbstractUi5Formatter { } // default encoding to "ISO-8859-1" if not specified - if (!project.resources.configuration.propertiesFileEncoding) { - project.resources.configuration.propertiesFileEncoding = "ISO-8859-1"; + if (!project.resources.configuration.propertiesSourceFileEncoding) { + project.resources.configuration.propertiesSourceFileEncoding = "ISO-8859-1"; } - if (!["ISO-8859-1", "UTF-8"].includes(project.resources.configuration.propertiesFileEncoding)) { + if (!["ISO-8859-1", "UTF-8"].includes(project.resources.configuration.propertiesSourceFileEncoding)) { throw new Error(`Invalid properties file encoding specified for project ${project.id}: ` + - `encoding provided: ${project.resources.configuration.propertiesFileEncoding}. Must be either "ISO-8859-1" or "UTF-8".`); + `encoding provided: ${project.resources.configuration.propertiesSourceFileEncoding}. Must be either "ISO-8859-1" or "UTF-8".`); } const absolutePath = path.join(project.path, project.resources.configuration.paths.webapp); diff --git a/lib/types/library/LibraryBuilder.js b/lib/types/library/LibraryBuilder.js index 489fdf595..8834deda6 100644 --- a/lib/types/library/LibraryBuilder.js +++ b/lib/types/library/LibraryBuilder.js @@ -27,13 +27,13 @@ class LibraryBuilder extends AbstractBuilder { } this.addTask("escapeNonAsciiCharacters", () => { - const propertiesFileEncoding = project.resources + const propertiesSourceFileEncoding = project.resources && project.resources.configuration - && project.resources.configuration.propertiesFileEncoding; + && project.resources.configuration.propertiesSourceFileEncoding; return tasks.escapeNonAsciiCharacters({ workspace: resourceCollections.workspace, options: { - encoding: propertiesFileEncoding, + encoding: propertiesSourceFileEncoding, pattern: "/**/*.properties" } }); diff --git a/lib/types/library/LibraryFormatter.js b/lib/types/library/LibraryFormatter.js index 9091f32f2..7a0c1734c 100644 --- a/lib/types/library/LibraryFormatter.js +++ b/lib/types/library/LibraryFormatter.js @@ -341,12 +341,12 @@ class LibraryFormatter extends AbstractUi5Formatter { } // default encoding to "ISO-8859-1" if not specified - if (!project.resources.configuration.propertiesFileEncoding) { - project.resources.configuration.propertiesFileEncoding = "ISO-8859-1"; + if (!project.resources.configuration.propertiesSourceFileEncoding) { + project.resources.configuration.propertiesSourceFileEncoding = "ISO-8859-1"; } - if (!["ISO-8859-1", "UTF-8"].includes(project.resources.configuration.propertiesFileEncoding)) { + if (!["ISO-8859-1", "UTF-8"].includes(project.resources.configuration.propertiesSourceFileEncoding)) { throw new Error(`Invalid properties file encoding specified for project ${project.id}: ` + - `encoding provided: ${project.resources.configuration.propertiesFileEncoding}. Must be either "ISO-8859-1" or "UTF-8".`); + `encoding provided: ${project.resources.configuration.propertiesSourceFileEncoding}. Must be either "ISO-8859-1" or "UTF-8".`); } const absoluteSrcPath = path.join(project.path, project.resources.configuration.paths.src); diff --git a/test/lib/builder/builder.js b/test/lib/builder/builder.js index eeb898c36..fff0e3d9a 100644 --- a/test/lib/builder/builder.js +++ b/test/lib/builder/builder.js @@ -535,7 +535,7 @@ const applicationATree = { "paths": { "webapp": "webapp" }, - "propertiesFileEncoding": "ISO-8859-1" + "propertiesSourceFileEncoding": "ISO-8859-1" }, "pathMappings": { "/": "webapp" @@ -559,7 +559,7 @@ const applicationATreeBadType = { "paths": { "webapp": "webapp" }, - "propertiesFileEncoding": "ISO-8859-1" + "propertiesSourceFileEncoding": "ISO-8859-1" }, "pathMappings": { "/": "webapp" @@ -585,7 +585,7 @@ const applicationGTree = { "paths": { "webapp": "webapp" }, - "propertiesFileEncoding": "ISO-8859-1" + "propertiesSourceFileEncoding": "ISO-8859-1" }, "pathMappings": { "/": "webapp" @@ -620,7 +620,7 @@ const applicationGTreeWithExcludes = { "paths": { "webapp": "webapp" }, - "propertiesFileEncoding": "ISO-8859-1" + "propertiesSourceFileEncoding": "ISO-8859-1" }, "pathMappings": { "/": "webapp" @@ -656,7 +656,7 @@ const applicationGTreeComponentPreloadPaths = { "paths": { "webapp": "webapp" }, - "propertiesFileEncoding": "ISO-8859-1" + "propertiesSourceFileEncoding": "ISO-8859-1" }, "pathMappings": { "/": "webapp" @@ -688,7 +688,7 @@ const applicationHTree = { "paths": { "webapp": "webapp" }, - "propertiesFileEncoding": "ISO-8859-1" + "propertiesSourceFileEncoding": "ISO-8859-1" }, "pathMappings": { "/": "webapp" @@ -749,7 +749,7 @@ const applicationITree = { "paths": { "webapp": "webapp" }, - "propertiesFileEncoding": "ISO-8859-1" + "propertiesSourceFileEncoding": "ISO-8859-1" }, "pathMappings": { "/": "webapp" @@ -804,7 +804,7 @@ const libraryDTree = { "src": "main/src", "test": "main/test" }, - "propertiesFileEncoding": "ISO-8859-1" + "propertiesSourceFileEncoding": "ISO-8859-1" }, "pathMappings": { "/resources/": "main/src", @@ -857,7 +857,7 @@ const libraryETree = { "src": "src", "test": "test" }, - "propertiesFileEncoding": "ISO-8859-1" + "propertiesSourceFileEncoding": "ISO-8859-1" }, "pathMappings": { "/resources/": "src", @@ -910,7 +910,7 @@ const libraryHTree = { "src": "main/src", "test": "main/test" }, - "propertiesFileEncoding": "ISO-8859-1" + "propertiesSourceFileEncoding": "ISO-8859-1" }, "pathMappings": { "/resources/": "main/src", @@ -996,7 +996,7 @@ const libraryITree = { "src": "main/src", "test": "main/test" }, - "propertiesFileEncoding": "ISO-8859-1" + "propertiesSourceFileEncoding": "ISO-8859-1" }, "pathMappings": { "/resources/": "main/src" diff --git a/test/lib/lbt/bundle/AutoSplitter.js b/test/lib/lbt/bundle/AutoSplitter.js index 15fec1a24..162a6aa78 100644 --- a/test/lib/lbt/bundle/AutoSplitter.js +++ b/test/lib/lbt/bundle/AutoSplitter.js @@ -23,7 +23,7 @@ function createMockPool(dependencies) { return { "resources": { "configuration": { - "propertiesFileEncoding": "ISO-8859-1" + "propertiesSourceFileEncoding": "ISO-8859-1" } } }; @@ -213,7 +213,7 @@ test("_calcMinSize: properties resource", async (t) => { return { "resources": { "configuration": { - "propertiesFileEncoding": "ISO-8859-1" + "propertiesSourceFileEncoding": "ISO-8859-1" } } }; diff --git a/test/lib/tasks/bundlers/generateManifestBundle.js b/test/lib/tasks/bundlers/generateManifestBundle.js index 5f79bcd0c..4abc55015 100644 --- a/test/lib/tasks/bundlers/generateManifestBundle.js +++ b/test/lib/tasks/bundlers/generateManifestBundle.js @@ -190,7 +190,7 @@ const applicationBTree = { "paths": { "webapp": "webapp" }, - "propertiesFileEncoding": "ISO-8859-1" + "propertiesSourceFileEncoding": "ISO-8859-1" }, "pathMappings": { "/": "webapp" diff --git a/test/lib/tasks/bundlers/generateStandaloneAppBundle.js b/test/lib/tasks/bundlers/generateStandaloneAppBundle.js index 1d9e8b240..032e6903d 100644 --- a/test/lib/tasks/bundlers/generateStandaloneAppBundle.js +++ b/test/lib/tasks/bundlers/generateStandaloneAppBundle.js @@ -330,7 +330,7 @@ const applicationBTree = { "paths": { "webapp": "webapp" }, - "propertiesFileEncoding": "ISO-8859-1" + "propertiesSourceFileEncoding": "ISO-8859-1" }, "pathMappings": { "/": "webapp" diff --git a/test/lib/types/application/ApplicationFormatter.js b/test/lib/types/application/ApplicationFormatter.js index b12f81ee7..6b45060df 100644 --- a/test/lib/types/application/ApplicationFormatter.js +++ b/test/lib/types/application/ApplicationFormatter.js @@ -98,7 +98,7 @@ test("validate: empty resources", async (t) => { test("validate: test invalid encoding", async (t) => { const myProject = clone(applicationBTree); - myProject.resources.configuration.propertiesFileEncoding = "test"; + myProject.resources.configuration.propertiesSourceFileEncoding = "test"; const applicationFormatter = new ApplicationFormatter({project: myProject}); const error = await t.throwsAsync(applicationFormatter.validate(myProject)); diff --git a/test/lib/types/library/LibraryFormatter.js b/test/lib/types/library/LibraryFormatter.js index 6b82d341d..cc63e049d 100644 --- a/test/lib/types/library/LibraryFormatter.js +++ b/test/lib/types/library/LibraryFormatter.js @@ -110,12 +110,12 @@ test("validate: test directory does not exist", async (t) => { await libraryFormatter.validate(myProject); // Missing test directory is not an error t.deepEqual(myProject.resources.configuration.paths.test, null, "Project test path configuration is set to null"); - t.deepEqual(myProject.resources.configuration.propertiesFileEncoding, "ISO-8859-1", "Project resources propertiesFileEncoding is set to ISO-8859-1"); + t.deepEqual(myProject.resources.configuration.propertiesSourceFileEncoding, "ISO-8859-1", "Project resources propertiesSourceFileEncoding is set to ISO-8859-1"); }); test("validate: test invalid encoding", async (t) => { const myProject = clone(libraryETree); - myProject.resources.configuration.propertiesFileEncoding = "test"; + myProject.resources.configuration.propertiesSourceFileEncoding = "test"; const libraryFormatter = new LibraryFormatter({project: myProject}); const error = await t.throwsAsync(libraryFormatter.validate(myProject)); From 8d43a49b6468e1b18f48ddfbca3f851862489c62 Mon Sep 17 00:00:00 2001 From: Tobias Sorn Date: Fri, 26 Jul 2019 13:44:49 +0200 Subject: [PATCH 24/27] Change property name for encoding propertiesSourceFileEncoding -> propertiesFileSourceEncoding --- lib/lbt/utils/escapePropertiesFiles.js | 6 ++--- lib/types/application/ApplicationBuilder.js | 6 ++--- lib/types/application/ApplicationFormatter.js | 8 +++---- lib/types/library/LibraryBuilder.js | 6 ++--- lib/types/library/LibraryFormatter.js | 8 +++---- package-lock.json | 10 ++++----- test/lib/builder/builder.js | 22 +++++++++---------- test/lib/lbt/bundle/AutoSplitter.js | 4 ++-- .../tasks/bundlers/generateManifestBundle.js | 2 +- .../bundlers/generateStandaloneAppBundle.js | 2 +- .../types/application/ApplicationFormatter.js | 2 +- test/lib/types/library/LibraryFormatter.js | 4 ++-- 12 files changed, 40 insertions(+), 40 deletions(-) diff --git a/lib/lbt/utils/escapePropertiesFiles.js b/lib/lbt/utils/escapePropertiesFiles.js index 82855ebd2..3a8c183ff 100644 --- a/lib/lbt/utils/escapePropertiesFiles.js +++ b/lib/lbt/utils/escapePropertiesFiles.js @@ -11,11 +11,11 @@ const nonAsciiEscaper = require("../../processors/nonAsciiEscaper"); * @returns {Promise} resolves with the escaped string content of the given Resource */ module.exports = async function(resource) { - const propertiesSourceFileEncoding = resource.getProject() + const propertiesFileSourceEncoding = resource.getProject() && resource.getProject().resources && resource.getProject().resources.configuration - && resource.getProject().resources.configuration.propertiesSourceFileEncoding; - const encoding = nonAsciiEscaper.getEncodingFromAlias(propertiesSourceFileEncoding || "ISO-8859-1"); + && resource.getProject().resources.configuration.propertiesFileSourceEncoding; + const encoding = nonAsciiEscaper.getEncodingFromAlias(propertiesFileSourceEncoding || "ISO-8859-1"); await nonAsciiEscaper({ resources: [resource.resource], options: { diff --git a/lib/types/application/ApplicationBuilder.js b/lib/types/application/ApplicationBuilder.js index dac819d16..67043e786 100644 --- a/lib/types/application/ApplicationBuilder.js +++ b/lib/types/application/ApplicationBuilder.js @@ -28,13 +28,13 @@ class ApplicationBuilder extends AbstractBuilder { } this.addTask("escapeNonAsciiCharacters", () => { - const propertiesSourceFileEncoding = project.resources + const propertiesFileSourceEncoding = project.resources && project.resources.configuration - && project.resources.configuration.propertiesSourceFileEncoding; + && project.resources.configuration.propertiesFileSourceEncoding; return tasks.escapeNonAsciiCharacters({ workspace: resourceCollections.workspace, options: { - encoding: propertiesSourceFileEncoding, + encoding: propertiesFileSourceEncoding, pattern: "/**/*.properties" } }); diff --git a/lib/types/application/ApplicationFormatter.js b/lib/types/application/ApplicationFormatter.js index 49e0b79a7..2a3fb0b3e 100644 --- a/lib/types/application/ApplicationFormatter.js +++ b/lib/types/application/ApplicationFormatter.js @@ -134,12 +134,12 @@ class ApplicationFormatter extends AbstractUi5Formatter { } // default encoding to "ISO-8859-1" if not specified - if (!project.resources.configuration.propertiesSourceFileEncoding) { - project.resources.configuration.propertiesSourceFileEncoding = "ISO-8859-1"; + if (!project.resources.configuration.propertiesFileSourceEncoding) { + project.resources.configuration.propertiesFileSourceEncoding = "ISO-8859-1"; } - if (!["ISO-8859-1", "UTF-8"].includes(project.resources.configuration.propertiesSourceFileEncoding)) { + if (!["ISO-8859-1", "UTF-8"].includes(project.resources.configuration.propertiesFileSourceEncoding)) { throw new Error(`Invalid properties file encoding specified for project ${project.id}: ` + - `encoding provided: ${project.resources.configuration.propertiesSourceFileEncoding}. Must be either "ISO-8859-1" or "UTF-8".`); + `encoding provided: ${project.resources.configuration.propertiesFileSourceEncoding}. Must be either "ISO-8859-1" or "UTF-8".`); } const absolutePath = path.join(project.path, project.resources.configuration.paths.webapp); diff --git a/lib/types/library/LibraryBuilder.js b/lib/types/library/LibraryBuilder.js index 8834deda6..b1125b794 100644 --- a/lib/types/library/LibraryBuilder.js +++ b/lib/types/library/LibraryBuilder.js @@ -27,13 +27,13 @@ class LibraryBuilder extends AbstractBuilder { } this.addTask("escapeNonAsciiCharacters", () => { - const propertiesSourceFileEncoding = project.resources + const propertiesFileSourceEncoding = project.resources && project.resources.configuration - && project.resources.configuration.propertiesSourceFileEncoding; + && project.resources.configuration.propertiesFileSourceEncoding; return tasks.escapeNonAsciiCharacters({ workspace: resourceCollections.workspace, options: { - encoding: propertiesSourceFileEncoding, + encoding: propertiesFileSourceEncoding, pattern: "/**/*.properties" } }); diff --git a/lib/types/library/LibraryFormatter.js b/lib/types/library/LibraryFormatter.js index 7a0c1734c..7339d7184 100644 --- a/lib/types/library/LibraryFormatter.js +++ b/lib/types/library/LibraryFormatter.js @@ -341,12 +341,12 @@ class LibraryFormatter extends AbstractUi5Formatter { } // default encoding to "ISO-8859-1" if not specified - if (!project.resources.configuration.propertiesSourceFileEncoding) { - project.resources.configuration.propertiesSourceFileEncoding = "ISO-8859-1"; + if (!project.resources.configuration.propertiesFileSourceEncoding) { + project.resources.configuration.propertiesFileSourceEncoding = "ISO-8859-1"; } - if (!["ISO-8859-1", "UTF-8"].includes(project.resources.configuration.propertiesSourceFileEncoding)) { + if (!["ISO-8859-1", "UTF-8"].includes(project.resources.configuration.propertiesFileSourceEncoding)) { throw new Error(`Invalid properties file encoding specified for project ${project.id}: ` + - `encoding provided: ${project.resources.configuration.propertiesSourceFileEncoding}. Must be either "ISO-8859-1" or "UTF-8".`); + `encoding provided: ${project.resources.configuration.propertiesFileSourceEncoding}. Must be either "ISO-8859-1" or "UTF-8".`); } const absoluteSrcPath = path.join(project.path, project.resources.configuration.paths.src); diff --git a/package-lock.json b/package-lock.json index 4ec9425eb..1b7bf02da 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4407,7 +4407,7 @@ }, "find-up": { "version": "3.0.0", - "resolved": "https://registry.npmjs.org/find-up/-/find-up-3.0.0.tgz", + "resolved": false, "integrity": "sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg==", "dev": true, "requires": { @@ -4416,7 +4416,7 @@ }, "locate-path": { "version": "3.0.0", - "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-3.0.0.tgz", + "resolved": false, "integrity": "sha512-7AO748wWnIhNqAuaty2ZWHkQHRSNfPVIsPIfwEOWO22AmaoVrWavlOcMR5nzTLNYvp36X220/maaRsrec1G65A==", "dev": true, "requires": { @@ -4436,7 +4436,7 @@ }, "p-locate": { "version": "3.0.0", - "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-3.0.0.tgz", + "resolved": false, "integrity": "sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ==", "dev": true, "requires": { @@ -4445,13 +4445,13 @@ }, "path-exists": { "version": "3.0.0", - "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz", + "resolved": false, "integrity": "sha1-zg6+ql94yxiSXqfYENe1mwEP1RU=", "dev": true }, "resolve-from": { "version": "4.0.0", - "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", + "resolved": false, "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", "dev": true }, diff --git a/test/lib/builder/builder.js b/test/lib/builder/builder.js index fff0e3d9a..efc48ea0e 100644 --- a/test/lib/builder/builder.js +++ b/test/lib/builder/builder.js @@ -535,7 +535,7 @@ const applicationATree = { "paths": { "webapp": "webapp" }, - "propertiesSourceFileEncoding": "ISO-8859-1" + "propertiesFileSourceEncoding": "ISO-8859-1" }, "pathMappings": { "/": "webapp" @@ -559,7 +559,7 @@ const applicationATreeBadType = { "paths": { "webapp": "webapp" }, - "propertiesSourceFileEncoding": "ISO-8859-1" + "propertiesFileSourceEncoding": "ISO-8859-1" }, "pathMappings": { "/": "webapp" @@ -585,7 +585,7 @@ const applicationGTree = { "paths": { "webapp": "webapp" }, - "propertiesSourceFileEncoding": "ISO-8859-1" + "propertiesFileSourceEncoding": "ISO-8859-1" }, "pathMappings": { "/": "webapp" @@ -620,7 +620,7 @@ const applicationGTreeWithExcludes = { "paths": { "webapp": "webapp" }, - "propertiesSourceFileEncoding": "ISO-8859-1" + "propertiesFileSourceEncoding": "ISO-8859-1" }, "pathMappings": { "/": "webapp" @@ -656,7 +656,7 @@ const applicationGTreeComponentPreloadPaths = { "paths": { "webapp": "webapp" }, - "propertiesSourceFileEncoding": "ISO-8859-1" + "propertiesFileSourceEncoding": "ISO-8859-1" }, "pathMappings": { "/": "webapp" @@ -688,7 +688,7 @@ const applicationHTree = { "paths": { "webapp": "webapp" }, - "propertiesSourceFileEncoding": "ISO-8859-1" + "propertiesFileSourceEncoding": "ISO-8859-1" }, "pathMappings": { "/": "webapp" @@ -749,7 +749,7 @@ const applicationITree = { "paths": { "webapp": "webapp" }, - "propertiesSourceFileEncoding": "ISO-8859-1" + "propertiesFileSourceEncoding": "ISO-8859-1" }, "pathMappings": { "/": "webapp" @@ -804,7 +804,7 @@ const libraryDTree = { "src": "main/src", "test": "main/test" }, - "propertiesSourceFileEncoding": "ISO-8859-1" + "propertiesFileSourceEncoding": "ISO-8859-1" }, "pathMappings": { "/resources/": "main/src", @@ -857,7 +857,7 @@ const libraryETree = { "src": "src", "test": "test" }, - "propertiesSourceFileEncoding": "ISO-8859-1" + "propertiesFileSourceEncoding": "ISO-8859-1" }, "pathMappings": { "/resources/": "src", @@ -910,7 +910,7 @@ const libraryHTree = { "src": "main/src", "test": "main/test" }, - "propertiesSourceFileEncoding": "ISO-8859-1" + "propertiesFileSourceEncoding": "ISO-8859-1" }, "pathMappings": { "/resources/": "main/src", @@ -996,7 +996,7 @@ const libraryITree = { "src": "main/src", "test": "main/test" }, - "propertiesSourceFileEncoding": "ISO-8859-1" + "propertiesFileSourceEncoding": "ISO-8859-1" }, "pathMappings": { "/resources/": "main/src" diff --git a/test/lib/lbt/bundle/AutoSplitter.js b/test/lib/lbt/bundle/AutoSplitter.js index 162a6aa78..d0d285213 100644 --- a/test/lib/lbt/bundle/AutoSplitter.js +++ b/test/lib/lbt/bundle/AutoSplitter.js @@ -23,7 +23,7 @@ function createMockPool(dependencies) { return { "resources": { "configuration": { - "propertiesSourceFileEncoding": "ISO-8859-1" + "propertiesFileSourceEncoding": "ISO-8859-1" } } }; @@ -213,7 +213,7 @@ test("_calcMinSize: properties resource", async (t) => { return { "resources": { "configuration": { - "propertiesSourceFileEncoding": "ISO-8859-1" + "propertiesFileSourceEncoding": "ISO-8859-1" } } }; diff --git a/test/lib/tasks/bundlers/generateManifestBundle.js b/test/lib/tasks/bundlers/generateManifestBundle.js index 4abc55015..8ec5d5938 100644 --- a/test/lib/tasks/bundlers/generateManifestBundle.js +++ b/test/lib/tasks/bundlers/generateManifestBundle.js @@ -190,7 +190,7 @@ const applicationBTree = { "paths": { "webapp": "webapp" }, - "propertiesSourceFileEncoding": "ISO-8859-1" + "propertiesFileSourceEncoding": "ISO-8859-1" }, "pathMappings": { "/": "webapp" diff --git a/test/lib/tasks/bundlers/generateStandaloneAppBundle.js b/test/lib/tasks/bundlers/generateStandaloneAppBundle.js index 032e6903d..9415cb129 100644 --- a/test/lib/tasks/bundlers/generateStandaloneAppBundle.js +++ b/test/lib/tasks/bundlers/generateStandaloneAppBundle.js @@ -330,7 +330,7 @@ const applicationBTree = { "paths": { "webapp": "webapp" }, - "propertiesSourceFileEncoding": "ISO-8859-1" + "propertiesFileSourceEncoding": "ISO-8859-1" }, "pathMappings": { "/": "webapp" diff --git a/test/lib/types/application/ApplicationFormatter.js b/test/lib/types/application/ApplicationFormatter.js index 6b45060df..0daddfac4 100644 --- a/test/lib/types/application/ApplicationFormatter.js +++ b/test/lib/types/application/ApplicationFormatter.js @@ -98,7 +98,7 @@ test("validate: empty resources", async (t) => { test("validate: test invalid encoding", async (t) => { const myProject = clone(applicationBTree); - myProject.resources.configuration.propertiesSourceFileEncoding = "test"; + myProject.resources.configuration.propertiesFileSourceEncoding = "test"; const applicationFormatter = new ApplicationFormatter({project: myProject}); const error = await t.throwsAsync(applicationFormatter.validate(myProject)); diff --git a/test/lib/types/library/LibraryFormatter.js b/test/lib/types/library/LibraryFormatter.js index cc63e049d..356d0cc97 100644 --- a/test/lib/types/library/LibraryFormatter.js +++ b/test/lib/types/library/LibraryFormatter.js @@ -110,12 +110,12 @@ test("validate: test directory does not exist", async (t) => { await libraryFormatter.validate(myProject); // Missing test directory is not an error t.deepEqual(myProject.resources.configuration.paths.test, null, "Project test path configuration is set to null"); - t.deepEqual(myProject.resources.configuration.propertiesSourceFileEncoding, "ISO-8859-1", "Project resources propertiesSourceFileEncoding is set to ISO-8859-1"); + t.deepEqual(myProject.resources.configuration.propertiesFileSourceEncoding, "ISO-8859-1", "Project resources propertiesFileSourceEncoding is set to ISO-8859-1"); }); test("validate: test invalid encoding", async (t) => { const myProject = clone(libraryETree); - myProject.resources.configuration.propertiesSourceFileEncoding = "test"; + myProject.resources.configuration.propertiesFileSourceEncoding = "test"; const libraryFormatter = new LibraryFormatter({project: myProject}); const error = await t.throwsAsync(libraryFormatter.validate(myProject)); From 6b917b4328cd328960c8d2325765721d75c6289d Mon Sep 17 00:00:00 2001 From: Tobias Sorn Date: Fri, 26 Jul 2019 14:00:38 +0200 Subject: [PATCH 25/27] change name of escapePropertiesFile --- lib/lbt/bundle/AutoSplitter.js | 4 ++-- lib/lbt/bundle/Builder.js | 4 ++-- .../{escapePropertiesFiles.js => escapePropertiesFile.js} | 2 +- lib/tasks/escapeNonAsciiCharacters.js | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) rename lib/lbt/utils/{escapePropertiesFiles.js => escapePropertiesFile.js} (96%) diff --git a/lib/lbt/bundle/AutoSplitter.js b/lib/lbt/bundle/AutoSplitter.js index e24d2e8ef..20c12a065 100644 --- a/lib/lbt/bundle/AutoSplitter.js +++ b/lib/lbt/bundle/AutoSplitter.js @@ -5,7 +5,7 @@ const {pd} = require("pretty-data"); const ModuleName = require("../utils/ModuleName"); const {SectionType} = require("./BundleDefinition"); -const escapePropertiesFiles = require("../utils/escapePropertiesFiles"); +const escapePropertiesFile = require("../utils/escapePropertiesFile"); const log = require("@ui5/logger").getLogger("lbt:bundle:AutoSplitter"); const copyrightCommentsPattern = /copyright|\(c\)(?:[0-9]+|\s+[0-9A-za-z])|released under|license|\u00a9/i; @@ -230,7 +230,7 @@ class AutoSplitter { // Since AutoSplitter is also used when splitting non-project resources (e.g. dependencies) // *.properties files should be escaped if encoding option is specified - const fileContent = await escapePropertiesFiles(resource); + const fileContent = await escapePropertiesFile(resource); return fileContent.length; } else if ( this.optimizeXMLViews && /\.view.xml$/.test(module) ) { diff --git a/lib/lbt/bundle/Builder.js b/lib/lbt/bundle/Builder.js index 211cc5769..74bc73f73 100644 --- a/lib/lbt/bundle/Builder.js +++ b/lib/lbt/bundle/Builder.js @@ -12,7 +12,7 @@ const {Syntax} = esprima; const {isMethodCall} = require("../utils/ASTUtils"); const ModuleName = require("../utils/ModuleName"); const UI5ClientConstants = require("../UI5ClientConstants"); -const escapePropertiesFiles = require("../utils/escapePropertiesFiles"); +const escapePropertiesFile = require("../utils/escapePropertiesFile"); const BundleResolver = require("./Resolver"); const BundleSplitter = require("./AutoSplitter"); @@ -445,7 +445,7 @@ class BundleBuilder { } else if ( /\.properties$/.test(module) ) { // Since the Builder is also used when building non-project resources (e.g. dependencies) // *.properties files should be escaped if encoding option is specified - const fileContent = await escapePropertiesFiles(resource); + const fileContent = await escapePropertiesFile(resource); outW.write( makeStringLiteral( fileContent ) ); } else { diff --git a/lib/lbt/utils/escapePropertiesFiles.js b/lib/lbt/utils/escapePropertiesFile.js similarity index 96% rename from lib/lbt/utils/escapePropertiesFiles.js rename to lib/lbt/utils/escapePropertiesFile.js index 3a8c183ff..3e8158398 100644 --- a/lib/lbt/utils/escapePropertiesFiles.js +++ b/lib/lbt/utils/escapePropertiesFile.js @@ -25,5 +25,5 @@ module.exports = async function(resource) { const fileContent = await resource.buffer(); - return fileContent.toString(encoding); + return fileContent.toString(); }; diff --git a/lib/tasks/escapeNonAsciiCharacters.js b/lib/tasks/escapeNonAsciiCharacters.js index b755e1d37..5acb377b6 100644 --- a/lib/tasks/escapeNonAsciiCharacters.js +++ b/lib/tasks/escapeNonAsciiCharacters.js @@ -4,7 +4,7 @@ const nonAsciiEscaper = require("../processors/nonAsciiEscaper"); * Task to escape non ascii characters in properties files resources. * * @public - * @alias module:@ui5/builder.tasks.escapePropertiesFiles + * @alias module:@ui5/builder.tasks.escapeNonAsciiCharacters * @param {Object} parameters Parameters * @param {module:@ui5/fs.DuplexCollection} parameters.workspace DuplexCollection to read and write files * @param {Object} parameters.options Options From 30ada8b825f64617837f108cd69b26c3c4e2d8ee Mon Sep 17 00:00:00 2001 From: Tobias Sorn Date: Mon, 29 Jul 2019 14:39:45 +0200 Subject: [PATCH 26/27] adjust test for builder and formatters Separated default encoding tests --- test/lib/builder/builder.js | 6 +++--- test/lib/types/application/ApplicationFormatter.js | 9 +++++++++ test/lib/types/library/LibraryFormatter.js | 10 +++++++++- 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/test/lib/builder/builder.js b/test/lib/builder/builder.js index efc48ea0e..91292d72e 100644 --- a/test/lib/builder/builder.js +++ b/test/lib/builder/builder.js @@ -342,8 +342,7 @@ test("Build theme.j even without an library", (t) => { const expectedPath = "./test/expected/build/theme.j/dest"; return builder.build({ tree: themeJTree, - destPath, - excludedTasks: ["escapeNonAsciiCharacters"] + destPath }).then(() => { return findFiles(expectedPath); }).then((expectedFiles) => { @@ -1047,7 +1046,8 @@ const themeJTree = { "paths": { "src": "main/src", "test": "main/test" - } + }, + "propertiesFileSourceEncoding": "ISO-8859-1" }, "pathMappings": { "/resources/": "main/src" diff --git a/test/lib/types/application/ApplicationFormatter.js b/test/lib/types/application/ApplicationFormatter.js index 0daddfac4..67919da95 100644 --- a/test/lib/types/application/ApplicationFormatter.js +++ b/test/lib/types/application/ApplicationFormatter.js @@ -96,6 +96,15 @@ test("validate: empty resources", async (t) => { t.deepEqual(myProject.resources.configuration.paths.webapp, "webapp", "default webapp directory is set"); }); +test("validate: empty encoding", async (t) => { + const myProject = clone(applicationBTree); + delete myProject.resources.configuration.propertiesFileSourceEncoding; + const applicationFormatter = new ApplicationFormatter({project: myProject}); + + await applicationFormatter.validate(myProject); + t.deepEqual(myProject.resources.configuration.propertiesFileSourceEncoding, "ISO-8859-1", "default resources encoding is set"); +}); + test("validate: test invalid encoding", async (t) => { const myProject = clone(applicationBTree); myProject.resources.configuration.propertiesFileSourceEncoding = "test"; diff --git a/test/lib/types/library/LibraryFormatter.js b/test/lib/types/library/LibraryFormatter.js index 356d0cc97..8a2ec4ab9 100644 --- a/test/lib/types/library/LibraryFormatter.js +++ b/test/lib/types/library/LibraryFormatter.js @@ -88,6 +88,15 @@ test("validate: empty resources", async (t) => { t.deepEqual(myProject.resources.configuration.paths.test, "test", "default test directory is set"); }); +test("validate: empty encoding", async (t) => { + const myProject = clone(libraryETree); + delete myProject.resources.configuration.propertiesFileSourceEncoding; + const libraryFormatter = new LibraryFormatter({project: myProject}); + + await libraryFormatter.validate(myProject); + t.deepEqual(myProject.resources.configuration.propertiesFileSourceEncoding, "ISO-8859-1", "default resources encoding is set"); +}); + test("validate: src directory does not exist", async (t) => { const myProject = clone(libraryETree); const libraryFormatter = new LibraryFormatter({project: myProject}); @@ -110,7 +119,6 @@ test("validate: test directory does not exist", async (t) => { await libraryFormatter.validate(myProject); // Missing test directory is not an error t.deepEqual(myProject.resources.configuration.paths.test, null, "Project test path configuration is set to null"); - t.deepEqual(myProject.resources.configuration.propertiesFileSourceEncoding, "ISO-8859-1", "Project resources propertiesFileSourceEncoding is set to ISO-8859-1"); }); test("validate: test invalid encoding", async (t) => { From 46a7742453820815623791864f4631ad5b444c6b Mon Sep 17 00:00:00 2001 From: Tobias Sorn Date: Mon, 29 Jul 2019 15:04:56 +0200 Subject: [PATCH 27/27] remove duplicate await --- test/lib/types/library/LibraryFormatter.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/lib/types/library/LibraryFormatter.js b/test/lib/types/library/LibraryFormatter.js index 8a2ec4ab9..5d13f6ebc 100644 --- a/test/lib/types/library/LibraryFormatter.js +++ b/test/lib/types/library/LibraryFormatter.js @@ -104,7 +104,7 @@ test("validate: src directory does not exist", async (t) => { dirExists.onFirstCall().resolves(false); dirExists.onSecondCall().resolves(true); - const error = await await t.throwsAsync(libraryFormatter.validate(myProject)); + const error = await t.throwsAsync(libraryFormatter.validate(myProject)); t.regex(error.message, /^Could not find source directory of project library\.e\.id: (?!(undefined))+/, "Missing source directory caused error"); });