From a1a7dc30ae053a1544af895e247ea70522e885d9 Mon Sep 17 00:00:00 2001 From: Jakub Kosmydel <104823336+kosmydel@users.noreply.github.com> Date: Thu, 14 Sep 2023 08:27:00 +0200 Subject: [PATCH 1/7] Handle invisible characters in forms --- src/CONST.ts | 2 + src/components/Form.js | 40 +++++-- src/libs/ValidationUtils.js | 3 +- src/libs/isEmptyString.ts | 13 +++ src/libs/removeInvisibleCharacters.ts | 26 +++++ src/pages/workspace/WorkspaceSettingsPage.js | 3 +- tests/unit/isEmptyString.js | 89 ++++++++++++++ tests/unit/removeInvisibleCharacters.js | 116 +++++++++++++++++++ 8 files changed, 278 insertions(+), 14 deletions(-) create mode 100644 src/libs/isEmptyString.ts create mode 100644 src/libs/removeInvisibleCharacters.ts create mode 100644 tests/unit/isEmptyString.js create mode 100644 tests/unit/removeInvisibleCharacters.js diff --git a/src/CONST.ts b/src/CONST.ts index 1ef2f3e83246..6654d35e4464 100755 --- a/src/CONST.ts +++ b/src/CONST.ts @@ -1245,6 +1245,8 @@ const CONST = { DATE_TIME_FORMAT: /^\d{2}-\d{2} \d{2}:\d{2} [AP]M$/, ATTACHMENT_ROUTE: /\/r\/(\d*)\/attachment/, ILLEGAL_FILENAME_CHARACTERS: /\/|<|>|\*|"|:|\?|\\|\|/g, + + INVISIBLE_CHARACTERS: /[\p{C}\p{Z}]/gu, }, PRONOUNS: { diff --git a/src/components/Form.js b/src/components/Form.js index ef6c3ea10474..2eaf22b00e85 100644 --- a/src/components/Form.js +++ b/src/components/Form.js @@ -18,6 +18,7 @@ import stylePropTypes from '../styles/stylePropTypes'; import {withNetwork} from './OnyxProvider'; import networkPropTypes from './networkPropTypes'; import Visibility from '../libs/Visibility'; +import removeInvisibleCharacters from '../libs/removeInvisibleCharacters'; const propTypes = { /** A unique Onyx key identifying the form */ @@ -115,20 +116,32 @@ function Form(props) { const hasServerError = useMemo(() => Boolean(props.formState) && !_.isEmpty(props.formState.errors), [props.formState]); + /** + * This function is used to remove invisible characters from strings before validation and submission. + * + * @param {Object} values - An object containing the value of each inputID, e.g. {inputID1: value1, inputID2: value2} + * @returns {Object} - An object containing the processed values of each inputID + */ + const prepareValues = useCallback((values) => { + const trimmedStringValues = {}; + _.each(values, (inputValue, inputID) => { + if (_.isString(inputValue)) { + trimmedStringValues[inputID] = removeInvisibleCharacters(inputValue); + } else { + trimmedStringValues[inputID] = inputValue; + } + }); + return trimmedStringValues; + }, []); + /** * @param {Object} values - An object containing the value of each inputID, e.g. {inputID1: value1, inputID2: value2} * @returns {Object} - An object containing the errors for each inputID, e.g. {inputID1: error1, inputID2: error2} */ const onValidate = useCallback( (values, shouldClearServerError = true) => { - const trimmedStringValues = {}; - _.each(values, (inputValue, inputID) => { - if (_.isString(inputValue)) { - trimmedStringValues[inputID] = inputValue.trim(); - } else { - trimmedStringValues[inputID] = inputValue; - } - }); + // Trim all string values + const trimmedStringValues = prepareValues(values); if (shouldClearServerError) { FormActions.setErrors(props.formID, null); @@ -186,7 +199,7 @@ function Form(props) { return touchedInputErrors; }, - [errors, touchedInputs, props.formID, validate], + [prepareValues, props.formID, validate, errors], ); useEffect(() => { @@ -223,11 +236,14 @@ function Form(props) { return; } + // Trim all string values + const trimmedStringValues = prepareValues(inputValues); + // Touches all form inputs so we can validate the entire form _.each(inputRefs.current, (inputRef, inputID) => (touchedInputs.current[inputID] = true)); // Validate form and return early if any errors are found - if (!_.isEmpty(onValidate(inputValues))) { + if (!_.isEmpty(onValidate(trimmedStringValues))) { return; } @@ -237,8 +253,8 @@ function Form(props) { } // Call submit handler - onSubmit(inputValues); - }, [props.formState, onSubmit, inputRefs, inputValues, onValidate, touchedInputs, props.network.isOffline, props.enabledWhenOffline]); + onSubmit(trimmedStringValues); + }, [props.formState.isLoading, props.network.isOffline, props.enabledWhenOffline, prepareValues, inputValues, onValidate, onSubmit]); /** * Loops over Form's children and automatically supplies Form props to them diff --git a/src/libs/ValidationUtils.js b/src/libs/ValidationUtils.js index 81b91e2101be..16d6681bb7d6 100644 --- a/src/libs/ValidationUtils.js +++ b/src/libs/ValidationUtils.js @@ -5,6 +5,7 @@ import {parsePhoneNumber} from 'awesome-phonenumber'; import CONST from '../CONST'; import * as CardUtils from './CardUtils'; import * as LoginUtils from './LoginUtils'; +import isEmptyString from './isEmptyString'; /** * Implements the Luhn Algorithm, a checksum formula used to validate credit card @@ -84,7 +85,7 @@ function isValidPastDate(date) { */ function isRequiredFulfilled(value) { if (_.isString(value)) { - return !_.isEmpty(value.trim()); + return !isEmptyString(value); } if (_.isDate(value)) { return isValidDate(value); diff --git a/src/libs/isEmptyString.ts b/src/libs/isEmptyString.ts new file mode 100644 index 000000000000..2cc28880b916 --- /dev/null +++ b/src/libs/isEmptyString.ts @@ -0,0 +1,13 @@ +import CONST from '../CONST'; + +/** + * Checks if the string would be empty if all invisible characters were removed. + */ +function isEmptyString(value: string): boolean { + // \p{C} matches all 'Other' characters + // \p{Z} matches all separators (spaces etc.) + // Source: http://www.unicode.org/reports/tr18/#General_Category_Property + return value.replace(CONST.REGEX.INVISIBLE_CHARACTERS, '') === ''; +} + +export default isEmptyString; diff --git a/src/libs/removeInvisibleCharacters.ts b/src/libs/removeInvisibleCharacters.ts new file mode 100644 index 000000000000..1096223a25ab --- /dev/null +++ b/src/libs/removeInvisibleCharacters.ts @@ -0,0 +1,26 @@ +/** + * Remove invisible characters from a string except for spaces and format characters for emoji, and trim it. + */ +function removeInvisibleCharacters(value: string): string { + let result = value; + + // Remove spaces: + // - \u200B: zero-width space + // - \u00A0: non-breaking space + // - \u2060: word joiner + result = result.replace(/[\u200B\u00A0\u2060]/g, ''); + + // Remove all characters from the 'Other' (C) category except for format characters (Cf) + // because some of them they are used for emojis + result = result.replace(/[\p{Cc}\p{Cs}\p{Co}\p{Cn}]/gu, ''); + + // Remove characters from the (Cf) category that are not used for emojis + result = result.replace(/[\u200E-\u200F]/g, ''); + + // Remove all characters from the 'Separator' (Z) category except for Space Separator (Zs) + result = result.replace(/[\p{Zl}\p{Zp}]/gu, ''); + + return result.trim(); +} + +export default removeInvisibleCharacters; diff --git a/src/pages/workspace/WorkspaceSettingsPage.js b/src/pages/workspace/WorkspaceSettingsPage.js index 7aff9093c4dd..f9451264b00e 100644 --- a/src/pages/workspace/WorkspaceSettingsPage.js +++ b/src/pages/workspace/WorkspaceSettingsPage.js @@ -24,6 +24,7 @@ import Avatar from '../../components/Avatar'; import Navigation from '../../libs/Navigation/Navigation'; import ROUTES from '../../ROUTES'; import withWindowDimensions, {windowDimensionsPropTypes} from '../../components/withWindowDimensions'; +import * as ValidationUtils from '../../libs/ValidationUtils'; const propTypes = { // The currency list constant object from Onyx @@ -69,7 +70,7 @@ function WorkspaceSettingsPage(props) { const errors = {}; const name = values.name.trim(); - if (!name || !name.length) { + if (!ValidationUtils.isRequiredFulfilled(name)) { errors.name = 'workspace.editor.nameIsRequiredError'; } else if ([...name].length > CONST.WORKSPACE_NAME_CHARACTER_LIMIT) { // Uses the spread syntax to count the number of Unicode code points instead of the number of UTF-16 diff --git a/tests/unit/isEmptyString.js b/tests/unit/isEmptyString.js new file mode 100644 index 000000000000..bd905ea2e173 --- /dev/null +++ b/tests/unit/isEmptyString.js @@ -0,0 +1,89 @@ +import _ from 'underscore'; +import isEmpty from '../../src/libs/isEmptyString'; +import enEmojis from '../../assets/emojis/en'; + +describe('libs/isEmpty', () => { + it('basic tests', () => { + expect(isEmpty('test')).toBe(false); + expect(isEmpty('test test')).toBe(false); + expect(isEmpty('test test test')).toBe(false); + expect(isEmpty(' ')).toBe(true); + }); + it('trim spaces', () => { + expect(isEmpty(' test')).toBe(false); + expect(isEmpty('test ')).toBe(false); + expect(isEmpty(' test ')).toBe(false); + }); + it('remove invisible characters', () => { + expect(isEmpty('\u200B')).toBe(true); + expect(isEmpty('\u200B')).toBe(true); + expect(isEmpty('\u200B ')).toBe(true); + expect(isEmpty('\u200B \u200B')).toBe(true); + expect(isEmpty('\u200B \u200B ')).toBe(true); + }); + it('remove invisible characters (Cc)', () => { + expect(isEmpty('\u0000')).toBe(true); + expect(isEmpty('\u0001')).toBe(true); + expect(isEmpty('\u0009')).toBe(true); + }); + it('remove invisible characters (Cf)', () => { + expect(isEmpty('\u200E')).toBe(true); + expect(isEmpty('\u200F')).toBe(true); + expect(isEmpty('\u2060')).toBe(true); + }); + it('remove invisible characters (Cs)', () => { + expect(isEmpty('\uD800')).toBe(true); + expect(isEmpty('\uD801')).toBe(true); + expect(isEmpty('\uD802')).toBe(true); + }); + it('remove invisible characters (Co)', () => { + expect(isEmpty('\uE000')).toBe(true); + expect(isEmpty('\uE001')).toBe(true); + expect(isEmpty('\uE002')).toBe(true); + }); + it('remove invisible characters (Zl)', () => { + expect(isEmpty('\u2028')).toBe(true); + expect(isEmpty('\u2029')).toBe(true); + expect(isEmpty('\u202A')).toBe(true); + }); + it('basic check emojis not removed', () => { + expect(isEmpty('😀')).toBe(false); + }); + it('all emojis not removed', () => { + _.keys(enEmojis).forEach((key) => { + expect(isEmpty(key)).toBe(false); + }); + }); + it('remove invisible characters (editpad)', () => { + expect(isEmpty('\u0020')).toBe(true); + expect(isEmpty('\u00A0')).toBe(true); + expect(isEmpty('\u2000')).toBe(true); + expect(isEmpty('\u2001')).toBe(true); + expect(isEmpty('\u2002')).toBe(true); + expect(isEmpty('\u2003')).toBe(true); + expect(isEmpty('\u2004')).toBe(true); + expect(isEmpty('\u2005')).toBe(true); + expect(isEmpty('\u2006')).toBe(true); + expect(isEmpty('\u2007')).toBe(true); + expect(isEmpty('\u2008')).toBe(true); + expect(isEmpty('\u2009')).toBe(true); + expect(isEmpty('\u200A')).toBe(true); + expect(isEmpty('\u2028')).toBe(true); + expect(isEmpty('\u205F')).toBe(true); + expect(isEmpty('\u3000')).toBe(true); + expect(isEmpty(' ')).toBe(true); + }); + it('other tests', () => { + expect(isEmpty('\u200D')).toBe(true); + expect(isEmpty('\uD83C\uDFF4\uDB40\uDC67\uDB40\uDC62\uDB40\uDC65\uDB40\uDC6E\uDB40\uDC67\uDB40\uDC7F')).toBe(false); + expect(isEmpty('\uD83C')).toBe(true); + expect(isEmpty('\uDFF4')).toBe(true); + expect(isEmpty('\uDB40')).toBe(true); + expect(isEmpty('\uDC67')).toBe(true); + expect(isEmpty('\uDC62')).toBe(true); + expect(isEmpty('\uDC65')).toBe(true); + expect(isEmpty('\uDC6E')).toBe(true); + expect(isEmpty('\uDC67')).toBe(true); + expect(isEmpty('\uDC7F')).toBe(true); + }); +}); diff --git a/tests/unit/removeInvisibleCharacters.js b/tests/unit/removeInvisibleCharacters.js new file mode 100644 index 000000000000..dd6c8867a04d --- /dev/null +++ b/tests/unit/removeInvisibleCharacters.js @@ -0,0 +1,116 @@ +import _ from 'underscore'; +import removeInvisible from '../../src/libs/removeInvisibleCharacters'; +import enEmojis from '../../assets/emojis/en'; + +describe('libs/removeInvisible', () => { + it('basic tests', () => { + expect(removeInvisible('test')).toBe('test'); + expect(removeInvisible('test test')).toBe('test test'); + expect(removeInvisible('abcdefghijklmnopqrstuvwxyz')).toBe('abcdefghijklmnopqrstuvwxyz'); + expect(removeInvisible('ABCDEFGHIJKLMNOPQRSTUVWXYZ')).toBe('ABCDEFGHIJKLMNOPQRSTUVWXYZ'); + expect(removeInvisible('0123456789')).toBe('0123456789'); + expect(removeInvisible('!@#$%^&*()_+-=[]{}|;:\'",.<>/?`~')).toBe('!@#$%^&*()_+-=[]{}|;:\'",.<>/?`~'); + expect(removeInvisible('')).toBe(''); + expect(removeInvisible(' ')).toBe(''); + }); + it('other alphabets, list of all characters', () => { + // arabic + expect(removeInvisible('أبجدية عربية')).toBe('أبجدية عربية'); + // chinese + expect(removeInvisible('的一是了我不人在他们')).toBe('的一是了我不人在他们'); + // cyrillic + expect(removeInvisible('абвгдезиклмнопр')).toBe('абвгдезиклмнопр'); + // greek + expect(removeInvisible('αβγδεζηθικλμνξοπρ')).toBe('αβγδεζηθικλμνξοπρ'); + // hebrew + expect(removeInvisible('אבגדהוזחטיכלמנ')).toBe('אבגדהוזחטיכלמנ'); + // hindi + expect(removeInvisible('अआइईउऊऋऍऎ')).toBe('अआइईउऊऋऍऎ'); + // japanese + expect(removeInvisible('あいうえおかきくけこ')).toBe('あいうえおかきくけこ'); + // korean + expect(removeInvisible('가나다라마바사아자')).toBe('가나다라마바사아자'); + // thai + expect(removeInvisible('กขคงจฉชซ')).toBe('กขคงจฉชซ'); + }); + it('trim spaces', () => { + expect(removeInvisible(' test')).toBe('test'); + expect(removeInvisible('test ')).toBe('test'); + expect(removeInvisible(' test ')).toBe('test'); + }); + it('remove invisible characters', () => { + expect(removeInvisible('test\u200B')).toBe('test'); + expect(removeInvisible('test\u200Btest')).toBe('testtest'); + expect(removeInvisible('test\u200B test')).toBe('test test'); + expect(removeInvisible('test\u200B test\u200B')).toBe('test test'); + expect(removeInvisible('test\u200B test\u200B test')).toBe('test test test'); + }); + it('remove invisible characters (Cc)', () => { + expect(removeInvisible('test\u0000')).toBe('test'); + expect(removeInvisible('test\u0001')).toBe('test'); + expect(removeInvisible('test\u0009')).toBe('test'); + }); + it('remove invisible characters (Cf)', () => { + expect(removeInvisible('test\u200E')).toBe('test'); + expect(removeInvisible('test\u200F')).toBe('test'); + expect(removeInvisible('test\u2060')).toBe('test'); + }); + it('check other visible characters (Cs)', () => { + expect(removeInvisible('test\uD800')).toBe('test'); + expect(removeInvisible('test\uD801')).toBe('test'); + expect(removeInvisible('test\uD802')).toBe('test'); + }); + it('check other visible characters (Co)', () => { + expect(removeInvisible('test\uE000')).toBe('test'); + expect(removeInvisible('test\uE001')).toBe('test'); + expect(removeInvisible('test\uE002')).toBe('test'); + }); + it('remove invisible characters (Cn)', () => { + expect(removeInvisible('test\uFFF0')).toBe('test'); + expect(removeInvisible('test\uFFF1')).toBe('test'); + expect(removeInvisible('test\uFFF2')).toBe('test'); + }); + it('remove invisible characters (Zl)', () => { + expect(removeInvisible('test\u2028')).toBe('test'); + expect(removeInvisible('test\u2029')).toBe('test'); + }); + it('basic check emojis not removed', () => { + expect(removeInvisible('test😀')).toBe('test😀'); + expect(removeInvisible('test😀😀')).toBe('test😀😀'); + expect(removeInvisible('test😀😀😀')).toBe('test😀😀😀'); + }); + it('all emojis not removed', () => { + _.keys(enEmojis).forEach((key) => { + expect(removeInvisible(key)).toBe(key); + }); + }); + it('remove invisible characters (editpad)', () => { + expect(removeInvisible('test\u0020')).toBe('test'); + expect(removeInvisible('test\u00A0')).toBe('test'); + expect(removeInvisible('test\u2000')).toBe('test'); + expect(removeInvisible('test\u2001')).toBe('test'); + expect(removeInvisible('test\u2002')).toBe('test'); + expect(removeInvisible('test\u2003')).toBe('test'); + expect(removeInvisible('test\u2004')).toBe('test'); + expect(removeInvisible('test\u2005')).toBe('test'); + expect(removeInvisible('test\u2006')).toBe('test'); + expect(removeInvisible('test\u2007')).toBe('test'); + expect(removeInvisible('test\u2008')).toBe('test'); + expect(removeInvisible('test\u2009')).toBe('test'); + expect(removeInvisible('test\u200A')).toBe('test'); + expect(removeInvisible('test\u2028')).toBe('test'); + expect(removeInvisible('test\u205F')).toBe('test'); + expect(removeInvisible('test\u3000')).toBe('test'); + expect(removeInvisible('test ')).toBe('test'); + }); + it('other tests', () => { + expect(removeInvisible('\uD83D\uDE36\u200D\uD83C\uDF2B\uFE0F')).toBe('😶‍🌫️'); + expect(removeInvisible('\u200D')).toBe('‍'); + expect(removeInvisible('⁠')).toBe(''); + expect(removeInvisible('⁠test')).toBe('test'); + expect(removeInvisible('test⁠test')).toBe('testtest'); + expect(removeInvisible('  ‎ ‏ ⁠ 　 ')).toBe(''); + expect(removeInvisible('te ‎‏⁠st')).toBe('test'); + expect(removeInvisible('\uD83C\uDFF4\uDB40\uDC67\uDB40\uDC62\uDB40\uDC65\uDB40\uDC6E\uDB40\uDC67\uDB40\uDC7F')).toBe('🏴󠁧󠁢󠁥󠁮󠁧󠁿'); + }); +}); From 335052800df668ce54128692e7b38ccb902d61dd Mon Sep 17 00:00:00 2001 From: Jakub Kosmydel <104823336+kosmydel@users.noreply.github.com> Date: Thu, 14 Sep 2023 12:06:52 +0200 Subject: [PATCH 2/7] update Form story, fix typos --- src/libs/isEmptyString.ts | 2 +- src/libs/removeInvisibleCharacters.ts | 2 +- src/stories/Form.stories.js | 17 +++++++++-------- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/src/libs/isEmptyString.ts b/src/libs/isEmptyString.ts index 2cc28880b916..4301f4e9bbb2 100644 --- a/src/libs/isEmptyString.ts +++ b/src/libs/isEmptyString.ts @@ -1,7 +1,7 @@ import CONST from '../CONST'; /** - * Checks if the string would be empty if all invisible characters were removed. + * Check if the string would be empty if all invisible characters were removed. */ function isEmptyString(value: string): boolean { // \p{C} matches all 'Other' characters diff --git a/src/libs/removeInvisibleCharacters.ts b/src/libs/removeInvisibleCharacters.ts index 1096223a25ab..96ed1380a530 100644 --- a/src/libs/removeInvisibleCharacters.ts +++ b/src/libs/removeInvisibleCharacters.ts @@ -11,7 +11,7 @@ function removeInvisibleCharacters(value: string): string { result = result.replace(/[\u200B\u00A0\u2060]/g, ''); // Remove all characters from the 'Other' (C) category except for format characters (Cf) - // because some of them they are used for emojis + // because some of them are used for emojis result = result.replace(/[\p{Cc}\p{Cs}\p{Co}\p{Cn}]/gu, ''); // Remove characters from the (Cf) category that are not used for emojis diff --git a/src/stories/Form.stories.js b/src/stories/Form.stories.js index c13fa23ac098..6efa1bcc97de 100644 --- a/src/stories/Form.stories.js +++ b/src/stories/Form.stories.js @@ -12,6 +12,7 @@ import CheckboxWithLabel from '../components/CheckboxWithLabel'; import Text from '../components/Text'; import NetworkConnection from '../libs/NetworkConnection'; import CONST from '../CONST'; +import * as ValidationUtils from '../libs/ValidationUtils'; /** * We use the Component Story Format for writing stories. Follow the docs here: @@ -166,28 +167,28 @@ const defaultArgs = { submitButtonText: 'Submit', validate: (values) => { const errors = {}; - if (!values.routingNumber) { + if (!ValidationUtils.isRequiredFulfilled(values.routingNumber)) { errors.routingNumber = 'Please enter a routing number'; } - if (!values.accountNumber) { + if (!ValidationUtils.isRequiredFulfilled(values.accountNumber)) { errors.accountNumber = 'Please enter an account number'; } - if (!values.street) { + if (!ValidationUtils.isRequiredFulfilled(values.street)) { errors.street = 'Please enter an address'; } - if (!values.dob) { + if (!ValidationUtils.isRequiredFulfilled(values.dob)) { errors.dob = 'Please enter your date of birth'; } - if (!values.pickFruit) { + if (!ValidationUtils.isRequiredFulfilled(values.pickFruit)) { errors.pickFruit = 'Please select a fruit'; } - if (!values.pickAnotherFruit) { + if (!ValidationUtils.isRequiredFulfilled(values.pickAnotherFruit)) { errors.pickAnotherFruit = 'Please select a fruit'; } - if (!values.state) { + if (!ValidationUtils.isRequiredFulfilled(values.state)) { errors.state = 'Please select a state'; } - if (!values.checkbox) { + if (!ValidationUtils.isRequiredFulfilled(values.checkbox)) { errors.checkbox = 'You must accept the Terms of Service to continue'; } return errors; From dfde954cc4e21a90bf9603bf5a01c49799e5c4d0 Mon Sep 17 00:00:00 2001 From: Jakub Kosmydel <104823336+kosmydel@users.noreply.github.com> Date: Tue, 19 Sep 2023 13:33:07 +0200 Subject: [PATCH 3/7] fix invisible characters first/last name --- src/libs/removeInvisibleCharacters.ts | 7 +++++++ tests/unit/removeInvisibleCharacters.js | 8 ++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/libs/removeInvisibleCharacters.ts b/src/libs/removeInvisibleCharacters.ts index 96ed1380a530..9c979cba1f62 100644 --- a/src/libs/removeInvisibleCharacters.ts +++ b/src/libs/removeInvisibleCharacters.ts @@ -1,3 +1,5 @@ +import isEmptyString from './isEmptyString'; + /** * Remove invisible characters from a string except for spaces and format characters for emoji, and trim it. */ @@ -20,6 +22,11 @@ function removeInvisibleCharacters(value: string): string { // Remove all characters from the 'Separator' (Z) category except for Space Separator (Zs) result = result.replace(/[\p{Zl}\p{Zp}]/gu, ''); + // If the result consist of only invisible characters, return an empty string + if (isEmptyString(result)) { + return ''; + } + return result.trim(); } diff --git a/tests/unit/removeInvisibleCharacters.js b/tests/unit/removeInvisibleCharacters.js index dd6c8867a04d..e4e399f379c4 100644 --- a/tests/unit/removeInvisibleCharacters.js +++ b/tests/unit/removeInvisibleCharacters.js @@ -105,12 +105,16 @@ describe('libs/removeInvisible', () => { }); it('other tests', () => { expect(removeInvisible('\uD83D\uDE36\u200D\uD83C\uDF2B\uFE0F')).toBe('😶‍🌫️'); - expect(removeInvisible('\u200D')).toBe('‍'); - expect(removeInvisible('⁠')).toBe(''); expect(removeInvisible('⁠test')).toBe('test'); expect(removeInvisible('test⁠test')).toBe('testtest'); expect(removeInvisible('  ‎ ‏ ⁠ 　 ')).toBe(''); expect(removeInvisible('te ‎‏⁠st')).toBe('test'); expect(removeInvisible('\uD83C\uDFF4\uDB40\uDC67\uDB40\uDC62\uDB40\uDC65\uDB40\uDC6E\uDB40\uDC67\uDB40\uDC7F')).toBe('🏴󠁧󠁢󠁥󠁮󠁧󠁿'); }); + it('special scenarios', () => { + // Normally we do not remove this character, because it is used in Emojis. + // But if the String consist of only invisible characters, we can safely remove it. + expect(removeInvisible('\u200D')).toBe(''); + expect(removeInvisible('⁠')).toBe(''); + }); }); From c04004c48cf7de8046b51b539c8dd13624286a39 Mon Sep 17 00:00:00 2001 From: Jakub Kosmydel <104823336+kosmydel@users.noreply.github.com> Date: Tue, 17 Oct 2023 11:46:15 +0200 Subject: [PATCH 4/7] add solution to migrated Form component --- src/components/Form/FormProvider.js | 39 ++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/src/components/Form/FormProvider.js b/src/components/Form/FormProvider.js index ada40c24ed89..5ff66c508830 100644 --- a/src/components/Form/FormProvider.js +++ b/src/components/Form/FormProvider.js @@ -12,6 +12,7 @@ import {withNetwork} from '../OnyxProvider'; import stylePropTypes from '../../styles/stylePropTypes'; import networkPropTypes from '../networkPropTypes'; import CONST from '../../CONST'; +import removeInvisibleCharacters from '../../libs/removeInvisibleCharacters'; const propTypes = { /** A unique Onyx key identifying the form */ @@ -106,16 +107,27 @@ function FormProvider({validate, formID, shouldValidateOnBlur, shouldValidateOnC const [errors, setErrors] = useState({}); const hasServerError = useMemo(() => Boolean(formState) && !_.isEmpty(formState.errors), [formState]); + /** + * This function is used to remove invisible characters from strings before validation and submission. + * + * @param {Object} values - An object containing the value of each inputID, e.g. {inputID1: value1, inputID2: value2} + * @returns {Object} - An object containing the processed values of each inputID + */ + const prepareValues = useCallback((values) => { + const trimmedStringValues = {}; + _.each(values, (inputValue, inputID) => { + if (_.isString(inputValue)) { + trimmedStringValues[inputID] = removeInvisibleCharacters(inputValue); + } else { + trimmedStringValues[inputID] = inputValue; + } + }); + return trimmedStringValues; + }, []); + const onValidate = useCallback( (values, shouldClearServerError = true) => { - const trimmedStringValues = {}; - _.each(values, (inputValue, inputID) => { - if (_.isString(inputValue)) { - trimmedStringValues[inputID] = inputValue.trim(); - } else { - trimmedStringValues[inputID] = inputValue; - } - }); + const trimmedStringValues = prepareValues(values); if (shouldClearServerError) { FormActions.setErrors(formID, null); @@ -167,7 +179,7 @@ function FormProvider({validate, formID, shouldValidateOnBlur, shouldValidateOnC return touchedInputErrors; }, - [errors, formID, validate], + [errors, formID, prepareValues, validate], ); /** @@ -186,11 +198,14 @@ function FormProvider({validate, formID, shouldValidateOnBlur, shouldValidateOnC return; } + // Prepare values before submitting + const trimmedStringValues = prepareValues(inputValues); + // Touches all form inputs so we can validate the entire form _.each(inputRefs.current, (inputRef, inputID) => (touchedInputs.current[inputID] = true)); // Validate form and return early if any errors are found - if (!_.isEmpty(onValidate(inputValues))) { + if (!_.isEmpty(onValidate(trimmedStringValues))) { return; } @@ -199,8 +214,8 @@ function FormProvider({validate, formID, shouldValidateOnBlur, shouldValidateOnC return; } - onSubmit(inputValues); - }, [enabledWhenOffline, formState.isLoading, inputValues, network.isOffline, onSubmit, onValidate]); + onSubmit(trimmedStringValues); + }, [enabledWhenOffline, formState.isLoading, inputValues, network.isOffline, onSubmit, onValidate, prepareValues]); const registerInput = useCallback( (inputID, propsToParse = {}) => { From 31e20734205caa4546a6275ee99d5252892869ca Mon Sep 17 00:00:00 2001 From: Jakub Kosmydel <104823336+kosmydel@users.noreply.github.com> Date: Tue, 17 Oct 2023 12:27:55 +0200 Subject: [PATCH 5/7] fix hangul filler character --- src/CONST.ts | 4 +++- src/libs/isEmptyString.ts | 8 +++++++- tests/unit/isEmptyString.js | 3 +++ 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/CONST.ts b/src/CONST.ts index f3a28ba0fc5d..50ee0a3d2f98 100755 --- a/src/CONST.ts +++ b/src/CONST.ts @@ -1331,7 +1331,9 @@ const CONST = { ENCODE_PERCENT_CHARACTER: /%(25)+/g, - INVISIBLE_CHARACTERS: /[\p{C}\p{Z}]/gu, + INVISIBLE_CHARACTERS_GROUPS: /[\p{C}\p{Z}]/gu, + + OTHER_INVISIBLE_CHARACTERS: /[\u3164]/g, }, PRONOUNS: { diff --git a/src/libs/isEmptyString.ts b/src/libs/isEmptyString.ts index 4301f4e9bbb2..0b6c9d3fc8ee 100644 --- a/src/libs/isEmptyString.ts +++ b/src/libs/isEmptyString.ts @@ -7,7 +7,13 @@ function isEmptyString(value: string): boolean { // \p{C} matches all 'Other' characters // \p{Z} matches all separators (spaces etc.) // Source: http://www.unicode.org/reports/tr18/#General_Category_Property - return value.replace(CONST.REGEX.INVISIBLE_CHARACTERS, '') === ''; + let transformed = value.replace(CONST.REGEX.INVISIBLE_CHARACTERS_GROUPS, ''); + + // Remove other invisible characters that are not in the above unicode categories + transformed = transformed.replace(CONST.REGEX.OTHER_INVISIBLE_CHARACTERS, ''); + + // Check if after removing invisible characters the string is empty + return transformed === ''; } export default isEmptyString; diff --git a/tests/unit/isEmptyString.js b/tests/unit/isEmptyString.js index bd905ea2e173..12e70d1ebcc4 100644 --- a/tests/unit/isEmptyString.js +++ b/tests/unit/isEmptyString.js @@ -85,5 +85,8 @@ describe('libs/isEmpty', () => { expect(isEmpty('\uDC6E')).toBe(true); expect(isEmpty('\uDC67')).toBe(true); expect(isEmpty('\uDC7F')).toBe(true); + + // A special test, an invisible character from other Unicode categories than format and control + expect(isEmpty('\u3164')).toBe(true); }); }); From 685df2d02a26980d854ccba1eee26562cda2ce98 Mon Sep 17 00:00:00 2001 From: Jakub Kosmydel <104823336+kosmydel@users.noreply.github.com> Date: Mon, 6 Nov 2023 12:48:30 +0100 Subject: [PATCH 6/7] refactor --- src/components/Form.js | 4 +- src/components/Form/FormProvider.js | 4 +- src/libs/StringUtils.ts | 48 +++++++- src/libs/ValidationUtils.ts | 4 +- src/libs/isEmptyString.ts | 19 --- src/libs/removeInvisibleCharacters.ts | 33 ------ tests/unit/isEmptyString.js | 120 +++++++++---------- tests/unit/removeInvisibleCharacters.js | 146 ++++++++++++------------ 8 files changed, 186 insertions(+), 192 deletions(-) delete mode 100644 src/libs/isEmptyString.ts delete mode 100644 src/libs/removeInvisibleCharacters.ts diff --git a/src/components/Form.js b/src/components/Form.js index 83d1ff1d5dfd..f1ea795efa42 100644 --- a/src/components/Form.js +++ b/src/components/Form.js @@ -6,7 +6,7 @@ import {withOnyx} from 'react-native-onyx'; import _ from 'underscore'; import compose from '@libs/compose'; import * as ErrorUtils from '@libs/ErrorUtils'; -import removeInvisibleCharacters from '@libs/removeInvisibleCharacters'; +import StringUtils from '@libs/StringUtils'; import Visibility from '@libs/Visibility'; import stylePropTypes from '@styles/stylePropTypes'; import styles from '@styles/styles'; @@ -131,7 +131,7 @@ function Form(props) { const trimmedStringValues = {}; _.each(values, (inputValue, inputID) => { if (_.isString(inputValue)) { - trimmedStringValues[inputID] = removeInvisibleCharacters(inputValue); + trimmedStringValues[inputID] = StringUtils.removeInvisibleCharacters(inputValue); } else { trimmedStringValues[inputID] = inputValue; } diff --git a/src/components/Form/FormProvider.js b/src/components/Form/FormProvider.js index 2561e8aa0ab5..10ae32feedbe 100644 --- a/src/components/Form/FormProvider.js +++ b/src/components/Form/FormProvider.js @@ -6,7 +6,7 @@ import _ from 'underscore'; import networkPropTypes from '@components/networkPropTypes'; import {withNetwork} from '@components/OnyxProvider'; import compose from '@libs/compose'; -import removeInvisibleCharacters from '@libs/removeInvisibleCharacters'; +import StringUtils from '@libs/StringUtils'; import Visibility from '@libs/Visibility'; import stylePropTypes from '@styles/stylePropTypes'; import * as FormActions from '@userActions/FormActions'; @@ -117,7 +117,7 @@ function FormProvider({validate, formID, shouldValidateOnBlur, shouldValidateOnC const trimmedStringValues = {}; _.each(values, (inputValue, inputID) => { if (_.isString(inputValue)) { - trimmedStringValues[inputID] = removeInvisibleCharacters(inputValue); + trimmedStringValues[inputID] = StringUtils.removeInvisibleCharacters(inputValue); } else { trimmedStringValues[inputID] = inputValue; } diff --git a/src/libs/StringUtils.ts b/src/libs/StringUtils.ts index 290380ce2cff..3c721302e5c0 100644 --- a/src/libs/StringUtils.ts +++ b/src/libs/StringUtils.ts @@ -10,4 +10,50 @@ function sanitizeString(str: string): string { return _.deburr(str).toLowerCase().replaceAll(CONST.REGEX.NON_ALPHABETIC_AND_NON_LATIN_CHARS, ''); } -export default {sanitizeString}; +/** + * Check if the string would be empty if all invisible characters were removed. + */ +function isEmptyString(value: string): boolean { + // \p{C} matches all 'Other' characters + // \p{Z} matches all separators (spaces etc.) + // Source: http://www.unicode.org/reports/tr18/#General_Category_Property + let transformed = value.replace(CONST.REGEX.INVISIBLE_CHARACTERS_GROUPS, ''); + + // Remove other invisible characters that are not in the above unicode categories + transformed = transformed.replace(CONST.REGEX.OTHER_INVISIBLE_CHARACTERS, ''); + + // Check if after removing invisible characters the string is empty + return transformed === ''; +} + +/** + * Remove invisible characters from a string except for spaces and format characters for emoji, and trim it. + */ +function removeInvisibleCharacters(value: string): string { + let result = value; + + // Remove spaces: + // - \u200B: zero-width space + // - \u00A0: non-breaking space + // - \u2060: word joiner + result = result.replace(/[\u200B\u00A0\u2060]/g, ''); + + // Remove all characters from the 'Other' (C) category except for format characters (Cf) + // because some of them are used for emojis + result = result.replace(/[\p{Cc}\p{Cs}\p{Co}\p{Cn}]/gu, ''); + + // Remove characters from the (Cf) category that are not used for emojis + result = result.replace(/[\u200E-\u200F]/g, ''); + + // Remove all characters from the 'Separator' (Z) category except for Space Separator (Zs) + result = result.replace(/[\p{Zl}\p{Zp}]/gu, ''); + + // If the result consist of only invisible characters, return an empty string + if (isEmptyString(result)) { + return ''; + } + + return result.trim(); +} + +export default {sanitizeString, isEmptyString, removeInvisibleCharacters}; diff --git a/src/libs/ValidationUtils.ts b/src/libs/ValidationUtils.ts index 287c985d58e1..6c9b3463ca72 100644 --- a/src/libs/ValidationUtils.ts +++ b/src/libs/ValidationUtils.ts @@ -8,8 +8,8 @@ import CONST from '@src/CONST'; import {Report} from '@src/types/onyx'; import * as OnyxCommon from '@src/types/onyx/OnyxCommon'; import * as CardUtils from './CardUtils'; -import isEmptyString from './isEmptyString'; import * as LoginUtils from './LoginUtils'; +import StringUtils from './StringUtils'; /** * Implements the Luhn Algorithm, a checksum formula used to validate credit card @@ -74,7 +74,7 @@ function isValidPastDate(date: string | Date): boolean { */ function isRequiredFulfilled(value: string | Date | unknown[] | Record): boolean { if (typeof value === 'string') { - return !isEmptyString(value); + return !StringUtils.isEmptyString(value); } if (isDate(value)) { diff --git a/src/libs/isEmptyString.ts b/src/libs/isEmptyString.ts deleted file mode 100644 index b2f7829b89e6..000000000000 --- a/src/libs/isEmptyString.ts +++ /dev/null @@ -1,19 +0,0 @@ -import CONST from '@src/CONST'; - -/** - * Check if the string would be empty if all invisible characters were removed. - */ -function isEmptyString(value: string): boolean { - // \p{C} matches all 'Other' characters - // \p{Z} matches all separators (spaces etc.) - // Source: http://www.unicode.org/reports/tr18/#General_Category_Property - let transformed = value.replace(CONST.REGEX.INVISIBLE_CHARACTERS_GROUPS, ''); - - // Remove other invisible characters that are not in the above unicode categories - transformed = transformed.replace(CONST.REGEX.OTHER_INVISIBLE_CHARACTERS, ''); - - // Check if after removing invisible characters the string is empty - return transformed === ''; -} - -export default isEmptyString; diff --git a/src/libs/removeInvisibleCharacters.ts b/src/libs/removeInvisibleCharacters.ts deleted file mode 100644 index 9c979cba1f62..000000000000 --- a/src/libs/removeInvisibleCharacters.ts +++ /dev/null @@ -1,33 +0,0 @@ -import isEmptyString from './isEmptyString'; - -/** - * Remove invisible characters from a string except for spaces and format characters for emoji, and trim it. - */ -function removeInvisibleCharacters(value: string): string { - let result = value; - - // Remove spaces: - // - \u200B: zero-width space - // - \u00A0: non-breaking space - // - \u2060: word joiner - result = result.replace(/[\u200B\u00A0\u2060]/g, ''); - - // Remove all characters from the 'Other' (C) category except for format characters (Cf) - // because some of them are used for emojis - result = result.replace(/[\p{Cc}\p{Cs}\p{Co}\p{Cn}]/gu, ''); - - // Remove characters from the (Cf) category that are not used for emojis - result = result.replace(/[\u200E-\u200F]/g, ''); - - // Remove all characters from the 'Separator' (Z) category except for Space Separator (Zs) - result = result.replace(/[\p{Zl}\p{Zp}]/gu, ''); - - // If the result consist of only invisible characters, return an empty string - if (isEmptyString(result)) { - return ''; - } - - return result.trim(); -} - -export default removeInvisibleCharacters; diff --git a/tests/unit/isEmptyString.js b/tests/unit/isEmptyString.js index ff87322d7659..0de9b791fa97 100644 --- a/tests/unit/isEmptyString.js +++ b/tests/unit/isEmptyString.js @@ -1,92 +1,92 @@ import _ from 'underscore'; import enEmojis from '../../assets/emojis/en'; -import isEmpty from '../../src/libs/isEmptyString'; +import StringUtils from '../../src/libs/StringUtils'; -describe('libs/isEmpty', () => { +describe('libs/StringUtils.isEmptyString', () => { it('basic tests', () => { - expect(isEmpty('test')).toBe(false); - expect(isEmpty('test test')).toBe(false); - expect(isEmpty('test test test')).toBe(false); - expect(isEmpty(' ')).toBe(true); + expect(StringUtils.isEmptyString('test')).toBe(false); + expect(StringUtils.isEmptyString('test test')).toBe(false); + expect(StringUtils.isEmptyString('test test test')).toBe(false); + expect(StringUtils.isEmptyString(' ')).toBe(true); }); it('trim spaces', () => { - expect(isEmpty(' test')).toBe(false); - expect(isEmpty('test ')).toBe(false); - expect(isEmpty(' test ')).toBe(false); + expect(StringUtils.isEmptyString(' test')).toBe(false); + expect(StringUtils.isEmptyString('test ')).toBe(false); + expect(StringUtils.isEmptyString(' test ')).toBe(false); }); it('remove invisible characters', () => { - expect(isEmpty('\u200B')).toBe(true); - expect(isEmpty('\u200B')).toBe(true); - expect(isEmpty('\u200B ')).toBe(true); - expect(isEmpty('\u200B \u200B')).toBe(true); - expect(isEmpty('\u200B \u200B ')).toBe(true); + expect(StringUtils.isEmptyString('\u200B')).toBe(true); + expect(StringUtils.isEmptyString('\u200B')).toBe(true); + expect(StringUtils.isEmptyString('\u200B ')).toBe(true); + expect(StringUtils.isEmptyString('\u200B \u200B')).toBe(true); + expect(StringUtils.isEmptyString('\u200B \u200B ')).toBe(true); }); it('remove invisible characters (Cc)', () => { - expect(isEmpty('\u0000')).toBe(true); - expect(isEmpty('\u0001')).toBe(true); - expect(isEmpty('\u0009')).toBe(true); + expect(StringUtils.isEmptyString('\u0000')).toBe(true); + expect(StringUtils.isEmptyString('\u0001')).toBe(true); + expect(StringUtils.isEmptyString('\u0009')).toBe(true); }); it('remove invisible characters (Cf)', () => { - expect(isEmpty('\u200E')).toBe(true); - expect(isEmpty('\u200F')).toBe(true); - expect(isEmpty('\u2060')).toBe(true); + expect(StringUtils.isEmptyString('\u200E')).toBe(true); + expect(StringUtils.isEmptyString('\u200F')).toBe(true); + expect(StringUtils.isEmptyString('\u2060')).toBe(true); }); it('remove invisible characters (Cs)', () => { - expect(isEmpty('\uD800')).toBe(true); - expect(isEmpty('\uD801')).toBe(true); - expect(isEmpty('\uD802')).toBe(true); + expect(StringUtils.isEmptyString('\uD800')).toBe(true); + expect(StringUtils.isEmptyString('\uD801')).toBe(true); + expect(StringUtils.isEmptyString('\uD802')).toBe(true); }); it('remove invisible characters (Co)', () => { - expect(isEmpty('\uE000')).toBe(true); - expect(isEmpty('\uE001')).toBe(true); - expect(isEmpty('\uE002')).toBe(true); + expect(StringUtils.isEmptyString('\uE000')).toBe(true); + expect(StringUtils.isEmptyString('\uE001')).toBe(true); + expect(StringUtils.isEmptyString('\uE002')).toBe(true); }); it('remove invisible characters (Zl)', () => { - expect(isEmpty('\u2028')).toBe(true); - expect(isEmpty('\u2029')).toBe(true); - expect(isEmpty('\u202A')).toBe(true); + expect(StringUtils.isEmptyString('\u2028')).toBe(true); + expect(StringUtils.isEmptyString('\u2029')).toBe(true); + expect(StringUtils.isEmptyString('\u202A')).toBe(true); }); it('basic check emojis not removed', () => { - expect(isEmpty('😀')).toBe(false); + expect(StringUtils.isEmptyString('😀')).toBe(false); }); it('all emojis not removed', () => { _.keys(enEmojis).forEach((key) => { - expect(isEmpty(key)).toBe(false); + expect(StringUtils.isEmptyString(key)).toBe(false); }); }); it('remove invisible characters (editpad)', () => { - expect(isEmpty('\u0020')).toBe(true); - expect(isEmpty('\u00A0')).toBe(true); - expect(isEmpty('\u2000')).toBe(true); - expect(isEmpty('\u2001')).toBe(true); - expect(isEmpty('\u2002')).toBe(true); - expect(isEmpty('\u2003')).toBe(true); - expect(isEmpty('\u2004')).toBe(true); - expect(isEmpty('\u2005')).toBe(true); - expect(isEmpty('\u2006')).toBe(true); - expect(isEmpty('\u2007')).toBe(true); - expect(isEmpty('\u2008')).toBe(true); - expect(isEmpty('\u2009')).toBe(true); - expect(isEmpty('\u200A')).toBe(true); - expect(isEmpty('\u2028')).toBe(true); - expect(isEmpty('\u205F')).toBe(true); - expect(isEmpty('\u3000')).toBe(true); - expect(isEmpty(' ')).toBe(true); + expect(StringUtils.isEmptyString('\u0020')).toBe(true); + expect(StringUtils.isEmptyString('\u00A0')).toBe(true); + expect(StringUtils.isEmptyString('\u2000')).toBe(true); + expect(StringUtils.isEmptyString('\u2001')).toBe(true); + expect(StringUtils.isEmptyString('\u2002')).toBe(true); + expect(StringUtils.isEmptyString('\u2003')).toBe(true); + expect(StringUtils.isEmptyString('\u2004')).toBe(true); + expect(StringUtils.isEmptyString('\u2005')).toBe(true); + expect(StringUtils.isEmptyString('\u2006')).toBe(true); + expect(StringUtils.isEmptyString('\u2007')).toBe(true); + expect(StringUtils.isEmptyString('\u2008')).toBe(true); + expect(StringUtils.isEmptyString('\u2009')).toBe(true); + expect(StringUtils.isEmptyString('\u200A')).toBe(true); + expect(StringUtils.isEmptyString('\u2028')).toBe(true); + expect(StringUtils.isEmptyString('\u205F')).toBe(true); + expect(StringUtils.isEmptyString('\u3000')).toBe(true); + expect(StringUtils.isEmptyString(' ')).toBe(true); }); it('other tests', () => { - expect(isEmpty('\u200D')).toBe(true); - expect(isEmpty('\uD83C\uDFF4\uDB40\uDC67\uDB40\uDC62\uDB40\uDC65\uDB40\uDC6E\uDB40\uDC67\uDB40\uDC7F')).toBe(false); - expect(isEmpty('\uD83C')).toBe(true); - expect(isEmpty('\uDFF4')).toBe(true); - expect(isEmpty('\uDB40')).toBe(true); - expect(isEmpty('\uDC67')).toBe(true); - expect(isEmpty('\uDC62')).toBe(true); - expect(isEmpty('\uDC65')).toBe(true); - expect(isEmpty('\uDC6E')).toBe(true); - expect(isEmpty('\uDC67')).toBe(true); - expect(isEmpty('\uDC7F')).toBe(true); + expect(StringUtils.isEmptyString('\u200D')).toBe(true); + expect(StringUtils.isEmptyString('\uD83C\uDFF4\uDB40\uDC67\uDB40\uDC62\uDB40\uDC65\uDB40\uDC6E\uDB40\uDC67\uDB40\uDC7F')).toBe(false); + expect(StringUtils.isEmptyString('\uD83C')).toBe(true); + expect(StringUtils.isEmptyString('\uDFF4')).toBe(true); + expect(StringUtils.isEmptyString('\uDB40')).toBe(true); + expect(StringUtils.isEmptyString('\uDC67')).toBe(true); + expect(StringUtils.isEmptyString('\uDC62')).toBe(true); + expect(StringUtils.isEmptyString('\uDC65')).toBe(true); + expect(StringUtils.isEmptyString('\uDC6E')).toBe(true); + expect(StringUtils.isEmptyString('\uDC67')).toBe(true); + expect(StringUtils.isEmptyString('\uDC7F')).toBe(true); // A special test, an invisible character from other Unicode categories than format and control - expect(isEmpty('\u3164')).toBe(true); + expect(StringUtils.isEmptyString('\u3164')).toBe(true); }); }); diff --git a/tests/unit/removeInvisibleCharacters.js b/tests/unit/removeInvisibleCharacters.js index e5c7795222cf..f0495b004583 100644 --- a/tests/unit/removeInvisibleCharacters.js +++ b/tests/unit/removeInvisibleCharacters.js @@ -1,120 +1,120 @@ import _ from 'underscore'; import enEmojis from '../../assets/emojis/en'; -import removeInvisible from '../../src/libs/removeInvisibleCharacters'; +import StringUtils from '../../src/libs/StringUtils'; -describe('libs/removeInvisible', () => { +describe('libs/StringUtils.removeInvisibleCharacters', () => { it('basic tests', () => { - expect(removeInvisible('test')).toBe('test'); - expect(removeInvisible('test test')).toBe('test test'); - expect(removeInvisible('abcdefghijklmnopqrstuvwxyz')).toBe('abcdefghijklmnopqrstuvwxyz'); - expect(removeInvisible('ABCDEFGHIJKLMNOPQRSTUVWXYZ')).toBe('ABCDEFGHIJKLMNOPQRSTUVWXYZ'); - expect(removeInvisible('0123456789')).toBe('0123456789'); - expect(removeInvisible('!@#$%^&*()_+-=[]{}|;:\'",.<>/?`~')).toBe('!@#$%^&*()_+-=[]{}|;:\'",.<>/?`~'); - expect(removeInvisible('')).toBe(''); - expect(removeInvisible(' ')).toBe(''); + expect(StringUtils.removeInvisibleCharacters('test')).toBe('test'); + expect(StringUtils.removeInvisibleCharacters('test test')).toBe('test test'); + expect(StringUtils.removeInvisibleCharacters('abcdefghijklmnopqrstuvwxyz')).toBe('abcdefghijklmnopqrstuvwxyz'); + expect(StringUtils.removeInvisibleCharacters('ABCDEFGHIJKLMNOPQRSTUVWXYZ')).toBe('ABCDEFGHIJKLMNOPQRSTUVWXYZ'); + expect(StringUtils.removeInvisibleCharacters('0123456789')).toBe('0123456789'); + expect(StringUtils.removeInvisibleCharacters('!@#$%^&*()_+-=[]{}|;:\'",.<>/?`~')).toBe('!@#$%^&*()_+-=[]{}|;:\'",.<>/?`~'); + expect(StringUtils.removeInvisibleCharacters('')).toBe(''); + expect(StringUtils.removeInvisibleCharacters(' ')).toBe(''); }); it('other alphabets, list of all characters', () => { // arabic - expect(removeInvisible('أبجدية عربية')).toBe('أبجدية عربية'); + expect(StringUtils.removeInvisibleCharacters('أبجدية عربية')).toBe('أبجدية عربية'); // chinese - expect(removeInvisible('的一是了我不人在他们')).toBe('的一是了我不人在他们'); + expect(StringUtils.removeInvisibleCharacters('的一是了我不人在他们')).toBe('的一是了我不人在他们'); // cyrillic - expect(removeInvisible('абвгдезиклмнопр')).toBe('абвгдезиклмнопр'); + expect(StringUtils.removeInvisibleCharacters('абвгдезиклмнопр')).toBe('абвгдезиклмнопр'); // greek - expect(removeInvisible('αβγδεζηθικλμνξοπρ')).toBe('αβγδεζηθικλμνξοπρ'); + expect(StringUtils.removeInvisibleCharacters('αβγδεζηθικλμνξοπρ')).toBe('αβγδεζηθικλμνξοπρ'); // hebrew - expect(removeInvisible('אבגדהוזחטיכלמנ')).toBe('אבגדהוזחטיכלמנ'); + expect(StringUtils.removeInvisibleCharacters('אבגדהוזחטיכלמנ')).toBe('אבגדהוזחטיכלמנ'); // hindi - expect(removeInvisible('अआइईउऊऋऍऎ')).toBe('अआइईउऊऋऍऎ'); + expect(StringUtils.removeInvisibleCharacters('अआइईउऊऋऍऎ')).toBe('अआइईउऊऋऍऎ'); // japanese - expect(removeInvisible('あいうえおかきくけこ')).toBe('あいうえおかきくけこ'); + expect(StringUtils.removeInvisibleCharacters('あいうえおかきくけこ')).toBe('あいうえおかきくけこ'); // korean - expect(removeInvisible('가나다라마바사아자')).toBe('가나다라마바사아자'); + expect(StringUtils.removeInvisibleCharacters('가나다라마바사아자')).toBe('가나다라마바사아자'); // thai - expect(removeInvisible('กขคงจฉชซ')).toBe('กขคงจฉชซ'); + expect(StringUtils.removeInvisibleCharacters('กขคงจฉชซ')).toBe('กขคงจฉชซ'); }); it('trim spaces', () => { - expect(removeInvisible(' test')).toBe('test'); - expect(removeInvisible('test ')).toBe('test'); - expect(removeInvisible(' test ')).toBe('test'); + expect(StringUtils.removeInvisibleCharacters(' test')).toBe('test'); + expect(StringUtils.removeInvisibleCharacters('test ')).toBe('test'); + expect(StringUtils.removeInvisibleCharacters(' test ')).toBe('test'); }); it('remove invisible characters', () => { - expect(removeInvisible('test\u200B')).toBe('test'); - expect(removeInvisible('test\u200Btest')).toBe('testtest'); - expect(removeInvisible('test\u200B test')).toBe('test test'); - expect(removeInvisible('test\u200B test\u200B')).toBe('test test'); - expect(removeInvisible('test\u200B test\u200B test')).toBe('test test test'); + expect(StringUtils.removeInvisibleCharacters('test\u200B')).toBe('test'); + expect(StringUtils.removeInvisibleCharacters('test\u200Btest')).toBe('testtest'); + expect(StringUtils.removeInvisibleCharacters('test\u200B test')).toBe('test test'); + expect(StringUtils.removeInvisibleCharacters('test\u200B test\u200B')).toBe('test test'); + expect(StringUtils.removeInvisibleCharacters('test\u200B test\u200B test')).toBe('test test test'); }); it('remove invisible characters (Cc)', () => { - expect(removeInvisible('test\u0000')).toBe('test'); - expect(removeInvisible('test\u0001')).toBe('test'); - expect(removeInvisible('test\u0009')).toBe('test'); + expect(StringUtils.removeInvisibleCharacters('test\u0000')).toBe('test'); + expect(StringUtils.removeInvisibleCharacters('test\u0001')).toBe('test'); + expect(StringUtils.removeInvisibleCharacters('test\u0009')).toBe('test'); }); it('remove invisible characters (Cf)', () => { - expect(removeInvisible('test\u200E')).toBe('test'); - expect(removeInvisible('test\u200F')).toBe('test'); - expect(removeInvisible('test\u2060')).toBe('test'); + expect(StringUtils.removeInvisibleCharacters('test\u200E')).toBe('test'); + expect(StringUtils.removeInvisibleCharacters('test\u200F')).toBe('test'); + expect(StringUtils.removeInvisibleCharacters('test\u2060')).toBe('test'); }); it('check other visible characters (Cs)', () => { - expect(removeInvisible('test\uD800')).toBe('test'); - expect(removeInvisible('test\uD801')).toBe('test'); - expect(removeInvisible('test\uD802')).toBe('test'); + expect(StringUtils.removeInvisibleCharacters('test\uD800')).toBe('test'); + expect(StringUtils.removeInvisibleCharacters('test\uD801')).toBe('test'); + expect(StringUtils.removeInvisibleCharacters('test\uD802')).toBe('test'); }); it('check other visible characters (Co)', () => { - expect(removeInvisible('test\uE000')).toBe('test'); - expect(removeInvisible('test\uE001')).toBe('test'); - expect(removeInvisible('test\uE002')).toBe('test'); + expect(StringUtils.removeInvisibleCharacters('test\uE000')).toBe('test'); + expect(StringUtils.removeInvisibleCharacters('test\uE001')).toBe('test'); + expect(StringUtils.removeInvisibleCharacters('test\uE002')).toBe('test'); }); it('remove invisible characters (Cn)', () => { - expect(removeInvisible('test\uFFF0')).toBe('test'); - expect(removeInvisible('test\uFFF1')).toBe('test'); - expect(removeInvisible('test\uFFF2')).toBe('test'); + expect(StringUtils.removeInvisibleCharacters('test\uFFF0')).toBe('test'); + expect(StringUtils.removeInvisibleCharacters('test\uFFF1')).toBe('test'); + expect(StringUtils.removeInvisibleCharacters('test\uFFF2')).toBe('test'); }); it('remove invisible characters (Zl)', () => { - expect(removeInvisible('test\u2028')).toBe('test'); - expect(removeInvisible('test\u2029')).toBe('test'); + expect(StringUtils.removeInvisibleCharacters('test\u2028')).toBe('test'); + expect(StringUtils.removeInvisibleCharacters('test\u2029')).toBe('test'); }); it('basic check emojis not removed', () => { - expect(removeInvisible('test😀')).toBe('test😀'); - expect(removeInvisible('test😀😀')).toBe('test😀😀'); - expect(removeInvisible('test😀😀😀')).toBe('test😀😀😀'); + expect(StringUtils.removeInvisibleCharacters('test😀')).toBe('test😀'); + expect(StringUtils.removeInvisibleCharacters('test😀😀')).toBe('test😀😀'); + expect(StringUtils.removeInvisibleCharacters('test😀😀😀')).toBe('test😀😀😀'); }); it('all emojis not removed', () => { _.keys(enEmojis).forEach((key) => { - expect(removeInvisible(key)).toBe(key); + expect(StringUtils.removeInvisibleCharacters(key)).toBe(key); }); }); it('remove invisible characters (editpad)', () => { - expect(removeInvisible('test\u0020')).toBe('test'); - expect(removeInvisible('test\u00A0')).toBe('test'); - expect(removeInvisible('test\u2000')).toBe('test'); - expect(removeInvisible('test\u2001')).toBe('test'); - expect(removeInvisible('test\u2002')).toBe('test'); - expect(removeInvisible('test\u2003')).toBe('test'); - expect(removeInvisible('test\u2004')).toBe('test'); - expect(removeInvisible('test\u2005')).toBe('test'); - expect(removeInvisible('test\u2006')).toBe('test'); - expect(removeInvisible('test\u2007')).toBe('test'); - expect(removeInvisible('test\u2008')).toBe('test'); - expect(removeInvisible('test\u2009')).toBe('test'); - expect(removeInvisible('test\u200A')).toBe('test'); - expect(removeInvisible('test\u2028')).toBe('test'); - expect(removeInvisible('test\u205F')).toBe('test'); - expect(removeInvisible('test\u3000')).toBe('test'); - expect(removeInvisible('test ')).toBe('test'); + expect(StringUtils.removeInvisibleCharacters('test\u0020')).toBe('test'); + expect(StringUtils.removeInvisibleCharacters('test\u00A0')).toBe('test'); + expect(StringUtils.removeInvisibleCharacters('test\u2000')).toBe('test'); + expect(StringUtils.removeInvisibleCharacters('test\u2001')).toBe('test'); + expect(StringUtils.removeInvisibleCharacters('test\u2002')).toBe('test'); + expect(StringUtils.removeInvisibleCharacters('test\u2003')).toBe('test'); + expect(StringUtils.removeInvisibleCharacters('test\u2004')).toBe('test'); + expect(StringUtils.removeInvisibleCharacters('test\u2005')).toBe('test'); + expect(StringUtils.removeInvisibleCharacters('test\u2006')).toBe('test'); + expect(StringUtils.removeInvisibleCharacters('test\u2007')).toBe('test'); + expect(StringUtils.removeInvisibleCharacters('test\u2008')).toBe('test'); + expect(StringUtils.removeInvisibleCharacters('test\u2009')).toBe('test'); + expect(StringUtils.removeInvisibleCharacters('test\u200A')).toBe('test'); + expect(StringUtils.removeInvisibleCharacters('test\u2028')).toBe('test'); + expect(StringUtils.removeInvisibleCharacters('test\u205F')).toBe('test'); + expect(StringUtils.removeInvisibleCharacters('test\u3000')).toBe('test'); + expect(StringUtils.removeInvisibleCharacters('test ')).toBe('test'); }); it('other tests', () => { - expect(removeInvisible('\uD83D\uDE36\u200D\uD83C\uDF2B\uFE0F')).toBe('😶‍🌫️'); - expect(removeInvisible('⁠test')).toBe('test'); - expect(removeInvisible('test⁠test')).toBe('testtest'); - expect(removeInvisible('  ‎ ‏ ⁠ 　 ')).toBe(''); - expect(removeInvisible('te ‎‏⁠st')).toBe('test'); - expect(removeInvisible('\uD83C\uDFF4\uDB40\uDC67\uDB40\uDC62\uDB40\uDC65\uDB40\uDC6E\uDB40\uDC67\uDB40\uDC7F')).toBe('🏴󠁧󠁢󠁥󠁮󠁧󠁿'); + expect(StringUtils.removeInvisibleCharacters('\uD83D\uDE36\u200D\uD83C\uDF2B\uFE0F')).toBe('😶‍🌫️'); + expect(StringUtils.removeInvisibleCharacters('⁠test')).toBe('test'); + expect(StringUtils.removeInvisibleCharacters('test⁠test')).toBe('testtest'); + expect(StringUtils.removeInvisibleCharacters('  ‎ ‏ ⁠ 　 ')).toBe(''); + expect(StringUtils.removeInvisibleCharacters('te ‎‏⁠st')).toBe('test'); + expect(StringUtils.removeInvisibleCharacters('\uD83C\uDFF4\uDB40\uDC67\uDB40\uDC62\uDB40\uDC65\uDB40\uDC6E\uDB40\uDC67\uDB40\uDC7F')).toBe('🏴󠁧󠁢󠁥󠁮󠁧󠁿'); }); it('special scenarios', () => { // Normally we do not remove this character, because it is used in Emojis. // But if the String consist of only invisible characters, we can safely remove it. - expect(removeInvisible('\u200D')).toBe(''); - expect(removeInvisible('⁠')).toBe(''); + expect(StringUtils.removeInvisibleCharacters('\u200D')).toBe(''); + expect(StringUtils.removeInvisibleCharacters('⁠')).toBe(''); }); }); From 7f44418cfb1da3d5228900bcf172dee8ed6475d3 Mon Sep 17 00:00:00 2001 From: Jakub Kosmydel <104823336+kosmydel@users.noreply.github.com> Date: Mon, 6 Nov 2023 13:14:03 +0100 Subject: [PATCH 7/7] extract prepareValues function --- src/components/Form.js | 28 +++++----------------------- src/components/Form/FormProvider.js | 28 +++++----------------------- src/libs/ValidationUtils.ts | 20 ++++++++++++++++++++ 3 files changed, 30 insertions(+), 46 deletions(-) diff --git a/src/components/Form.js b/src/components/Form.js index f1ea795efa42..372c7a0c5d9b 100644 --- a/src/components/Form.js +++ b/src/components/Form.js @@ -6,7 +6,7 @@ import {withOnyx} from 'react-native-onyx'; import _ from 'underscore'; import compose from '@libs/compose'; import * as ErrorUtils from '@libs/ErrorUtils'; -import StringUtils from '@libs/StringUtils'; +import * as ValidationUtils from '@libs/ValidationUtils'; import Visibility from '@libs/Visibility'; import stylePropTypes from '@styles/stylePropTypes'; import styles from '@styles/styles'; @@ -121,24 +121,6 @@ function Form(props) { const hasServerError = useMemo(() => Boolean(props.formState) && !_.isEmpty(props.formState.errors), [props.formState]); - /** - * This function is used to remove invisible characters from strings before validation and submission. - * - * @param {Object} values - An object containing the value of each inputID, e.g. {inputID1: value1, inputID2: value2} - * @returns {Object} - An object containing the processed values of each inputID - */ - const prepareValues = useCallback((values) => { - const trimmedStringValues = {}; - _.each(values, (inputValue, inputID) => { - if (_.isString(inputValue)) { - trimmedStringValues[inputID] = StringUtils.removeInvisibleCharacters(inputValue); - } else { - trimmedStringValues[inputID] = inputValue; - } - }); - return trimmedStringValues; - }, []); - /** * @param {Object} values - An object containing the value of each inputID, e.g. {inputID1: value1, inputID2: value2} * @returns {Object} - An object containing the errors for each inputID, e.g. {inputID1: error1, inputID2: error2} @@ -146,7 +128,7 @@ function Form(props) { const onValidate = useCallback( (values, shouldClearServerError = true) => { // Trim all string values - const trimmedStringValues = prepareValues(values); + const trimmedStringValues = ValidationUtils.prepareValues(values); if (shouldClearServerError) { FormActions.setErrors(props.formID, null); @@ -204,7 +186,7 @@ function Form(props) { return touchedInputErrors; }, - [prepareValues, props.formID, validate, errors], + [props.formID, validate, errors], ); useEffect(() => { @@ -242,7 +224,7 @@ function Form(props) { } // Trim all string values - const trimmedStringValues = prepareValues(inputValues); + const trimmedStringValues = ValidationUtils.prepareValues(inputValues); // Touches all form inputs so we can validate the entire form _.each(inputRefs.current, (inputRef, inputID) => (touchedInputs.current[inputID] = true)); @@ -259,7 +241,7 @@ function Form(props) { // Call submit handler onSubmit(trimmedStringValues); - }, [props.formState.isLoading, props.network.isOffline, props.enabledWhenOffline, prepareValues, inputValues, onValidate, onSubmit]); + }, [props.formState.isLoading, props.network.isOffline, props.enabledWhenOffline, inputValues, onValidate, onSubmit]); /** * Loops over Form's children and automatically supplies Form props to them diff --git a/src/components/Form/FormProvider.js b/src/components/Form/FormProvider.js index 10ae32feedbe..fa0cc3ebd723 100644 --- a/src/components/Form/FormProvider.js +++ b/src/components/Form/FormProvider.js @@ -6,7 +6,7 @@ import _ from 'underscore'; import networkPropTypes from '@components/networkPropTypes'; import {withNetwork} from '@components/OnyxProvider'; import compose from '@libs/compose'; -import StringUtils from '@libs/StringUtils'; +import * as ValidationUtils from '@libs/ValidationUtils'; import Visibility from '@libs/Visibility'; import stylePropTypes from '@styles/stylePropTypes'; import * as FormActions from '@userActions/FormActions'; @@ -107,27 +107,9 @@ function FormProvider({validate, formID, shouldValidateOnBlur, shouldValidateOnC const [errors, setErrors] = useState({}); const hasServerError = useMemo(() => Boolean(formState) && !_.isEmpty(formState.errors), [formState]); - /** - * This function is used to remove invisible characters from strings before validation and submission. - * - * @param {Object} values - An object containing the value of each inputID, e.g. {inputID1: value1, inputID2: value2} - * @returns {Object} - An object containing the processed values of each inputID - */ - const prepareValues = useCallback((values) => { - const trimmedStringValues = {}; - _.each(values, (inputValue, inputID) => { - if (_.isString(inputValue)) { - trimmedStringValues[inputID] = StringUtils.removeInvisibleCharacters(inputValue); - } else { - trimmedStringValues[inputID] = inputValue; - } - }); - return trimmedStringValues; - }, []); - const onValidate = useCallback( (values, shouldClearServerError = true) => { - const trimmedStringValues = prepareValues(values); + const trimmedStringValues = ValidationUtils.prepareValues(values); if (shouldClearServerError) { FormActions.setErrors(formID, null); @@ -179,7 +161,7 @@ function FormProvider({validate, formID, shouldValidateOnBlur, shouldValidateOnC return touchedInputErrors; }, - [errors, formID, prepareValues, validate], + [errors, formID, validate], ); /** @@ -199,7 +181,7 @@ function FormProvider({validate, formID, shouldValidateOnBlur, shouldValidateOnC } // Prepare values before submitting - const trimmedStringValues = prepareValues(inputValues); + const trimmedStringValues = ValidationUtils.prepareValues(inputValues); // Touches all form inputs so we can validate the entire form _.each(inputRefs.current, (inputRef, inputID) => (touchedInputs.current[inputID] = true)); @@ -215,7 +197,7 @@ function FormProvider({validate, formID, shouldValidateOnBlur, shouldValidateOnC } onSubmit(trimmedStringValues); - }, [enabledWhenOffline, formState.isLoading, inputValues, network.isOffline, onSubmit, onValidate, prepareValues]); + }, [enabledWhenOffline, formState.isLoading, inputValues, network.isOffline, onSubmit, onValidate]); const registerInput = useCallback( (inputID, propsToParse = {}) => { diff --git a/src/libs/ValidationUtils.ts b/src/libs/ValidationUtils.ts index 6c9b3463ca72..7c49006c10a5 100644 --- a/src/libs/ValidationUtils.ts +++ b/src/libs/ValidationUtils.ts @@ -353,6 +353,25 @@ function isValidAccountRoute(accountID: number): boolean { return CONST.REGEX.NUMBER.test(String(accountID)) && accountID > 0; } +type ValuesType = Record; + +/** + * This function is used to remove invisible characters from strings before validation and submission. + */ +function prepareValues(values: ValuesType): ValuesType { + const trimmedStringValues: ValuesType = {}; + + for (const [inputID, inputValue] of Object.entries(values)) { + if (typeof inputValue === 'string') { + trimmedStringValues[inputID] = StringUtils.removeInvisibleCharacters(inputValue); + } else { + trimmedStringValues[inputID] = inputValue; + } + } + + return trimmedStringValues; +} + export { meetsMinimumAgeRequirement, meetsMaximumAgeRequirement, @@ -386,4 +405,5 @@ export { isNumeric, isValidAccountRoute, isValidRecoveryCode, + prepareValues, };