Handle invisible characters in forms

src/CONST.ts

@@ -1245,6 +1245,8 @@ const CONST = {
         DATE_TIME_FORMAT: /^\d{2}-\d{2} \d{2}:\d{2} [AP]M$/,
         ATTACHMENT_ROUTE: /\/r\/(\d*)\/attachment/,
         ILLEGAL_FILENAME_CHARACTERS: /\/|<|>|\*|"|:|\?|\\|\|/g,
+
+        INVISIBLE_CHARACTERS: /[\p{C}\p{Z}]/gu,
     },
 
     PRONOUNS: {

src/components/Form.js

@@ -18,6 +18,7 @@ import stylePropTypes from '../styles/stylePropTypes';
 import {withNetwork} from './OnyxProvider';
 import networkPropTypes from './networkPropTypes';
 import Visibility from '../libs/Visibility';
+import removeInvisibleCharacters from '../libs/removeInvisibleCharacters';
 
 const propTypes = {
     /** A unique Onyx key identifying the form */
@@ -115,20 +116,32 @@ function Form(props) {
 
     const hasServerError = useMemo(() => Boolean(props.formState) && !_.isEmpty(props.formState.errors), [props.formState]);
 
+    /**
+     * This function is used to remove invisible characters from strings before validation and submission.
+     *
+     * @param {Object} values - An object containing the value of each inputID, e.g. {inputID1: value1, inputID2: value2}
+     * @returns {Object} - An object containing the processed values of each inputID
+     */
+    const prepareValues = useCallback((values) => {
+        const trimmedStringValues = {};
+        _.each(values, (inputValue, inputID) => {
+            if (_.isString(inputValue)) {
+                trimmedStringValues[inputID] = removeInvisibleCharacters(inputValue);
+            } else {
+                trimmedStringValues[inputID] = inputValue;
+            }
+        });
+        return trimmedStringValues;
+    }, []);
+
     /**
      * @param {Object} values - An object containing the value of each inputID, e.g. {inputID1: value1, inputID2: value2}
      * @returns {Object} - An object containing the errors for each inputID, e.g. {inputID1: error1, inputID2: error2}
      */
     const onValidate = useCallback(
         (values, shouldClearServerError = true) => {
-            const trimmedStringValues = {};
-            _.each(values, (inputValue, inputID) => {
-                if (_.isString(inputValue)) {
-                    trimmedStringValues[inputID] = inputValue.trim();
-                } else {
-                    trimmedStringValues[inputID] = inputValue;
-                }
-            });
+            // Trim all string values
+            const trimmedStringValues = prepareValues(values);
 
             if (shouldClearServerError) {
                 FormActions.setErrors(props.formID, null);
@@ -186,7 +199,7 @@ function Form(props) {
 
             return touchedInputErrors;
         },
-        [errors, touchedInputs, props.formID, validate],
+        [prepareValues, props.formID, validate, errors],
     );
 
     useEffect(() => {
@@ -223,11 +236,14 @@ function Form(props) {
             return;
         }
 
+        // Trim all string values
+        const trimmedStringValues = prepareValues(inputValues);
+
         // Touches all form inputs so we can validate the entire form
         _.each(inputRefs.current, (inputRef, inputID) => (touchedInputs.current[inputID] = true));
 
         // Validate form and return early if any errors are found
-        if (!_.isEmpty(onValidate(inputValues))) {
+        if (!_.isEmpty(onValidate(trimmedStringValues))) {
             return;
         }
 
@@ -237,8 +253,8 @@ function Form(props) {
         }
 
         // Call submit handler
-        onSubmit(inputValues);
-    }, [props.formState, onSubmit, inputRefs, inputValues, onValidate, touchedInputs, props.network.isOffline, props.enabledWhenOffline]);
+        onSubmit(trimmedStringValues);
+    }, [props.formState.isLoading, props.network.isOffline, props.enabledWhenOffline, prepareValues, inputValues, onValidate, onSubmit]);
 
     /**
      * Loops over Form's children and automatically supplies Form props to them

src/libs/ValidationUtils.js

@@ -5,6 +5,7 @@ import {parsePhoneNumber} from 'awesome-phonenumber';
 import CONST from '../CONST';
 import * as CardUtils from './CardUtils';
 import * as LoginUtils from './LoginUtils';
+import isEmptyString from './isEmptyString';
 
 /**
  * Implements the Luhn Algorithm, a checksum formula used to validate credit card
@@ -84,7 +85,7 @@ function isValidPastDate(date) {
  */
 function isRequiredFulfilled(value) {
     if (_.isString(value)) {
-        return !_.isEmpty(value.trim());
+        return !isEmptyString(value);
     }
     if (_.isDate(value)) {
         return isValidDate(value);

src/libs/isEmptyString.ts

@@ -0,0 +1,13 @@
+import CONST from '../CONST';
+
+/**
+ *  Checks if the string would be empty if all invisible characters were removed.
+ */
+function isEmptyString(value: string): boolean {
+    // \p{C} matches all 'Other' characters
+    // \p{Z} matches all separators (spaces etc.)
+    // Source: http://www.unicode.org/reports/tr18/#General_Category_Property
+    return value.replace(CONST.REGEX.INVISIBLE_CHARACTERS, '') === '';
+}
+
+export default isEmptyString;

src/libs/removeInvisibleCharacters.ts

@@ -0,0 +1,26 @@
+/**
+ *  Remove invisible characters from a string except for spaces and format characters for emoji, and trim it.
+ */
+function removeInvisibleCharacters(value: string): string {
+    let result = value;
+
+    // Remove spaces:
+    // - \u200B: zero-width space
+    // - \u00A0: non-breaking space
+    // - \u2060: word joiner
+    result = result.replace(/[\u200B\u00A0\u2060]/g, '');
+
+    // Remove all characters from the 'Other' (C) category except for format characters (Cf)
+    // because some of them they are used for emojis
+    result = result.replace(/[\p{Cc}\p{Cs}\p{Co}\p{Cn}]/gu, '');
+
+    // Remove characters from the (Cf) category that are not used for emojis
+    result = result.replace(/[\u200E-\u200F]/g, '');
+
+    // Remove all characters from the 'Separator' (Z) category except for Space Separator (Zs)
+    result = result.replace(/[\p{Zl}\p{Zp}]/gu, '');
+
+    return result.trim();
+}
+
+export default removeInvisibleCharacters;

src/pages/workspace/WorkspaceSettingsPage.js

@@ -24,6 +24,7 @@ import Avatar from '../../components/Avatar';
 import Navigation from '../../libs/Navigation/Navigation';
 import ROUTES from '../../ROUTES';
 import withWindowDimensions, {windowDimensionsPropTypes} from '../../components/withWindowDimensions';
+import * as ValidationUtils from '../../libs/ValidationUtils';
 
 const propTypes = {
     // The currency list constant object from Onyx
@@ -69,7 +70,7 @@ function WorkspaceSettingsPage(props) {
         const errors = {};
         const name = values.name.trim();
 
-        if (!name || !name.length) {
+        if (!ValidationUtils.isRequiredFulfilled(name)) {
             errors.name = 'workspace.editor.nameIsRequiredError';
         } else if ([...name].length > CONST.WORKSPACE_NAME_CHARACTER_LIMIT) {
             // Uses the spread syntax to count the number of Unicode code points instead of the number of UTF-16

tests/unit/isEmptyString.js

@@ -0,0 +1,89 @@
+import _ from 'underscore';
+import isEmpty from '../../src/libs/isEmptyString';
+import enEmojis from '../../assets/emojis/en';
+
+describe('libs/isEmpty', () => {
+    it('basic tests', () => {
+        expect(isEmpty('test')).toBe(false);
+        expect(isEmpty('test test')).toBe(false);
+        expect(isEmpty('test test test')).toBe(false);
+        expect(isEmpty(' ')).toBe(true);
+    });
+    it('trim spaces', () => {
+        expect(isEmpty(' test')).toBe(false);
+        expect(isEmpty('test ')).toBe(false);
+        expect(isEmpty(' test ')).toBe(false);
+    });
+    it('remove invisible characters', () => {
+        expect(isEmpty('\u200B')).toBe(true);
+        expect(isEmpty('\u200B')).toBe(true);
+        expect(isEmpty('\u200B ')).toBe(true);
+        expect(isEmpty('\u200B \u200B')).toBe(true);
+        expect(isEmpty('\u200B \u200B ')).toBe(true);
+    });
+    it('remove invisible characters (Cc)', () => {
+        expect(isEmpty('\u0000')).toBe(true);
+        expect(isEmpty('\u0001')).toBe(true);
+        expect(isEmpty('\u0009')).toBe(true);
+    });
+    it('remove invisible characters (Cf)', () => {
+        expect(isEmpty('\u200E')).toBe(true);
+        expect(isEmpty('\u200F')).toBe(true);
+        expect(isEmpty('\u2060')).toBe(true);
+    });
+    it('remove invisible characters (Cs)', () => {
+        expect(isEmpty('\uD800')).toBe(true);
+        expect(isEmpty('\uD801')).toBe(true);
+        expect(isEmpty('\uD802')).toBe(true);
+    });
+    it('remove invisible characters (Co)', () => {
+        expect(isEmpty('\uE000')).toBe(true);
+        expect(isEmpty('\uE001')).toBe(true);
+        expect(isEmpty('\uE002')).toBe(true);
+    });
+    it('remove invisible characters (Zl)', () => {
+        expect(isEmpty('\u2028')).toBe(true);
+        expect(isEmpty('\u2029')).toBe(true);
+        expect(isEmpty('\u202A')).toBe(true);
+    });
+    it('basic check emojis not removed', () => {
+        expect(isEmpty('😀')).toBe(false);
+    });
+    it('all emojis not removed', () => {
+        _.keys(enEmojis).forEach((key) => {
+            expect(isEmpty(key)).toBe(false);
+        });
+    });
+    it('remove invisible characters (editpad)', () => {
+        expect(isEmpty('\u0020')).toBe(true);
+        expect(isEmpty('\u00A0')).toBe(true);
+        expect(isEmpty('\u2000')).toBe(true);
+        expect(isEmpty('\u2001')).toBe(true);
+        expect(isEmpty('\u2002')).toBe(true);
+        expect(isEmpty('\u2003')).toBe(true);
+        expect(isEmpty('\u2004')).toBe(true);
+        expect(isEmpty('\u2005')).toBe(true);
+        expect(isEmpty('\u2006')).toBe(true);
+        expect(isEmpty('\u2007')).toBe(true);
+        expect(isEmpty('\u2008')).toBe(true);
+        expect(isEmpty('\u2009')).toBe(true);
+        expect(isEmpty('\u200A')).toBe(true);
+        expect(isEmpty('\u2028')).toBe(true);
+        expect(isEmpty('\u205F')).toBe(true);
+        expect(isEmpty('\u3000')).toBe(true);
+        expect(isEmpty(' ')).toBe(true);
+    });
+    it('other tests', () => {
+        expect(isEmpty('\u200D')).toBe(true);
+        expect(isEmpty('\uD83C\uDFF4\uDB40\uDC67\uDB40\uDC62\uDB40\uDC65\uDB40\uDC6E\uDB40\uDC67\uDB40\uDC7F')).toBe(false);
+        expect(isEmpty('\uD83C')).toBe(true);
+        expect(isEmpty('\uDFF4')).toBe(true);
+        expect(isEmpty('\uDB40')).toBe(true);
+        expect(isEmpty('\uDC67')).toBe(true);
+        expect(isEmpty('\uDC62')).toBe(true);
+        expect(isEmpty('\uDC65')).toBe(true);
+        expect(isEmpty('\uDC6E')).toBe(true);
+        expect(isEmpty('\uDC67')).toBe(true);
+        expect(isEmpty('\uDC7F')).toBe(true);
+    });
+});

tests/unit/removeInvisibleCharacters.js

@@ -0,0 +1,116 @@
+import _ from 'underscore';
+import removeInvisible from '../../src/libs/removeInvisibleCharacters';
+import enEmojis from '../../assets/emojis/en';
+
+describe('libs/removeInvisible', () => {
+    it('basic tests', () => {
+        expect(removeInvisible('test')).toBe('test');
+        expect(removeInvisible('test test')).toBe('test test');
+        expect(removeInvisible('abcdefghijklmnopqrstuvwxyz')).toBe('abcdefghijklmnopqrstuvwxyz');
+        expect(removeInvisible('ABCDEFGHIJKLMNOPQRSTUVWXYZ')).toBe('ABCDEFGHIJKLMNOPQRSTUVWXYZ');
+        expect(removeInvisible('0123456789')).toBe('0123456789');
+        expect(removeInvisible('!@#$%^&*()_+-=[]{}|;:\'",.<>/?`~')).toBe('!@#$%^&*()_+-=[]{}|;:\'",.<>/?`~');
+        expect(removeInvisible('')).toBe('');
+        expect(removeInvisible(' ')).toBe('');
+    });
+    it('other alphabets, list of all characters', () => {
+        // arabic
+        expect(removeInvisible('أبجدية عربية')).toBe('أبجدية عربية');
+        // chinese
+        expect(removeInvisible('的一是了我不人在他们')).toBe('的一是了我不人在他们');
+        // cyrillic
+        expect(removeInvisible('абвгдезиклмнопр')).toBe('абвгдезиклмнопр');
+        // greek
+        expect(removeInvisible('αβγδεζηθικλμνξοπρ')).toBe('αβγδεζηθικλμνξοπρ');
+        // hebrew
+        expect(removeInvisible('אבגדהוזחטיכלמנ')).toBe('אבגדהוזחטיכלמנ');
+        // hindi
+        expect(removeInvisible('अआइईउऊऋऍऎ')).toBe('अआइईउऊऋऍऎ');
+        // japanese
+        expect(removeInvisible('あいうえおかきくけこ')).toBe('あいうえおかきくけこ');
+        // korean
+        expect(removeInvisible('가나다라마바사아자')).toBe('가나다라마바사아자');
+        // thai
+        expect(removeInvisible('กขคงจฉชซ')).toBe('กขคงจฉชซ');
+    });
+    it('trim spaces', () => {
+        expect(removeInvisible(' test')).toBe('test');
+        expect(removeInvisible('test ')).toBe('test');
+        expect(removeInvisible(' test ')).toBe('test');
+    });
+    it('remove invisible characters', () => {
+        expect(removeInvisible('test\u200B')).toBe('test');
+        expect(removeInvisible('test\u200Btest')).toBe('testtest');
+        expect(removeInvisible('test\u200B test')).toBe('test test');
+        expect(removeInvisible('test\u200B test\u200B')).toBe('test test');
+        expect(removeInvisible('test\u200B test\u200B test')).toBe('test test test');
+    });
+    it('remove invisible characters (Cc)', () => {
+        expect(removeInvisible('test\u0000')).toBe('test');
+        expect(removeInvisible('test\u0001')).toBe('test');
+        expect(removeInvisible('test\u0009')).toBe('test');
+    });
+    it('remove invisible characters (Cf)', () => {
+        expect(removeInvisible('test\u200E')).toBe('test');
+        expect(removeInvisible('test\u200F')).toBe('test');
+        expect(removeInvisible('test\u2060')).toBe('test');
+    });
+    it('check other visible characters (Cs)', () => {
+        expect(removeInvisible('test\uD800')).toBe('test');
+        expect(removeInvisible('test\uD801')).toBe('test');
+        expect(removeInvisible('test\uD802')).toBe('test');
+    });
+    it('check other visible characters (Co)', () => {
+        expect(removeInvisible('test\uE000')).toBe('test');
+        expect(removeInvisible('test\uE001')).toBe('test');
+        expect(removeInvisible('test\uE002')).toBe('test');
+    });
+    it('remove invisible characters (Cn)', () => {
+        expect(removeInvisible('test\uFFF0')).toBe('test');
+        expect(removeInvisible('test\uFFF1')).toBe('test');
+        expect(removeInvisible('test\uFFF2')).toBe('test');
+    });
+    it('remove invisible characters (Zl)', () => {
+        expect(removeInvisible('test\u2028')).toBe('test');
+        expect(removeInvisible('test\u2029')).toBe('test');
+    });
+    it('basic check emojis not removed', () => {
+        expect(removeInvisible('test😀')).toBe('test😀');
+        expect(removeInvisible('test😀😀')).toBe('test😀😀');
+        expect(removeInvisible('test😀😀😀')).toBe('test😀😀😀');
+    });
+    it('all emojis not removed', () => {
+        _.keys(enEmojis).forEach((key) => {
+            expect(removeInvisible(key)).toBe(key);
+        });
+    });
+    it('remove invisible characters (editpad)', () => {
+        expect(removeInvisible('test\u0020')).toBe('test');
+        expect(removeInvisible('test\u00A0')).toBe('test');
+        expect(removeInvisible('test\u2000')).toBe('test');
+        expect(removeInvisible('test\u2001')).toBe('test');
+        expect(removeInvisible('test\u2002')).toBe('test');
+        expect(removeInvisible('test\u2003')).toBe('test');
+        expect(removeInvisible('test\u2004')).toBe('test');
+        expect(removeInvisible('test\u2005')).toBe('test');
+        expect(removeInvisible('test\u2006')).toBe('test');
+        expect(removeInvisible('test\u2007')).toBe('test');
+        expect(removeInvisible('test\u2008')).toBe('test');
+        expect(removeInvisible('test\u2009')).toBe('test');
+        expect(removeInvisible('test\u200A')).toBe('test');
+        expect(removeInvisible('test\u2028')).toBe('test');
+        expect(removeInvisible('test\u205F')).toBe('test');
+        expect(removeInvisible('test\u3000')).toBe('test');
+        expect(removeInvisible('test ')).toBe('test');
+    });
+    it('other tests', () => {
+        expect(removeInvisible('\uD83D\uDE36\u200D\uD83C\uDF2B\uFE0F')).toBe('😶‍🌫️');
+        expect(removeInvisible('\u200D')).toBe('‍');
+        expect(removeInvisible('⁠')).toBe('');
+        expect(removeInvisible('⁠test')).toBe('test');
+        expect(removeInvisible('test⁠test')).toBe('testtest');
+        expect(removeInvisible(' 
 	 ‎ ‏ ⁠        　 ')).toBe('');
+        expect(removeInvisible('te
	‎‏⁠st')).toBe('test');
+        expect(removeInvisible('\uD83C\uDFF4\uDB40\uDC67\uDB40\uDC62\uDB40\uDC65\uDB40\uDC6E\uDB40\uDC67\uDB40\uDC7F')).toBe('🏴󠁧󠁢󠁥󠁮󠁧󠁿');
+    });
+});