diff --git a/src/decode.spec.ts b/src/decode.spec.ts index f4a16599..3ec63004 100644 --- a/src/decode.spec.ts +++ b/src/decode.spec.ts @@ -13,6 +13,7 @@ describe("Decode test", () => { { input: ":", output: ":" }, { input: ":", output: ":" }, { input: ":", output: ":" }, + { input: "&#", output: "&#" }, { input: "&>", output: "&>" }, { input: "id=770&#anchor", output: "id=770&#anchor" }, ]; @@ -42,4 +43,240 @@ describe("Decode test", () => { it("should parse   followed by < (#852)", () => expect(entities.decodeHTML(" <")).toBe("\u00a0<")); + + it("should decode trailing legacy entities", () => { + expect(entities.decodeHTML("⨱×bar")).toBe("⨱×bar"); + }); + + it("should decode multi-byte entities", () => { + expect(entities.decodeHTML("≧̸")).toBe("≧̸"); + }); + + it("should not decode legacy entities followed by text in attribute mode", () => { + expect( + entities.decodeHTML("¬", entities.DecodingMode.Attribute) + ).toBe("¬"); + + expect( + entities.decodeHTML("¬i", entities.DecodingMode.Attribute) + ).toBe("¬i"); + + expect( + entities.decodeHTML("¬=", entities.DecodingMode.Attribute) + ).toBe("¬="); + + expect(entities.decodeHTMLAttribute("¬p")).toBe("¬p"); + expect(entities.decodeHTMLAttribute("¬P")).toBe("¬P"); + expect(entities.decodeHTMLAttribute("¬3")).toBe("¬3"); + }); +}); + +describe("EntityDecoder", () => { + it("should decode decimal entities", () => { + const cb = jest.fn(); + const decoder = new entities.EntityDecoder(entities.htmlDecodeTree, cb); + + expect(decoder.write("", 1)).toBe(-1); + expect(decoder.write("8;", 0)).toBe(5); + + expect(cb).toHaveBeenCalledTimes(1); + expect(cb).toHaveBeenCalledWith(":".charCodeAt(0), 5); + }); + + it("should decode hex entities", () => { + const cb = jest.fn(); + const decoder = new entities.EntityDecoder(entities.htmlDecodeTree, cb); + + expect(decoder.write(":", 1)).toBe(6); + + expect(cb).toHaveBeenCalledTimes(1); + expect(cb).toHaveBeenCalledWith(":".charCodeAt(0), 6); + }); + + it("should decode named entities", () => { + const cb = jest.fn(); + const decoder = new entities.EntityDecoder(entities.htmlDecodeTree, cb); + + expect(decoder.write("&", 1)).toBe(5); + + expect(cb).toHaveBeenCalledTimes(1); + expect(cb).toHaveBeenCalledWith("&".charCodeAt(0), 5); + }); + + it("should decode legacy entities", () => { + const cb = jest.fn(); + const decoder = new entities.EntityDecoder(entities.htmlDecodeTree, cb); + decoder.startEntity(entities.DecodingMode.Legacy); + + expect(decoder.write("&", 1)).toBe(-1); + + expect(cb).toHaveBeenCalledTimes(0); + + expect(decoder.end()).toBe(4); + + expect(cb).toHaveBeenCalledTimes(1); + expect(cb).toHaveBeenCalledWith("&".charCodeAt(0), 4); + }); + + it("should decode named entity written character by character", () => { + const cb = jest.fn(); + const decoder = new entities.EntityDecoder(entities.htmlDecodeTree, cb); + + for (const c of "amp") { + expect(decoder.write(c, 0)).toBe(-1); + } + expect(decoder.write(";", 0)).toBe(5); + + expect(cb).toHaveBeenCalledTimes(1); + expect(cb).toHaveBeenCalledWith("&".charCodeAt(0), 5); + }); + + it("should decode numeric entity written character by character", () => { + const cb = jest.fn(); + const decoder = new entities.EntityDecoder(entities.htmlDecodeTree, cb); + + for (const c of "#x3a") { + expect(decoder.write(c, 0)).toBe(-1); + } + expect(decoder.write(";", 0)).toBe(6); + + expect(cb).toHaveBeenCalledTimes(1); + expect(cb).toHaveBeenCalledWith(":".charCodeAt(0), 6); + }); + + it("should not fail if nothing is written", () => { + const cb = jest.fn(); + const decoder = new entities.EntityDecoder(entities.htmlDecodeTree, cb); + + expect(decoder.end()).toBe(0); + expect(cb).toHaveBeenCalledTimes(0); + }); + + describe("errors", () => { + it("should produce an error for a named entity without a semicolon", () => { + const errorHandlers = { + missingSemicolonAfterCharacterReference: jest.fn(), + absenceOfDigitsInNumericCharacterReference: jest.fn(), + validateNumericCharacterReference: jest.fn(), + }; + const cb = jest.fn(); + const decoder = new entities.EntityDecoder( + entities.htmlDecodeTree, + cb, + errorHandlers + ); + + decoder.startEntity(entities.DecodingMode.Legacy); + expect(decoder.write("&", 1)).toBe(5); + expect(cb).toHaveBeenCalledTimes(1); + expect(cb).toHaveBeenCalledWith("&".charCodeAt(0), 5); + expect( + errorHandlers.missingSemicolonAfterCharacterReference + ).toHaveBeenCalledTimes(0); + + decoder.startEntity(entities.DecodingMode.Legacy); + expect(decoder.write("&", 1)).toBe(-1); + expect(decoder.end()).toBe(4); + + expect(cb).toHaveBeenCalledTimes(2); + expect(cb).toHaveBeenLastCalledWith("&".charCodeAt(0), 4); + expect( + errorHandlers.missingSemicolonAfterCharacterReference + ).toHaveBeenCalledTimes(1); + }); + + it("should produce an error for a numeric entity without a semicolon", () => { + const errorHandlers = { + missingSemicolonAfterCharacterReference: jest.fn(), + absenceOfDigitsInNumericCharacterReference: jest.fn(), + validateNumericCharacterReference: jest.fn(), + }; + const cb = jest.fn(); + const decoder = new entities.EntityDecoder( + entities.htmlDecodeTree, + cb, + errorHandlers + ); + + decoder.startEntity(entities.DecodingMode.Legacy); + expect(decoder.write(":", 1)).toBe(-1); + expect(decoder.end()).toBe(5); + + expect(cb).toHaveBeenCalledTimes(1); + expect(cb).toHaveBeenCalledWith(0x3a, 5); + expect( + errorHandlers.missingSemicolonAfterCharacterReference + ).toHaveBeenCalledTimes(1); + expect( + errorHandlers.absenceOfDigitsInNumericCharacterReference + ).toHaveBeenCalledTimes(0); + expect( + errorHandlers.validateNumericCharacterReference + ).toHaveBeenCalledTimes(1); + expect( + errorHandlers.validateNumericCharacterReference + ).toHaveBeenCalledWith(0x3a); + }); + + it("should produce an error for numeric entities without digits", () => { + const errorHandlers = { + missingSemicolonAfterCharacterReference: jest.fn(), + absenceOfDigitsInNumericCharacterReference: jest.fn(), + validateNumericCharacterReference: jest.fn(), + }; + const cb = jest.fn(); + const decoder = new entities.EntityDecoder( + entities.htmlDecodeTree, + cb, + errorHandlers + ); + + decoder.startEntity(entities.DecodingMode.Legacy); + expect(decoder.write("&#", 1)).toBe(-1); + expect(decoder.end()).toBe(0); + + expect(cb).toHaveBeenCalledTimes(0); + expect( + errorHandlers.missingSemicolonAfterCharacterReference + ).toHaveBeenCalledTimes(0); + expect( + errorHandlers.absenceOfDigitsInNumericCharacterReference + ).toHaveBeenCalledTimes(1); + expect( + errorHandlers.absenceOfDigitsInNumericCharacterReference + ).toHaveBeenCalledWith(2); + expect( + errorHandlers.validateNumericCharacterReference + ).toHaveBeenCalledTimes(0); + }); + + it("should produce an error for hex entities without digits", () => { + const errorHandlers = { + missingSemicolonAfterCharacterReference: jest.fn(), + absenceOfDigitsInNumericCharacterReference: jest.fn(), + validateNumericCharacterReference: jest.fn(), + }; + const cb = jest.fn(); + const decoder = new entities.EntityDecoder( + entities.htmlDecodeTree, + cb, + errorHandlers + ); + + decoder.startEntity(entities.DecodingMode.Legacy); + expect(decoder.write("&#x", 1)).toBe(-1); + expect(decoder.end()).toBe(0); + + expect(cb).toHaveBeenCalledTimes(0); + expect( + errorHandlers.missingSemicolonAfterCharacterReference + ).toHaveBeenCalledTimes(0); + expect( + errorHandlers.absenceOfDigitsInNumericCharacterReference + ).toHaveBeenCalledTimes(1); + expect( + errorHandlers.validateNumericCharacterReference + ).toHaveBeenCalledTimes(0); + }); + }); }); diff --git a/src/decode.ts b/src/decode.ts index fbf914fb..2c40d94e 100644 --- a/src/decode.ts +++ b/src/decode.ts @@ -1,6 +1,9 @@ import htmlDecodeTree from "./generated/decode-data-html.js"; import xmlDecodeTree from "./generated/decode-data-xml.js"; -import decodeCodePoint from "./decode_codepoint.js"; +import decodeCodePoint, { + replaceCodePoint, + fromCodePoint, +} from "./decode_codepoint.js"; // Re-export for use by eg. htmlparser2 export { htmlDecodeTree, xmlDecodeTree, decodeCodePoint }; @@ -9,129 +12,516 @@ export { replaceCodePoint, fromCodePoint } from "./decode_codepoint.js"; const enum CharCodes { NUM = 35, // "#" SEMI = 59, // ";" + EQUALS = 61, // "=" ZERO = 48, // "0" NINE = 57, // "9" LOWER_A = 97, // "a" LOWER_F = 102, // "f" LOWER_X = 120, // "x" - /** Bit that needs to be set to convert an upper case ASCII character to lower case */ - To_LOWER_BIT = 0b100000, + LOWER_Z = 122, // "z" + UPPER_A = 65, // "A" + UPPER_F = 70, // "F" + UPPER_Z = 90, // "Z" } +/** Bit that needs to be set to convert an upper case ASCII character to lower case */ +const TO_LOWER_BIT = 0b100000; + export enum BinTrieFlags { VALUE_LENGTH = 0b1100_0000_0000_0000, BRANCH_LENGTH = 0b0011_1111_1000_0000, JUMP_TABLE = 0b0000_0000_0111_1111, } -function getDecoder(decodeTree: Uint16Array) { - return function decodeHTMLBinary(str: string, strict: boolean): string { - let ret = ""; - let lastIdx = 0; - let strIdx = 0; - - while ((strIdx = str.indexOf("&", strIdx)) >= 0) { - ret += str.slice(lastIdx, strIdx); - lastIdx = strIdx; - // Skip the "&" - strIdx += 1; - - // If we have a numeric entity, handle this separately. - if (str.charCodeAt(strIdx) === CharCodes.NUM) { - // Skip the leading "&#". For hex entities, also skip the leading "x". - let start = strIdx + 1; - let base = 10; - - let cp = str.charCodeAt(start); - if ((cp | CharCodes.To_LOWER_BIT) === CharCodes.LOWER_X) { - base = 16; - strIdx += 1; - start += 1; - } +function isNumber(code: number): boolean { + return code >= CharCodes.ZERO && code <= CharCodes.NINE; +} - do cp = str.charCodeAt(++strIdx); - while ( - (cp >= CharCodes.ZERO && cp <= CharCodes.NINE) || - (base === 16 && - (cp | CharCodes.To_LOWER_BIT) >= CharCodes.LOWER_A && - (cp | CharCodes.To_LOWER_BIT) <= CharCodes.LOWER_F) - ); +function isHexadecimalCharacter(code: number): boolean { + return ( + (code >= CharCodes.UPPER_A && code <= CharCodes.UPPER_F) || + (code >= CharCodes.LOWER_A && code <= CharCodes.LOWER_F) + ); +} - if (start !== strIdx) { - const entity = str.substring(start, strIdx); - const parsed = parseInt(entity, base); +function isAsciiAlphaNumeric(code: number): boolean { + return ( + (code >= CharCodes.UPPER_A && code <= CharCodes.UPPER_Z) || + (code >= CharCodes.LOWER_A && code <= CharCodes.LOWER_Z) || + isNumber(code) + ); +} - if (str.charCodeAt(strIdx) === CharCodes.SEMI) { - strIdx += 1; - } else if (strict) { - continue; - } +/** + * Checks if the given character is a valid end character for an entity in an attribute. + * + * Attribute values that aren't terminated properly aren't parsed, and shouldn't lead to a parser error. + * See the example in https://html.spec.whatwg.org/multipage/parsing.html#named-character-reference-state + */ +function isEntityInAttributeInvalidEnd(code: number): boolean { + return code === CharCodes.EQUALS || isAsciiAlphaNumeric(code); +} + +const enum EntityDecoderState { + EntityStart, + NumericStart, + NumericDecimal, + NumericHex, + NamedEntity, +} - ret += decodeCodePoint(parsed); - lastIdx = strIdx; +export enum DecodingMode { + /** Entities in text nodes that can end with any character. */ + Legacy = 0, + /** Only allow entities terminated with a semicolon. */ + Strict = 1, + /** Entities in attributes have limitations on ending characters. */ + Attribute = 2, +} + +/** + * Producers for character reference errors as defined in the HTML spec. + */ +export interface EntityErrorProducer { + missingSemicolonAfterCharacterReference(): void; + absenceOfDigitsInNumericCharacterReference( + consumedCharacters: number + ): void; + validateNumericCharacterReference(code: number): void; +} + +/** + * Token decoder with support of writing partial entities. + */ +export class EntityDecoder { + constructor( + /** The tree used to decode entities. */ + private readonly decodeTree: Uint16Array, + /** + * The function that is called when a codepoint is decoded. + * + * For multi-byte named entities, this will be called multiple times, + * with the second codepoint, and the same `consumed` value. + * + * @param codepoint The decoded codepoint. + * @param consumed The number of bytes consumed by the decoder. + */ + private readonly emitCodePoint: (cp: number, consumed: number) => void, + /** An object that is used to produce errors. */ + private readonly errors?: EntityErrorProducer + ) {} + + /** The current state of the decoder. */ + private state = EntityDecoderState.EntityStart; + /** Characters that were consumed while parsing an entity. */ + private consumed = 1; + /** + * The result of the entity. + * + * Either the result index of a numeric entity, or the codepoint of a + * numeric entity. + */ + private result = 0; + + /** The current index in the decode tree. */ + private treeIndex = 0; + /** The number of characters that were consumed in excess. */ + private excess = 1; + /** The mode in which the decoder is operating. */ + private decodeMode = DecodingMode.Strict; + + /** Resets the instance to make it reusable. */ + startEntity(decodeMode: DecodingMode): void { + this.decodeMode = decodeMode; + this.state = EntityDecoderState.EntityStart; + this.result = 0; + this.treeIndex = 0; + this.excess = 1; + this.consumed = 1; + } + + /** + * Write an entity to the decoder. This can be called multiple times with partial entities. + * If the entity is incomplete, the decoder will return -1. + * + * Mirrors the implementation of `getDecoder`, but with the ability to stop decoding if the + * entity is incomplete, and resume when the next string is written. + * + * @param string The string containing the entity (or a continuation of the entity). + * @param offset The offset at which the entity begins. Should be 0 if this is not the first call. + * @returns The number of characters that were consumed, or -1 if the entity is incomplete. + */ + write(str: string, offset: number): number { + switch (this.state) { + case EntityDecoderState.EntityStart: { + if (str.charCodeAt(offset) === CharCodes.NUM) { + this.state = EntityDecoderState.NumericStart; + this.consumed += 1; + return this.stateNumericStart(str, offset + 1); } + this.state = EntityDecoderState.NamedEntity; + return this.stateNamedEntity(str, offset); + } + + case EntityDecoderState.NumericStart: { + return this.stateNumericStart(str, offset); + } - continue; + case EntityDecoderState.NumericDecimal: { + return this.stateNumericDecimal(str, offset); } - let resultIdx = 0; - let excess = 1; - let treeIdx = 0; - let current = decodeTree[treeIdx]; + case EntityDecoderState.NumericHex: { + return this.stateNumericHex(str, offset); + } - for (; strIdx < str.length; strIdx++, excess++) { - treeIdx = determineBranch( - decodeTree, - current, - treeIdx + 1, - str.charCodeAt(strIdx) - ); + case EntityDecoderState.NamedEntity: { + return this.stateNamedEntity(str, offset); + } + } + } + + /** + * Switches between the numeric decimal and hexadecimal states. + * + * Equivalent to the `Numeric character reference state` in the HTML spec. + * + * @param str The string containing the entity (or a continuation of the entity). + * @param offset The current offset. + * @returns The number of characters that were consumed, or -1 if the entity is incomplete. + */ + private stateNumericStart(str: string, offset: number): number { + if (offset >= str.length) { + return -1; + } + + if ((str.charCodeAt(offset) | TO_LOWER_BIT) === CharCodes.LOWER_X) { + this.state = EntityDecoderState.NumericHex; + this.consumed += 1; + return this.stateNumericHex(str, offset + 1); + } + + this.state = EntityDecoderState.NumericDecimal; + return this.stateNumericDecimal(str, offset); + } + + private addToNumericResult( + str: string, + start: number, + end: number, + base: number + ): void { + if (start !== end) { + this.result = + this.result * base + parseInt(str.slice(start, end), base); + this.consumed += end - start; + } + } + + /** + * Parses a hexadecimal numeric entity. + * + * Equivalent to the `Hexademical character reference state` in the HTML spec. + * + * @param str The string containing the entity (or a continuation of the entity). + * @param offset The current offset. + * @returns The number of characters that were consumed, or -1 if the entity is incomplete. + */ + private stateNumericHex(str: string, offset: number): number { + const startIdx = offset; + + while (offset < str.length) { + const char = str.charCodeAt(offset); + if (isNumber(char) || isHexadecimalCharacter(char)) { + offset += 1; + } else { + this.addToNumericResult(str, startIdx, offset, 16); + return this.emitNumericEntity(char, 3); + } + } + + this.addToNumericResult(str, startIdx, offset, 16); + + return -1; + } + + /** + * Parses a decimal numeric entity. + * + * Equivalent to the `Decimal character reference state` in the HTML spec. + * + * @param str The string containing the entity (or a continuation of the entity). + * @param offset The current offset. + * @returns The number of characters that were consumed, or -1 if the entity is incomplete. + */ + private stateNumericDecimal(str: string, offset: number): number { + const startIdx = offset; + + while (offset < str.length) { + const char = str.charCodeAt(offset); + if (isNumber(char)) { + offset += 1; + } else { + this.addToNumericResult(str, startIdx, offset, 10); + return this.emitNumericEntity(char, 2); + } + } + + this.addToNumericResult(str, startIdx, offset, 10); - if (treeIdx < 0) break; + return -1; + } - current = decodeTree[treeIdx]; + /** + * Validate and emit a numeric entity. + * + * Implements the logic from the `Hexademical character reference start + * state` and `Numeric character reference end state` in the HTML spec. + * + * @param lastCp The last code point of the entity. Used to see if the + * entity was terminated with a semicolon. + * @param expectedLength The minimum number of characters that should be + * consumed. Used to validate that at least one digit + * was consumed. + * @returns The number of characters that were consumed. + */ + private emitNumericEntity(lastCp: number, expectedLength: number): number { + // Ensure we consumed at least one digit. + if (this.consumed <= expectedLength) { + this.errors?.absenceOfDigitsInNumericCharacterReference( + this.consumed + ); + return 0; + } - const masked = current & BinTrieFlags.VALUE_LENGTH; + // Figure out if this is a legit end of the entity + if (lastCp === CharCodes.SEMI) { + this.consumed += 1; + } else if (this.decodeMode === DecodingMode.Strict) { + return 0; + } - // If the branch is a value, store it and continue - if (masked) { - // If we have a legacy entity while parsing strictly, just skip the number of bytes - if (!strict || str.charCodeAt(strIdx) === CharCodes.SEMI) { - resultIdx = treeIdx; - excess = 0; - } + this.emitCodePoint(replaceCodePoint(this.result), this.consumed); - // The mask is the number of bytes of the value, including the current byte. - const valueLength = (masked >> 14) - 1; + if (this.errors) { + if (lastCp !== CharCodes.SEMI) { + this.errors.missingSemicolonAfterCharacterReference(); + } - if (valueLength === 0) break; + this.errors.validateNumericCharacterReference(this.result); + } + + return this.consumed; + } - treeIdx += valueLength; + /** + * Parses a named entity. + * + * Equivalent to the `Named character reference state` in the HTML spec. + * + * @param str The string containing the entity (or a continuation of the entity). + * @param offset The current offset. + * @returns The number of characters that were consumed, or -1 if the entity is incomplete. + */ + private stateNamedEntity(str: string, offset: number): number { + const { decodeTree } = this; + let current = decodeTree[this.treeIndex]; + // The mask is the number of bytes of the value, including the current byte. + let valueLength = (current & BinTrieFlags.VALUE_LENGTH) >> 14; + + for (; offset < str.length; offset++, this.excess++) { + const char = str.charCodeAt(offset); + + this.treeIndex = determineBranch( + decodeTree, + current, + this.treeIndex + Math.max(1, valueLength), + char + ); + + if (this.treeIndex < 0) { + return this.result === 0 || + // If we are parsing an attribute + (this.decodeMode === DecodingMode.Attribute && + // We shouldn't have consumed any characters after the entity, + (valueLength === 0 || + // And there should be no invalid characters. + isEntityInAttributeInvalidEnd(char))) + ? 0 + : this.emitNotTerminatedNamedEntity(); + } + + current = decodeTree[this.treeIndex]; + valueLength = (current & BinTrieFlags.VALUE_LENGTH) >> 14; + + // If the branch is a value, store it and continue + if (valueLength !== 0) { + // If the entity is terminated by a semicolon, we are done. + if (char === CharCodes.SEMI) { + return this.emitNamedEntityData( + this.treeIndex, + valueLength, + this.consumed + this.excess + ); + } + + // If we encounter a non-terminated (legacy) entity while parsing strictly, then ignore it. + if (this.decodeMode !== DecodingMode.Strict) { + this.result = this.treeIndex; + this.consumed += this.excess; + this.excess = 0; } } + } + + return -1; + } + + /** + * Emit a named entity that was not terminated with a semicolon. + * + * @returns The number of characters consumed. + */ + private emitNotTerminatedNamedEntity(): number { + const { result, decodeTree } = this; + + const valueLength = + (decodeTree[result] & BinTrieFlags.VALUE_LENGTH) >> 14; + + this.emitNamedEntityData(result, valueLength, this.consumed); + this.errors?.missingSemicolonAfterCharacterReference(); + + return this.consumed; + } + + /** + * Emit a named entity. + * + * @param result The index of the entity in the decode tree. + * @param valueLength The number of bytes in the entity. + * @param consumed The number of characters consumed. + * + * @returns The number of characters consumed. + */ + private emitNamedEntityData( + result: number, + valueLength: number, + consumed: number + ): number { + const { decodeTree } = this; + + this.emitCodePoint( + valueLength === 1 + ? decodeTree[result] & ~BinTrieFlags.VALUE_LENGTH + : decodeTree[result + 1], + consumed + ); + if (valueLength === 3) { + // For multi-byte values, we need to emit the second byte. + this.emitCodePoint(decodeTree[result + 2], consumed); + } + + return consumed; + } + + /** + * Signal to the parser that the end of the input was reached. + * + * Remaining data will be emitted and relevant errors will be produced. + * + * @returns The number of characters consumed. + */ + end(): number { + switch (this.state) { + case EntityDecoderState.NamedEntity: { + // Emit a named entity if we have one. + return this.result !== 0 && + (this.decodeMode !== DecodingMode.Attribute || + this.result === this.treeIndex) + ? this.emitNotTerminatedNamedEntity() + : 0; + } + // Otherwise, emit a numeric entity if we have one. + case EntityDecoderState.NumericDecimal: { + return this.emitNumericEntity(0, 2); + } + case EntityDecoderState.NumericHex: { + return this.emitNumericEntity(0, 3); + } + case EntityDecoderState.NumericStart: { + this.errors?.absenceOfDigitsInNumericCharacterReference( + this.consumed + ); + return 0; + } + case EntityDecoderState.EntityStart: { + // Return 0 if we have no entity. + return 0; + } + } + } +} - if (resultIdx !== 0) { - const valueLength = - (decodeTree[resultIdx] & BinTrieFlags.VALUE_LENGTH) >> 14; - ret += - valueLength === 1 - ? String.fromCharCode( - decodeTree[resultIdx] & ~BinTrieFlags.VALUE_LENGTH - ) - : valueLength === 2 - ? String.fromCharCode(decodeTree[resultIdx + 1]) - : String.fromCharCode( - decodeTree[resultIdx + 1], - decodeTree[resultIdx + 2] - ); - lastIdx = strIdx - excess + 1; +/** + * Creates a function that decodes entities in a string. + * + * @param decodeTree The decode tree. + * @returns A function that decodes entities in a string. + */ +function getDecoder(decodeTree: Uint16Array) { + let ret = ""; + const decoder = new EntityDecoder( + decodeTree, + (str) => (ret += fromCodePoint(str)) + ); + + return function decodeWithTrie( + str: string, + decodeMode: DecodingMode + ): string { + let lastIndex = 0; + let offset = 0; + + while ((offset = str.indexOf("&", offset)) >= 0) { + ret += str.slice(lastIndex, offset); + + decoder.startEntity(decodeMode); + + const len = decoder.write( + str, + // Skip the "&" + offset + 1 + ); + + if (len < 0) { + lastIndex = offset + decoder.end(); + break; } + + lastIndex = offset + len; + // If `len` is 0, skip the current `&` and continue. + offset = len === 0 ? lastIndex + 1 : lastIndex; } - return ret + str.slice(lastIdx); + const result = ret + str.slice(lastIndex); + + // Make sure we don't keep a reference to the final string. + ret = ""; + + return result; }; } +/** + * Determines the branch of the current node that is taken given the current + * character. This function is used to traverse the trie. + * + * @param decodeTree The trie. + * @param current The current node. + * @param nodeIdx The index right after the current node and its value. + * @param char The current character. + * @returns The index of the next node, or -1 if no branch is taken. + */ export function determineBranch( decodeTree: Uint16Array, current: number, @@ -181,31 +571,42 @@ const htmlDecoder = getDecoder(htmlDecodeTree); const xmlDecoder = getDecoder(xmlDecodeTree); /** - * Decodes an HTML string, allowing for entities not terminated by a semi-colon. + * Decodes an HTML string. + * + * @param str The string to decode. + * @param mode The decoding mode. + * @returns The decoded string. + */ +export function decodeHTML(str: string, mode = DecodingMode.Legacy): string { + return htmlDecoder(str, mode); +} + +/** + * Decodes an HTML string in an attribute. * * @param str The string to decode. * @returns The decoded string. */ -export function decodeHTML(str: string): string { - return htmlDecoder(str, false); +export function decodeHTMLAttribute(str: string): string { + return htmlDecoder(str, DecodingMode.Attribute); } /** - * Decodes an HTML string, requiring all entities to be terminated by a semi-colon. + * Decodes an HTML string, requiring all entities to be terminated by a semicolon. * * @param str The string to decode. * @returns The decoded string. */ export function decodeHTMLStrict(str: string): string { - return htmlDecoder(str, true); + return htmlDecoder(str, DecodingMode.Strict); } /** - * Decodes an XML string, requiring all entities to be terminated by a semi-colon. + * Decodes an XML string, requiring all entities to be terminated by a semicolon. * * @param str The string to decode. * @returns The decoded string. */ export function decodeXML(str: string): string { - return xmlDecoder(str, true); + return xmlDecoder(str, DecodingMode.Strict); } diff --git a/src/decode_codepoint.ts b/src/decode_codepoint.ts index 81966350..a0f17b48 100644 --- a/src/decode_codepoint.ts +++ b/src/decode_codepoint.ts @@ -2,6 +2,7 @@ const decodeMap = new Map([ [0, 65533], + // C1 Unicode control character reference replacements [128, 8364], [130, 8218], [131, 402], @@ -31,6 +32,9 @@ const decodeMap = new Map([ [159, 376], ]); +/** + * Polyfill for `String.fromCodePoint`. It is used to create a string from a Unicode code point. + */ export const fromCodePoint = // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition, node/no-unsupported-features/es-builtins String.fromCodePoint ?? @@ -49,6 +53,11 @@ export const fromCodePoint = return output; }; +/** + * Replace the given code point with a replacement character if it is a + * surrogate or is outside the valid range. Otherwise return the code + * point unchanged. + */ export function replaceCodePoint(codePoint: number) { if ((codePoint >= 0xd800 && codePoint <= 0xdfff) || codePoint > 0x10ffff) { return 0xfffd; @@ -57,6 +66,13 @@ export function replaceCodePoint(codePoint: number) { return decodeMap.get(codePoint) ?? codePoint; } +/** + * Replace the code point if relevant, then convert it to a string. + * + * @deprecated Use `fromCodePoint(replaceCodePoint(codePoint))` instead. + * @param codePoint The code point to decode. + * @returns The decoded code point. + */ export default function decodeCodePoint(codePoint: number): string { return fromCodePoint(replaceCodePoint(codePoint)); } diff --git a/src/escape.ts b/src/escape.ts index 97095bc2..c4183f91 100644 --- a/src/escape.ts +++ b/src/escape.ts @@ -68,6 +68,16 @@ export function encodeXML(str: string): string { */ export const escape = encodeXML; +/** + * Creates a function that escapes all characters matched by the given regular + * expression using the given map of characters to escape to their entities. + * + * @param regex Regular expression to match characters to escape. + * @param map Map of characters to escape to their entities. + * + * @returns Function that escapes all characters matched by the given regular + * expression using the given map of characters to escape to their entities. + */ function getEscaper( regex: RegExp, map: Map diff --git a/src/index.ts b/src/index.ts index 7cd178fa..856795d9 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,4 +1,4 @@ -import { decodeXML, decodeHTML, decodeHTMLStrict } from "./decode.js"; +import { decodeXML, decodeHTML, DecodingMode } from "./decode.js"; import { encodeHTML, encodeNonAsciiHTML } from "./encode.js"; import { encodeXML, @@ -15,14 +15,6 @@ export enum EntityLevel { HTML = 1, } -/** Determines whether some entities are allowed to be written without a trailing `;`. */ -export enum DecodingMode { - /** Support legacy HTML entities. */ - Legacy = 0, - /** Do not support legacy HTML entities. */ - Strict = 1, -} - export enum EncodingMode { /** * The output is UTF-8 encoded. Only characters that need escaping within @@ -82,13 +74,11 @@ export function decode( data: string, options: DecodingOptions | EntityLevel = EntityLevel.XML ): string { - const opts = typeof options === "number" ? { level: options } : options; + const level = typeof options === "number" ? options : options.level; - if (opts.level === EntityLevel.HTML) { - if (opts.mode === DecodingMode.Strict) { - return decodeHTMLStrict(data); - } - return decodeHTML(data); + if (level === EntityLevel.HTML) { + const mode = typeof options === "object" ? options.mode : undefined; + return decodeHTML(data, mode); } return decodeXML(data); @@ -106,15 +96,9 @@ export function decodeStrict( options: DecodingOptions | EntityLevel = EntityLevel.XML ): string { const opts = typeof options === "number" ? { level: options } : options; + opts.mode ??= DecodingMode.Strict; - if (opts.level === EntityLevel.HTML) { - if (opts.mode === DecodingMode.Legacy) { - return decodeHTML(data); - } - return decodeHTMLStrict(data); - } - - return decodeXML(data); + return decode(data, opts); } /** @@ -179,9 +163,12 @@ export { } from "./encode.js"; export { + EntityDecoder, + DecodingMode, decodeXML, decodeHTML, decodeHTMLStrict, + decodeHTMLAttribute, // Legacy aliases (deprecated) decodeHTML as decodeHTML4, decodeHTML as decodeHTML5,