From 728f8e65d63f3086b938d92906fb97c401b465c4 Mon Sep 17 00:00:00 2001 From: n1073645 Date: Tue, 25 Feb 2020 11:27:03 +0000 Subject: [PATCH 01/10] Magic rebuild --- src/core/config/scripts/generateConfig.mjs | 27 +++- src/core/lib/IP.mjs | 8 +- src/core/lib/Magic.mjs | 132 +++++++++++++---- src/core/lib/MagicCriteria.mjs | 12 ++ src/core/operations/A1Z26CipherDecode.mjs | 36 +++++ src/core/operations/BaconCipherDecode.mjs | 46 ++++++ src/core/operations/Bzip2Decompress.mjs | 16 +- src/core/operations/DechunkHTTPResponse.mjs | 11 ++ src/core/operations/DecodeNetBIOSName.mjs | 11 ++ src/core/operations/DefangIPAddresses.mjs | 22 ++- src/core/operations/FromBCD.mjs | 18 ++- src/core/operations/FromBase32.mjs | 17 ++- src/core/operations/FromBase58.mjs | 27 ++-- src/core/operations/FromBase64.mjs | 138 +++++++++--------- src/core/operations/FromDecimal.mjs | 68 +++++---- src/core/operations/FromHTMLEntity.mjs | 18 ++- src/core/operations/FromHex.mjs | 96 ++++++------ src/core/operations/FromHexContent.mjs | 11 ++ src/core/operations/FromHexdump.mjs | 18 ++- src/core/operations/FromMorseCode.mjs | 18 ++- src/core/operations/FromOctal.mjs | 68 +++++---- src/core/operations/FromQuotedPrintable.mjs | 18 ++- src/core/operations/FromUNIXTimestamp.mjs | 48 +++--- src/core/operations/Gunzip.mjs | 18 ++- src/core/operations/ObjectIdentifierToHex.mjs | 11 ++ src/core/operations/ParseSSHHostKey.mjs | 11 ++ .../operations/ParseUNIXFilePermissions.mjs | 11 ++ src/core/operations/ParseUserAgent.mjs | 11 ++ src/core/operations/ParseX509Certificate.mjs | 16 +- src/core/operations/RawInflate.mjs | 9 ++ src/core/operations/RegularExpression.mjs | 2 +- src/core/operations/RenderImage.mjs | 21 ++- src/core/operations/StripHTMLTags.mjs | 11 ++ src/core/operations/StripHTTPHeaders.mjs | 11 ++ src/core/operations/URLDecode.mjs | 18 ++- src/core/operations/Untar.mjs | 16 +- src/core/operations/Unzip.mjs | 18 ++- src/core/operations/ZlibInflate.mjs | 18 ++- 38 files changed, 742 insertions(+), 344 deletions(-) create mode 100644 src/core/lib/MagicCriteria.mjs diff --git a/src/core/config/scripts/generateConfig.mjs b/src/core/config/scripts/generateConfig.mjs index e8f99e404..6e090652b 100644 --- a/src/core/config/scripts/generateConfig.mjs +++ b/src/core/config/scripts/generateConfig.mjs @@ -42,13 +42,32 @@ for (const opObj in Ops) { outputType: op.presentType, flowControl: op.flowControl, manualBake: op.manualBake, - args: op.args + args: op.args, }; - if ("patterns" in op) { - operationConfig[op.name].patterns = op.patterns; + if ("checks" in op) { + if ("input" in op.checks) { + operationConfig[op.name].input = {}; + if ("regex" in op.checks.input) { + operationConfig[op.name].input.regex = op.checks.input.regex; + } + if ("entropy" in op.checks.input) { + operationConfig[op.name].input.entropy = op.checks.input.entropy; + } + } + if ("output" in op.checks) { + operationConfig[op.name].output = {}; + if ("regex" in op.checks.output) { + operationConfig[op.name].output.regex = op.checks.output.regex; + } + if ("entropy" in op.checks.output) { + operationConfig[op.name].output.entropy = op.checks.output.entropy; + } + if ("mime" in op.checks.output) { + operationConfig[op.name].output.mime = op.checks.output.mime; + } + } } - if (!(op.module in modules)) modules[op.module] = {}; modules[op.module][op.name] = opObj; diff --git a/src/core/lib/IP.mjs b/src/core/lib/IP.mjs index f9c54ad07..c97f87ab5 100644 --- a/src/core/lib/IP.mjs +++ b/src/core/lib/IP.mjs @@ -26,7 +26,7 @@ export function ipv4CidrRange(cidr, includeNetworkInfo, enumerateAddresses, allo let output = ""; if (cidrRange < 0 || cidrRange > 31) { - return "IPv4 CIDR must be less than 32"; + throw new OperationError("IPv4 CIDR must be less than 32"); } const mask = ~(0xFFFFFFFF >>> cidrRange), @@ -64,7 +64,7 @@ export function ipv6CidrRange(cidr, includeNetworkInfo) { cidrRange = parseInt(cidr[cidr.length-1], 10); if (cidrRange < 0 || cidrRange > 127) { - return "IPv6 CIDR must be less than 128"; + throw new OperationError("IPv6 CIDR must be less than 128"); } const ip1 = new Array(8), @@ -211,7 +211,7 @@ export function ipv4ListedRange(match, includeNetworkInfo, enumerateAddresses, a const network = strToIpv4(ipv4CidrList[i].split("/")[0]); const cidrRange = parseInt(ipv4CidrList[i].split("/")[1], 10); if (cidrRange < 0 || cidrRange > 31) { - return "IPv4 CIDR must be less than 32"; + throw new OperationError("IPv4 CIDR must be less than 32"); } const mask = ~(0xFFFFFFFF >>> cidrRange), cidrIp1 = network & mask, @@ -254,7 +254,7 @@ export function ipv6ListedRange(match, includeNetworkInfo) { const cidrRange = parseInt(ipv6CidrList[i].split("/")[1], 10); if (cidrRange < 0 || cidrRange > 127) { - return "IPv6 CIDR must be less than 128"; + throw new OperationError("IPv6 CIDR must be less than 128"); } const cidrIp1 = new Array(8), diff --git a/src/core/lib/Magic.mjs b/src/core/lib/Magic.mjs index 5052db84c..6de148eb2 100644 --- a/src/core/lib/Magic.mjs +++ b/src/core/lib/Magic.mjs @@ -2,7 +2,7 @@ import OperationConfig from "../config/OperationConfig.json"; import Utils, { isWorkerEnvironment } from "../Utils.mjs"; import Recipe from "../Recipe.mjs"; import Dish from "../Dish.mjs"; -import {detectFileType} from "./FileType.mjs"; +import {detectFileType, isType} from "./FileType.mjs"; import chiSquared from "chi-squared"; /** @@ -19,25 +19,24 @@ class Magic { * Magic constructor. * * @param {ArrayBuffer} buf - * @param {Object[]} [opPatterns] + * @param {Object} prevOp */ - constructor(buf, opPatterns) { + constructor(buf, opPatterns, prevOp) { this.inputBuffer = new Uint8Array(buf); this.inputStr = Utils.arrayBufferToStr(buf); - this.opPatterns = opPatterns || Magic._generateOpPatterns(); + this.opPatterns = opPatterns || Magic._generateOpCriteria(); + this.prevOp = prevOp; } /** - * Finds operations that claim to be able to decode the input based on regular - * expression matches. * - * @returns {Object[]} + * @param opPatterns */ - findMatchingOps() { + inputRegexMatch(opPatterns) { const matches = []; - for (let i = 0; i < this.opPatterns.length; i++) { - const pattern = this.opPatterns[i], + for (let i = 0; i < opPatterns.length; i++) { + const pattern = opPatterns[i], regex = new RegExp(pattern.match, pattern.flags); if (regex.test(this.inputStr)) { @@ -48,6 +47,34 @@ class Magic { return matches; } + /** + * + */ + entropyInputMatch(opPatterns) { + const matches = []; + + const entropyOfInput = this.calcEntropy(); + + for (let i = 0; i < opPatterns.length; i++) { + const currOp = opPatterns[i]; + if ((entropyOfInput > currOp.entropy[0]) && (entropyOfInput < currOp.entropy[1])) + matches.push(currOp); + } + return matches; + } + + /** + * Finds operations that claim to be able to decode the input based on regular + * expression matches. + * + * @returns {Object[]} + */ + findMatchingInputOps() { + let matches = this.inputRegexMatch(this.opPatterns.regex); + matches = matches.concat(this.entropyInputMatch(this.opPatterns.entropy)); + return [...new Set(matches)]; + } + /** * Attempts to detect the language of the input by comparing its byte frequency * to that of several known languages. @@ -264,6 +291,35 @@ class Magic { return results; } + /** + * + */ + checkRegexes(regexes) { + for (const elem of regexes) { + const regex = new RegExp(elem.match, elem.flags); + if (regex.test(this.inputStr)) + return true; + } + return false; + } + /** + * + */ + checkOutputFromPrevious() { + let score = 0; + if ("regex" in this.prevOp.output) { + if (this.checkRegexes(this.prevOp.output.regex)) score++; + } + if ("entropy" in this.prevOp.output) { + const inputEntropy = this.calcEntropy(); + if ((inputEntropy > this.prevOp.output.entropy[0]) && (inputEntropy < this.prevOp.output.entropy[1])) score++; + } + if ("mime" in this.prevOp.output) { + if (isType(this.prevOp.output.mime, this.inputBuffer)) score++; + } + return score > 0; + } + /** * Speculatively executes matching operations, recording metadata of each result. * @@ -281,8 +337,15 @@ class Magic { if (depth < 0) return []; // Find any operations that can be run on this data - const matchingOps = this.findMatchingOps(); + if (this.prevOp) { + if ("output" in this.prevOp) { + if (!(this.checkOutputFromPrevious())) { + return []; + } + } + } + const matchingOps = this.findMatchingInputOps(); let results = []; // Record the properties of the current data @@ -305,8 +368,7 @@ class Magic { const opConfig = { op: op.op, args: op.args - }, - output = await this._runRecipe([opConfig]); + }, output = await this._runRecipe([opConfig]); // If the recipe is repeating and returning the same data, do not continue if (prevOp && op.op === prevOp.op && _buffersEqual(output, this.inputBuffer)) { @@ -318,7 +380,8 @@ class Magic { return; } - const magic = new Magic(output, this.opPatterns), + + const magic = new Magic(output, this.opPatterns, OperationConfig[op.op]), speculativeResults = await magic.speculativeExecution( depth-1, extLang, intensive, [...recipeConfig, opConfig], op.useful, crib); @@ -330,7 +393,7 @@ class Magic { const bfEncodings = await this.bruteForce(); await Promise.all(bfEncodings.map(async enc => { - const magic = new Magic(enc.data, this.opPatterns), + const magic = new Magic(enc.data, this.opPatterns, undefined), bfResults = await magic.speculativeExecution( depth-1, extLang, false, [...recipeConfig, enc.conf], false, crib); @@ -447,24 +510,35 @@ class Magic { * @private * @returns {Object[]} */ - static _generateOpPatterns() { - const opPatterns = []; + static _generateOpCriteria() { + const opCriteria = { + regex: [], + entropy: [] + }; for (const op in OperationConfig) { - if (!("patterns" in OperationConfig[op])) continue; - - OperationConfig[op].patterns.forEach(pattern => { - opPatterns.push({ - op: op, - match: pattern.match, - flags: pattern.flags, - args: pattern.args, - useful: pattern.useful || false - }); - }); + if ("input" in OperationConfig[op]) { + if ("regex" in OperationConfig[op].input) + OperationConfig[op].input.regex.forEach(pattern => { + opCriteria.regex.push({ + op: op, + match: pattern.match, + flags: pattern.flags, + args: pattern.args, + useful: pattern.useful || false + }); + }); + if ("entropy" in OperationConfig[op].input) { + opCriteria.entropy.push({ + op: op, + entropy: OperationConfig[op].input.entropy.input, + args: OperationConfig[op].input.entropy.args + }); + } + } } - return opPatterns; + return opCriteria; } /** diff --git a/src/core/lib/MagicCriteria.mjs b/src/core/lib/MagicCriteria.mjs new file mode 100644 index 000000000..6cf889a10 --- /dev/null +++ b/src/core/lib/MagicCriteria.mjs @@ -0,0 +1,12 @@ +/** + * Constants for the entropy of text. + * + * @author n1073645 [n1073645@gmail.com] + * @copyright Crown Copyright 2020 + * @license Apache-2.0 +*/ +export const compressedToDecompressed = [6.5, 8]; + +export const binary = [1, 1.5]; + +export const entropyOfText = [3.5, 6]; diff --git a/src/core/operations/A1Z26CipherDecode.mjs b/src/core/operations/A1Z26CipherDecode.mjs index 0b1139450..4f4022fab 100644 --- a/src/core/operations/A1Z26CipherDecode.mjs +++ b/src/core/operations/A1Z26CipherDecode.mjs @@ -33,6 +33,42 @@ class A1Z26CipherDecode extends Operation { value: DELIM_OPTIONS } ]; + this.checks = { + input: { + regex: [ + { + match: "^\\s*([12]?[0-9] )+[12]?[0-9]\\s*$", + flags: "", + args: ["Space"] + }, + { + match: "^\\s*([12]?[0-9],)+[12]?[0-9]\\s*$", + flags: "", + args: ["Comma"] + }, + { + match: "^\\s*([12]?[0-9];)+[12]?[0-9]\\s*$", + flags: "", + args: ["Semi-colon"] + }, + { + match: "^\\s*([12]?[0-9]:)+[12]?[0-9]\\s*$", + flags: "", + args: ["Colon"] + }, + { + match: "^\\s*([12]?[0-9]\\n)+[12]?[0-9]\\s*$", + flags: "", + args: ["Line feed"] + }, + { + match: "^\\s*([12]?[0-9]\\r\\n)+[12]?[0-9]\\s*$", + flags: "", + args: ["CRLF"] + } + ] + } + }; } /** diff --git a/src/core/operations/BaconCipherDecode.mjs b/src/core/operations/BaconCipherDecode.mjs index 56d0946b4..81aa3846a 100644 --- a/src/core/operations/BaconCipherDecode.mjs +++ b/src/core/operations/BaconCipherDecode.mjs @@ -44,6 +44,52 @@ class BaconCipherDecode extends Operation { "value": false } ]; + this.checks = { + input: { + regex: [ + { + match: "^\\s*([01]{5}\\s?)+$", + flags: "", + args: ["Standard (I=J and U=V)", "0/1", false] + }, + { + match: "^\\s*([01]{5}\\s?)+$", + flags: "", + args: ["Standard (I=J and U=V)", "0/1", true] + }, + { + match: "^\\s*([AB]{5}\\s?)+$", + flags: "", + args: ["Standard (I=J and U=V)", "A/B", false] + }, + { + match: "^\\s*([AB]{5}\\s?)+$", + flags: "", + args: ["Standard (I=J and U=V)", "A/B", true] + }, + { + match: "^\\s*([01]{5}\\s?)+$", + flags: "", + args: ["Complete", "0/1", false] + }, + { + match: "^\\s*([01]{5}\\s?)+$", + flags: "", + args: ["Complete", "0/1", true] + }, + { + match: "^\\s*([AB]{5}\\s?)+$", + flags: "", + args: ["Complete", "A/B", false] + }, + { + match: "^\\s*([AB]{5}\\s?)+$", + flags: "", + args: ["Complete", "A/B", true] + } + ] + } + }; } /** diff --git a/src/core/operations/Bzip2Decompress.mjs b/src/core/operations/Bzip2Decompress.mjs index 3dba945ea..7b1108207 100644 --- a/src/core/operations/Bzip2Decompress.mjs +++ b/src/core/operations/Bzip2Decompress.mjs @@ -33,13 +33,17 @@ class Bzip2Decompress extends Operation { value: false } ]; - this.patterns = [ - { - "match": "^\\x42\\x5a\\x68", - "flags": "", - "args": [] + this.checks = { + input: { + regex: [ + { + "match": "^\\x42\\x5a\\x68", + "flags": "", + "args": [] + } + ] } - ]; + }; } /** diff --git a/src/core/operations/DechunkHTTPResponse.mjs b/src/core/operations/DechunkHTTPResponse.mjs index 6a4c38130..c8e008ea4 100644 --- a/src/core/operations/DechunkHTTPResponse.mjs +++ b/src/core/operations/DechunkHTTPResponse.mjs @@ -24,6 +24,17 @@ class DechunkHTTPResponse extends Operation { this.inputType = "string"; this.outputType = "string"; this.args = []; + this.checks = { + input: { + regex: [ + { + match: "^\\s*[0-9A-F]+\r\n", + flags: "i", + args: [] + } + ] + } + }; } /** diff --git a/src/core/operations/DecodeNetBIOSName.mjs b/src/core/operations/DecodeNetBIOSName.mjs index f4d89f4a6..2430043d3 100644 --- a/src/core/operations/DecodeNetBIOSName.mjs +++ b/src/core/operations/DecodeNetBIOSName.mjs @@ -30,6 +30,17 @@ class DecodeNetBIOSName extends Operation { "value": 65 } ]; + this.checks = { + input: { + regex: [ + { + match: "^\\s*\\S{32}$", + flags: "", + args: [65] + } + ] + } + }; } /** diff --git a/src/core/operations/DefangIPAddresses.mjs b/src/core/operations/DefangIPAddresses.mjs index 5623a049d..cecfab9c0 100644 --- a/src/core/operations/DefangIPAddresses.mjs +++ b/src/core/operations/DefangIPAddresses.mjs @@ -25,7 +25,27 @@ class DefangIPAddresses extends Operation { this.inputType = "string"; this.outputType = "string"; this.args = []; - + this.checks = { + input: { + regex: [ + { + match: "^\\s*(([0-9]{1,3}\\.){3}[0-9]{1,3}|([0-9a-f]{4}:){7}[0-9a-f]{4})\\s*$", + flags: "i", + args: [], + } + ] + }, + output: { + regex: [ + { + match: "^\\s*(([0-9]{1,3}\\[\\.\\]){3}[0-9]{1,3}|([0-9a-f]{4}\\[\\:\\]){7}[0-9a-f]{4})\\s*$", + flags: "i", + shouldMatch: true, + args: [] + } + ] + } + }; } /** diff --git a/src/core/operations/FromBCD.mjs b/src/core/operations/FromBCD.mjs index acbe468b3..907d40c66 100644 --- a/src/core/operations/FromBCD.mjs +++ b/src/core/operations/FromBCD.mjs @@ -49,13 +49,17 @@ class FromBCD extends Operation { "value": FORMAT } ]; - this.patterns = [ - { - match: "^(?:\\d{4} ){3,}\\d{4}$", - flags: "", - args: ["8 4 2 1", true, false, "Nibbles"] - }, - ]; + this.checks = { + input: { + regex: [ + { + match: "^(?:\\d{4} ){3,}\\d{4}$", + flags: "", + args: ["8 4 2 1", true, false, "Nibbles"] + }, + ] + } + }; } /** diff --git a/src/core/operations/FromBase32.mjs b/src/core/operations/FromBase32.mjs index a204b8306..5959a9e06 100644 --- a/src/core/operations/FromBase32.mjs +++ b/src/core/operations/FromBase32.mjs @@ -36,13 +36,18 @@ class FromBase32 extends Operation { value: true } ]; - this.patterns = [ + this.checks = { + input: { - match: "^(?:[A-Z2-7]{8})+(?:[A-Z2-7]{2}={6}|[A-Z2-7]{4}={4}|[A-Z2-7]{5}={3}|[A-Z2-7]{7}={1})?$", - flags: "", - args: ["A-Z2-7=", false] - }, - ]; + regex: [ + { + match: "^(?:[A-Z2-7]{8})+(?:[A-Z2-7]{2}={6}|[A-Z2-7]{4}={4}|[A-Z2-7]{5}={3}|[A-Z2-7]{7}={1})?$", + flags: "", + args: ["A-Z2-7=", false] + } + ] + } + }; } /** diff --git a/src/core/operations/FromBase58.mjs b/src/core/operations/FromBase58.mjs index 64668c3f9..d14529e78 100644 --- a/src/core/operations/FromBase58.mjs +++ b/src/core/operations/FromBase58.mjs @@ -38,18 +38,23 @@ class FromBase58 extends Operation { "value": true } ]; - this.patterns = [ + this.checks = { + input: { - match: "^[1-9A-HJ-NP-Za-km-z]{20,}$", - flags: "", - args: ["123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz", false] - }, - { - match: "^[1-9A-HJ-NP-Za-km-z]{20,}$", - flags: "", - args: ["rpshnaf39wBUDNEGHJKLM4PQRST7VWXYZ2bcdeCg65jkm8oFqi1tuvAxyz", false] - }, - ]; + regex: [ + { + match: "^[1-9A-HJ-NP-Za-km-z]{20,}$", + flags: "", + args: ["123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz", false] + }, + { + match: "^[1-9A-HJ-NP-Za-km-z]{20,}$", + flags: "", + args: ["rpshnaf39wBUDNEGHJKLM4PQRST7VWXYZ2bcdeCg65jkm8oFqi1tuvAxyz", false] + }, + ] + } + }; } /** diff --git a/src/core/operations/FromBase64.mjs b/src/core/operations/FromBase64.mjs index 6ee01b655..2a6c6cf41 100644 --- a/src/core/operations/FromBase64.mjs +++ b/src/core/operations/FromBase64.mjs @@ -36,73 +36,77 @@ class FromBase64 extends Operation { value: true } ]; - this.patterns = [ - { - match: "^\\s*(?:[A-Z\\d+/]{4})+(?:[A-Z\\d+/]{2}==|[A-Z\\d+/]{3}=)?\\s*$", - flags: "i", - args: ["A-Za-z0-9+/=", true] - }, - { - match: "^\\s*[A-Z\\d\\-_]{20,}\\s*$", - flags: "i", - args: ["A-Za-z0-9-_", true] - }, - { - match: "^\\s*(?:[A-Z\\d+\\-]{4}){5,}(?:[A-Z\\d+\\-]{2}==|[A-Z\\d+\\-]{3}=)?\\s*$", - flags: "i", - args: ["A-Za-z0-9+\\-=", true] - }, - { - match: "^\\s*(?:[A-Z\\d./]{4}){5,}(?:[A-Z\\d./]{2}==|[A-Z\\d./]{3}=)?\\s*$", - flags: "i", - args: ["./0-9A-Za-z=", true] - }, - { - match: "^\\s*[A-Z\\d_.]{20,}\\s*$", - flags: "i", - args: ["A-Za-z0-9_.", true] - }, - { - match: "^\\s*(?:[A-Z\\d._]{4}){5,}(?:[A-Z\\d._]{2}--|[A-Z\\d._]{3}-)?\\s*$", - flags: "i", - args: ["A-Za-z0-9._-", true] - }, - { - match: "^\\s*(?:[A-Z\\d+/]{4}){5,}(?:[A-Z\\d+/]{2}==|[A-Z\\d+/]{3}=)?\\s*$", - flags: "i", - args: ["0-9a-zA-Z+/=", true] - }, - { - match: "^\\s*(?:[A-Z\\d+/]{4}){5,}(?:[A-Z\\d+/]{2}==|[A-Z\\d+/]{3}=)?\\s*$", - flags: "i", - args: ["0-9A-Za-z+/=", true] - }, - { - match: "^[ !\"#$%&'()*+,\\-./\\d:;<=>?@A-Z[\\\\\\]^_]{20,}$", - flags: "", - args: [" -_", false] - }, - { - match: "^\\s*[A-Z\\d+\\-]{20,}\\s*$", - flags: "i", - args: ["+\\-0-9A-Za-z", true] - }, - { - match: "^\\s*[!\"#$%&'()*+,\\-0-689@A-NP-VX-Z[`a-fh-mp-r]{20,}\\s*$", - flags: "", - args: ["!-,-0-689@A-NP-VX-Z[`a-fh-mp-r", true] - }, - { - match: "^\\s*(?:[N-ZA-M\\d+/]{4}){5,}(?:[N-ZA-M\\d+/]{2}==|[N-ZA-M\\d+/]{3}=)?\\s*$", - flags: "i", - args: ["N-ZA-Mn-za-m0-9+/=", true] - }, - { - match: "^\\s*[A-Z\\d./]{20,}\\s*$", - flags: "i", - args: ["./0-9A-Za-z", true] - }, - ]; + this.checks = { + input: { + regex: [ + { + match: "^\\s*(?:[A-Z\\d+/]{4})+(?:[A-Z\\d+/]{2}==|[A-Z\\d+/]{3}=)?\\s*$", + flags: "i", + args: ["A-Za-z0-9+/=", true] + }, + { + match: "^\\s*[A-Z\\d\\-_]{20,}\\s*$", + flags: "i", + args: ["A-Za-z0-9-_", true] + }, + { + match: "^\\s*(?:[A-Z\\d+\\-]{4}){5,}(?:[A-Z\\d+\\-]{2}==|[A-Z\\d+\\-]{3}=)?\\s*$", + flags: "i", + args: ["A-Za-z0-9+\\-=", true] + }, + { + match: "^\\s*(?:[A-Z\\d./]{4}){5,}(?:[A-Z\\d./]{2}==|[A-Z\\d./]{3}=)?\\s*$", + flags: "i", + args: ["./0-9A-Za-z=", true] + }, + { + match: "^\\s*[A-Z\\d_.]{20,}\\s*$", + flags: "i", + args: ["A-Za-z0-9_.", true] + }, + { + match: "^\\s*(?:[A-Z\\d._]{4}){5,}(?:[A-Z\\d._]{2}--|[A-Z\\d._]{3}-)?\\s*$", + flags: "i", + args: ["A-Za-z0-9._-", true] + }, + { + match: "^\\s*(?:[A-Z\\d+/]{4}){5,}(?:[A-Z\\d+/]{2}==|[A-Z\\d+/]{3}=)?\\s*$", + flags: "i", + args: ["0-9a-zA-Z+/=", true] + }, + { + match: "^\\s*(?:[A-Z\\d+/]{4}){5,}(?:[A-Z\\d+/]{2}==|[A-Z\\d+/]{3}=)?\\s*$", + flags: "i", + args: ["0-9A-Za-z+/=", true] + }, + { + match: "^[ !\"#$%&'()*+,\\-./\\d:;<=>?@A-Z[\\\\\\]^_]{20,}$", + flags: "", + args: [" -_", false] + }, + { + match: "^\\s*[A-Z\\d+\\-]{20,}\\s*$", + flags: "i", + args: ["+\\-0-9A-Za-z", true] + }, + { + match: "^\\s*[!\"#$%&'()*+,\\-0-689@A-NP-VX-Z[`a-fh-mp-r]{20,}\\s*$", + flags: "", + args: ["!-,-0-689@A-NP-VX-Z[`a-fh-mp-r", true] + }, + { + match: "^\\s*(?:[N-ZA-M\\d+/]{4}){5,}(?:[N-ZA-M\\d+/]{2}==|[N-ZA-M\\d+/]{3}=)?\\s*$", + flags: "i", + args: ["N-ZA-Mn-za-m0-9+/=", true] + }, + { + match: "^\\s*[A-Z\\d./]{20,}\\s*$", + flags: "i", + args: ["./0-9A-Za-z", true] + }, + ], + } + }; } /** diff --git a/src/core/operations/FromDecimal.mjs b/src/core/operations/FromDecimal.mjs index 4248ce948..e1904a47f 100644 --- a/src/core/operations/FromDecimal.mjs +++ b/src/core/operations/FromDecimal.mjs @@ -36,38 +36,42 @@ class FromDecimal extends Operation { "value": false } ]; - this.patterns = [ - { - match: "^(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])(?: (?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5]))*$", - flags: "", - args: ["Space", false] - }, - { - match: "^(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])(?:,(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5]))*$", - flags: "", - args: ["Comma", false] - }, - { - match: "^(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])(?:;(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5]))*$", - flags: "", - args: ["Semi-colon", false] - }, - { - match: "^(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])(?::(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5]))*$", - flags: "", - args: ["Colon", false] - }, - { - match: "^(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])(?:\\n(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5]))*$", - flags: "", - args: ["Line feed", false] - }, - { - match: "^(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])(?:\\r\\n(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5]))*$", - flags: "", - args: ["CRLF", false] - }, - ]; + this.checks = { + input: { + regex: [ + { + match: "^(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])(?: (?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5]))*$", + flags: "", + args: ["Space", false] + }, + { + match: "^(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])(?:,(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5]))*$", + flags: "", + args: ["Comma", false] + }, + { + match: "^(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])(?:;(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5]))*$", + flags: "", + args: ["Semi-colon", false] + }, + { + match: "^(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])(?::(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5]))*$", + flags: "", + args: ["Colon", false] + }, + { + match: "^(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])(?:\\n(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5]))*$", + flags: "", + args: ["Line feed", false] + }, + { + match: "^(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])(?:\\r\\n(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5]))*$", + flags: "", + args: ["CRLF", false] + } + ] + } + }; } /** diff --git a/src/core/operations/FromHTMLEntity.mjs b/src/core/operations/FromHTMLEntity.mjs index 3d53a0e72..b4c94f8d9 100644 --- a/src/core/operations/FromHTMLEntity.mjs +++ b/src/core/operations/FromHTMLEntity.mjs @@ -25,13 +25,17 @@ class FromHTMLEntity extends Operation { this.inputType = "string"; this.outputType = "string"; this.args = []; - this.patterns = [ - { - match: "&(?:#\\d{2,3}|#x[\\da-f]{2}|[a-z]{2,6});", - flags: "i", - args: [] - }, - ]; + this.checks = { + input: { + regex: [ + { + match: "&(?:#\\d{2,3}|#x[\\da-f]{2}|[a-z]{2,6});", + flags: "i", + args: [] + } + ] + } + }; } /** diff --git a/src/core/operations/FromHex.mjs b/src/core/operations/FromHex.mjs index 6f70de9a6..94d540092 100644 --- a/src/core/operations/FromHex.mjs +++ b/src/core/operations/FromHex.mjs @@ -32,53 +32,57 @@ class FromHex extends Operation { value: FROM_HEX_DELIM_OPTIONS } ]; - this.patterns = [ - { - match: "^(?:[\\dA-F]{2})+$", - flags: "i", - args: ["None"] - }, - { - match: "^[\\dA-F]{2}(?: [\\dA-F]{2})*$", - flags: "i", - args: ["Space"] - }, - { - match: "^[\\dA-F]{2}(?:,[\\dA-F]{2})*$", - flags: "i", - args: ["Comma"] - }, - { - match: "^[\\dA-F]{2}(?:;[\\dA-F]{2})*$", - flags: "i", - args: ["Semi-colon"] - }, - { - match: "^[\\dA-F]{2}(?::[\\dA-F]{2})*$", - flags: "i", - args: ["Colon"] - }, - { - match: "^[\\dA-F]{2}(?:\\n[\\dA-F]{2})*$", - flags: "i", - args: ["Line feed"] - }, - { - match: "^[\\dA-F]{2}(?:\\r\\n[\\dA-F]{2})*$", - flags: "i", - args: ["CRLF"] - }, - { - match: "^[\\dA-F]{2}(?:0x[\\dA-F]{2})*$", - flags: "i", - args: ["0x"] - }, - { - match: "^[\\dA-F]{2}(?:\\\\x[\\dA-F]{2})*$", - flags: "i", - args: ["\\x"] + this.checks = { + input: { + regex: [ + { + match: "^(?:[\\dA-F]{2})+$", + flags: "i", + args: ["None"] + }, + { + match: "^[\\dA-F]{2}(?: [\\dA-F]{2})*$", + flags: "i", + args: ["Space"] + }, + { + match: "^[\\dA-F]{2}(?:,[\\dA-F]{2})*$", + flags: "i", + args: ["Comma"] + }, + { + match: "^[\\dA-F]{2}(?:;[\\dA-F]{2})*$", + flags: "i", + args: ["Semi-colon"] + }, + { + match: "^[\\dA-F]{2}(?::[\\dA-F]{2})*$", + flags: "i", + args: ["Colon"] + }, + { + match: "^[\\dA-F]{2}(?:\\n[\\dA-F]{2})*$", + flags: "i", + args: ["Line feed"] + }, + { + match: "^[\\dA-F]{2}(?:\\r\\n[\\dA-F]{2})*$", + flags: "i", + args: ["CRLF"] + }, + { + match: "^[\\dA-F]{2}(?:0x[\\dA-F]{2})*$", + flags: "i", + args: ["0x"] + }, + { + match: "^[\\dA-F]{2}(?:\\\\x[\\dA-F]{2})*$", + flags: "i", + args: ["\\x"] + } + ] } - ]; + }; } /** diff --git a/src/core/operations/FromHexContent.mjs b/src/core/operations/FromHexContent.mjs index deb101bfd..05f5087cb 100644 --- a/src/core/operations/FromHexContent.mjs +++ b/src/core/operations/FromHexContent.mjs @@ -26,6 +26,17 @@ class FromHexContent extends Operation { this.inputType = "string"; this.outputType = "byteArray"; this.args = []; + this.checks = { + input: { + regex: [ + { + match: "^\\s*.*?\\|([0-9a-f]{2})+\\|.*$", + flags: "i", + args: [] + } + ], + } + }; } /** diff --git a/src/core/operations/FromHexdump.mjs b/src/core/operations/FromHexdump.mjs index 65889a4b3..b2d8cfb4f 100644 --- a/src/core/operations/FromHexdump.mjs +++ b/src/core/operations/FromHexdump.mjs @@ -27,13 +27,17 @@ class FromHexdump extends Operation { this.inputType = "string"; this.outputType = "byteArray"; this.args = []; - this.patterns = [ - { - match: "^(?:(?:[\\dA-F]{4,16}h?:?)?[ \\t]*((?:[\\dA-F]{2} ){1,8}(?:[ \\t]|[\\dA-F]{2}-)(?:[\\dA-F]{2} ){1,8}|(?:[\\dA-F]{4} )*[\\dA-F]{4}|(?:[\\dA-F]{2} )*[\\dA-F]{2})[^\\n]*\\n?){2,}$", - flags: "i", - args: [] - }, - ]; + this.checks = { + input: { + regex: [ + { + match: "^(?:(?:[\\dA-F]{4,16}h?:?)?[ \\t]*((?:[\\dA-F]{2} ){1,8}(?:[ \\t]|[\\dA-F]{2}-)(?:[\\dA-F]{2} ){1,8}|(?:[\\dA-F]{4} )*[\\dA-F]{4}|(?:[\\dA-F]{2} )*[\\dA-F]{2})[^\\n]*\\n?){2,}$", + flags: "i", + args: [] + }, + ] + } + }; } /** diff --git a/src/core/operations/FromMorseCode.mjs b/src/core/operations/FromMorseCode.mjs index 3a2c40064..98a1dfdb2 100644 --- a/src/core/operations/FromMorseCode.mjs +++ b/src/core/operations/FromMorseCode.mjs @@ -37,13 +37,17 @@ class FromMorseCode extends Operation { "value": WORD_DELIM_OPTIONS } ]; - this.patterns = [ - { - match: "(?:^[-. \\n]{5,}$|^[_. \\n]{5,}$|^(?:dash|dot| |\\n){5,}$)", - flags: "i", - args: ["Space", "Line feed"] - }, - ]; + this.checks = { + input: { + regex: [ + { + match: "(?:^[-. \\n]{5,}$|^[_. \\n]{5,}$|^(?:dash|dot| |\\n){5,}$)", + flags: "i", + args: ["Space", "Line feed"] + } + ] + } + }; } /** diff --git a/src/core/operations/FromOctal.mjs b/src/core/operations/FromOctal.mjs index 6ff679657..f7ecde371 100644 --- a/src/core/operations/FromOctal.mjs +++ b/src/core/operations/FromOctal.mjs @@ -32,38 +32,42 @@ class FromOctal extends Operation { "value": DELIM_OPTIONS } ]; - this.patterns = [ - { - match: "^(?:[0-7]{1,2}|[123][0-7]{2})(?: (?:[0-7]{1,2}|[123][0-7]{2}))*$", - flags: "", - args: ["Space"] - }, - { - match: "^(?:[0-7]{1,2}|[123][0-7]{2})(?:,(?:[0-7]{1,2}|[123][0-7]{2}))*$", - flags: "", - args: ["Comma"] - }, - { - match: "^(?:[0-7]{1,2}|[123][0-7]{2})(?:;(?:[0-7]{1,2}|[123][0-7]{2}))*$", - flags: "", - args: ["Semi-colon"] - }, - { - match: "^(?:[0-7]{1,2}|[123][0-7]{2})(?::(?:[0-7]{1,2}|[123][0-7]{2}))*$", - flags: "", - args: ["Colon"] - }, - { - match: "^(?:[0-7]{1,2}|[123][0-7]{2})(?:\\n(?:[0-7]{1,2}|[123][0-7]{2}))*$", - flags: "", - args: ["Line feed"] - }, - { - match: "^(?:[0-7]{1,2}|[123][0-7]{2})(?:\\r\\n(?:[0-7]{1,2}|[123][0-7]{2}))*$", - flags: "", - args: ["CRLF"] - }, - ]; + this.checks = { + input: { + regex: [ + { + match: "^(?:[0-7]{1,2}|[123][0-7]{2})(?: (?:[0-7]{1,2}|[123][0-7]{2}))*$", + flags: "", + args: ["Space"] + }, + { + match: "^(?:[0-7]{1,2}|[123][0-7]{2})(?:,(?:[0-7]{1,2}|[123][0-7]{2}))*$", + flags: "", + args: ["Comma"] + }, + { + match: "^(?:[0-7]{1,2}|[123][0-7]{2})(?:;(?:[0-7]{1,2}|[123][0-7]{2}))*$", + flags: "", + args: ["Semi-colon"] + }, + { + match: "^(?:[0-7]{1,2}|[123][0-7]{2})(?::(?:[0-7]{1,2}|[123][0-7]{2}))*$", + flags: "", + args: ["Colon"] + }, + { + match: "^(?:[0-7]{1,2}|[123][0-7]{2})(?:\\n(?:[0-7]{1,2}|[123][0-7]{2}))*$", + flags: "", + args: ["Line feed"] + }, + { + match: "^(?:[0-7]{1,2}|[123][0-7]{2})(?:\\r\\n(?:[0-7]{1,2}|[123][0-7]{2}))*$", + flags: "", + args: ["CRLF"] + } + ] + } + }; } /** diff --git a/src/core/operations/FromQuotedPrintable.mjs b/src/core/operations/FromQuotedPrintable.mjs index 138fec27e..0ff1a6258 100644 --- a/src/core/operations/FromQuotedPrintable.mjs +++ b/src/core/operations/FromQuotedPrintable.mjs @@ -28,13 +28,17 @@ class FromQuotedPrintable extends Operation { this.inputType = "string"; this.outputType = "byteArray"; this.args = []; - this.patterns = [ - { - match: "^[\\x21-\\x3d\\x3f-\\x7e \\t]{0,76}(?:=[\\da-f]{2}|=\\r?\\n)(?:[\\x21-\\x3d\\x3f-\\x7e \\t]|=[\\da-f]{2}|=\\r?\\n)*$", - flags: "i", - args: [] - }, - ]; + this.checks = { + input: { + regex: [ + { + match: "^[\\x21-\\x3d\\x3f-\\x7e \\t]{0,76}(?:=[\\da-f]{2}|=\\r?\\n)(?:[\\x21-\\x3d\\x3f-\\x7e \\t]|=[\\da-f]{2}|=\\r?\\n)*$", + flags: "i", + args: [] + }, + ] + } + }; } /** diff --git a/src/core/operations/FromUNIXTimestamp.mjs b/src/core/operations/FromUNIXTimestamp.mjs index ff390c58d..57681ad29 100644 --- a/src/core/operations/FromUNIXTimestamp.mjs +++ b/src/core/operations/FromUNIXTimestamp.mjs @@ -33,28 +33,32 @@ class FromUNIXTimestamp extends Operation { "value": UNITS } ]; - this.patterns = [ - { - match: "^1?\\d{9}$", - flags: "", - args: ["Seconds (s)"] - }, - { - match: "^1?\\d{12}$", - flags: "", - args: ["Milliseconds (ms)"] - }, - { - match: "^1?\\d{15}$", - flags: "", - args: ["Microseconds (μs)"] - }, - { - match: "^1?\\d{18}$", - flags: "", - args: ["Nanoseconds (ns)"] - }, - ]; + this.checks = { + input: { + regex: [ + { + match: "^1?\\d{9}$", + flags: "", + args: ["Seconds (s)"] + }, + { + match: "^1?\\d{12}$", + flags: "", + args: ["Milliseconds (ms)"] + }, + { + match: "^1?\\d{15}$", + flags: "", + args: ["Microseconds (μs)"] + }, + { + match: "^1?\\d{18}$", + flags: "", + args: ["Nanoseconds (ns)"] + } + ] + } + }; } /** diff --git a/src/core/operations/Gunzip.mjs b/src/core/operations/Gunzip.mjs index ef487b063..9e6013db7 100644 --- a/src/core/operations/Gunzip.mjs +++ b/src/core/operations/Gunzip.mjs @@ -27,13 +27,17 @@ class Gunzip extends Operation { this.inputType = "ArrayBuffer"; this.outputType = "ArrayBuffer"; this.args = []; - this.patterns = [ - { - match: "^\\x1f\\x8b\\x08", - flags: "", - args: [] - }, - ]; + this.checks = { + input: { + regex: [ + { + match: "^\\x1f\\x8b\\x08", + flags: "", + args: [] + } + ] + } + }; } /** diff --git a/src/core/operations/ObjectIdentifierToHex.mjs b/src/core/operations/ObjectIdentifierToHex.mjs index 3e78cc034..b0b7c5321 100644 --- a/src/core/operations/ObjectIdentifierToHex.mjs +++ b/src/core/operations/ObjectIdentifierToHex.mjs @@ -25,6 +25,17 @@ class ObjectIdentifierToHex extends Operation { this.inputType = "string"; this.outputType = "string"; this.args = []; + this.checks = { + input: { + regex: [ + { + match: "^\\s*([0-9]{1,3}\\.)+[0-9]{1,3}\\s*$", + flags: "", + args: [] + } + ] + } + }; } /** diff --git a/src/core/operations/ParseSSHHostKey.mjs b/src/core/operations/ParseSSHHostKey.mjs index c9b0c2950..efc03948d 100644 --- a/src/core/operations/ParseSSHHostKey.mjs +++ b/src/core/operations/ParseSSHHostKey.mjs @@ -38,6 +38,17 @@ class ParseSSHHostKey extends Operation { ] } ]; + this.checks = { + input: { + regex: [ + { + match: "^\\s*([A-F\\d]{2}[,;:]){15,}[A-F\\d]{2}\\s*$", + flags: "i", + args: ["Hex"] + } + ] + } + }; } /** diff --git a/src/core/operations/ParseUNIXFilePermissions.mjs b/src/core/operations/ParseUNIXFilePermissions.mjs index 528e94b5b..14263834e 100644 --- a/src/core/operations/ParseUNIXFilePermissions.mjs +++ b/src/core/operations/ParseUNIXFilePermissions.mjs @@ -25,6 +25,17 @@ class ParseUNIXFilePermissions extends Operation { this.inputType = "string"; this.outputType = "string"; this.args = []; + this.checks = { + input: { + regex: [ + { + match: "^\\s*d[rxw-]{9}\\s*$", + flags: "", + args: [] + } + ] + } + }; } /** diff --git a/src/core/operations/ParseUserAgent.mjs b/src/core/operations/ParseUserAgent.mjs index 2c0d2c56c..f94532b03 100644 --- a/src/core/operations/ParseUserAgent.mjs +++ b/src/core/operations/ParseUserAgent.mjs @@ -25,6 +25,17 @@ class ParseUserAgent extends Operation { this.inputType = "string"; this.outputType = "string"; this.args = []; + this.checks = { + input: { + regex: [ + { + match: "^(User-Agent:|Mozilla\\/)[^\\n\\r]+\\s*$", + flags: "i", + args: [] + } + ] + } + }; } /** diff --git a/src/core/operations/ParseX509Certificate.mjs b/src/core/operations/ParseX509Certificate.mjs index 0a1a162e8..d551a903f 100644 --- a/src/core/operations/ParseX509Certificate.mjs +++ b/src/core/operations/ParseX509Certificate.mjs @@ -35,15 +35,17 @@ class ParseX509Certificate extends Operation { "value": ["PEM", "DER Hex", "Base64", "Raw"] } ]; - this.patterns = [ - { - "match": "^-+BEGIN CERTIFICATE-+\\r?\\n[\\da-z+/\\n\\r]+-+END CERTIFICATE-+\\r?\\n?$", - "flags": "i", - "args": [ - "PEM" + this.checks = { + input: { + regex: [ + { + "match": "^-+BEGIN CERTIFICATE-+\\r?\\n[\\da-z+/\\n\\r]+-+END CERTIFICATE-+\\r?\\n?$", + "flags": "i", + "args": ["PEM"] + } ] } - ]; + }; } /** diff --git a/src/core/operations/RawInflate.mjs b/src/core/operations/RawInflate.mjs index f8a938c51..b897a1782 100644 --- a/src/core/operations/RawInflate.mjs +++ b/src/core/operations/RawInflate.mjs @@ -8,6 +8,7 @@ import Operation from "../Operation.mjs"; import {INFLATE_BUFFER_TYPE} from "../lib/Zlib.mjs"; import rawinflate from "zlibjs/bin/rawinflate.min.js"; import OperationError from "../errors/OperationError.mjs"; +import * as criteria from "../lib/MagicCriteria.mjs"; const Zlib = rawinflate.Zlib; @@ -60,6 +61,14 @@ class RawInflate extends Operation { value: false } ]; + this.checks = { + input: { + entropy: { + input: [7.5, 8], + args: [0, 0, INFLATE_BUFFER_TYPE, false, false] + } + } + }; } /** diff --git a/src/core/operations/RegularExpression.mjs b/src/core/operations/RegularExpression.mjs index 5327870b4..5239e55f6 100644 --- a/src/core/operations/RegularExpression.mjs +++ b/src/core/operations/RegularExpression.mjs @@ -163,7 +163,7 @@ class RegularExpression extends Operation { case "List matches with capture groups": return Utils.escapeHtml(regexList(input, regex, displayTotal, true, true)); default: - return "Error: Invalid output format"; + throw new OperationError("Error: Invalid output format"); } } catch (err) { throw new OperationError("Invalid regex. Details: " + err.message); diff --git a/src/core/operations/RenderImage.mjs b/src/core/operations/RenderImage.mjs index 2401a90bf..1616d75eb 100644 --- a/src/core/operations/RenderImage.mjs +++ b/src/core/operations/RenderImage.mjs @@ -35,14 +35,21 @@ class RenderImage extends Operation { "value": ["Raw", "Base64", "Hex"] } ]; - this.patterns = [ - { - "match": "^(?:\\xff\\xd8\\xff|\\x89\\x50\\x4e\\x47|\\x47\\x49\\x46|.{8}\\x57\\x45\\x42\\x50|\\x42\\x4d)", - "flags": "", - "args": ["Raw"], - "useful": true + this.checks = { + input: { + regex: [ + { + "match": "^(?:\\xff\\xd8\\xff|\\x89\\x50\\x4e\\x47|\\x47\\x49\\x46|.{8}\\x57\\x45\\x42\\x50|\\x42\\x4d)", + "flags": "", + "args": ["Raw"], + "useful": true + } + ] + }, + output: { + mime: "image" } - ]; + }; } /** diff --git a/src/core/operations/StripHTMLTags.mjs b/src/core/operations/StripHTMLTags.mjs index 6935c1c06..f456f7202 100644 --- a/src/core/operations/StripHTMLTags.mjs +++ b/src/core/operations/StripHTMLTags.mjs @@ -35,6 +35,17 @@ class StripHTMLTags extends Operation { "value": true } ]; + this.checks = { + input: { + regex: [ + { + match: "^(\\S|\\s)*$", + flags: "i", + args: [true, true] + } + ] + } + }; } /** diff --git a/src/core/operations/StripHTTPHeaders.mjs b/src/core/operations/StripHTTPHeaders.mjs index e43360ed3..9cb811a1c 100644 --- a/src/core/operations/StripHTTPHeaders.mjs +++ b/src/core/operations/StripHTTPHeaders.mjs @@ -24,6 +24,17 @@ class StripHTTPHeaders extends Operation { this.inputType = "string"; this.outputType = "string"; this.args = []; + this.checks = { + input: { + regex: [ + { + match: "^\\s*HTTP(.|\\s)+?(\\r?\\n){2}", + flags: "", + args: [] + } + ] + } + }; } /** diff --git a/src/core/operations/URLDecode.mjs b/src/core/operations/URLDecode.mjs index 29f606238..33f3f2169 100644 --- a/src/core/operations/URLDecode.mjs +++ b/src/core/operations/URLDecode.mjs @@ -24,13 +24,17 @@ class URLDecode extends Operation { this.inputType = "string"; this.outputType = "string"; this.args = []; - this.patterns = [ - { - match: ".*(?:%[\\da-f]{2}.*){4}", - flags: "i", - args: [] - }, - ]; + this.checks = { + input: { + regex: [ + { + match: ".*(?:%[\\da-f]{2}.*){4}", + flags: "i", + args: [] + }, + ] + } + }; } /** diff --git a/src/core/operations/Untar.mjs b/src/core/operations/Untar.mjs index 78a469cee..deecbf7f4 100644 --- a/src/core/operations/Untar.mjs +++ b/src/core/operations/Untar.mjs @@ -27,13 +27,17 @@ class Untar extends Operation { this.outputType = "List"; this.presentType = "html"; this.args = []; - this.patterns = [ - { - "match": "^.{257}\\x75\\x73\\x74\\x61\\x72", - "flags": "", - "args": [] + this.checks = { + input: { + regex: [ + { + "match": "^.{257}\\x75\\x73\\x74\\x61\\x72", + "flags": "", + "args": [] + } + ] } - ]; + }; } /** diff --git a/src/core/operations/Unzip.mjs b/src/core/operations/Unzip.mjs index 3bca9401c..47126a43f 100644 --- a/src/core/operations/Unzip.mjs +++ b/src/core/operations/Unzip.mjs @@ -40,13 +40,17 @@ class Unzip extends Operation { value: false } ]; - this.patterns = [ - { - match: "^\\x50\\x4b(?:\\x03|\\x05|\\x07)(?:\\x04|\\x06|\\x08)", - flags: "", - args: ["", false] - }, - ]; + this.checks = { + input: { + regex: [ + { + match: "^\\x50\\x4b(?:\\x03|\\x05|\\x07)(?:\\x04|\\x06|\\x08)", + flags: "", + args: ["", false] + } + ] + } + }; } /** diff --git a/src/core/operations/ZlibInflate.mjs b/src/core/operations/ZlibInflate.mjs index 9f715c061..753e0ac92 100644 --- a/src/core/operations/ZlibInflate.mjs +++ b/src/core/operations/ZlibInflate.mjs @@ -59,13 +59,17 @@ class ZlibInflate extends Operation { value: false } ]; - this.patterns = [ - { - match: "^\\x78(\\x01|\\x9c|\\xda|\\x5e)", - flags: "", - args: [0, 0, "Adaptive", false, false] - }, - ]; + this.checks = { + input: { + regex: [ + { + match: "^\\x78(\\x01|\\x9c|\\xda|\\x5e)", + flags: "", + args: [0, 0, "Adaptive", false, false] + }, + ] + } + }; } /** From 2ba37af109e8236cc1e450867191fb1bddce5b8c Mon Sep 17 00:00:00 2001 From: n1073645 Date: Tue, 25 Feb 2020 11:33:35 +0000 Subject: [PATCH 02/10] extra signatures --- .../operations/EscapeUnicodeCharacters.mjs | 38 +++++---- src/core/operations/FromBinary.mjs | 78 ++++++++++--------- src/core/operations/ParseQRCode.mjs | 18 +++-- 3 files changed, 73 insertions(+), 61 deletions(-) diff --git a/src/core/operations/EscapeUnicodeCharacters.mjs b/src/core/operations/EscapeUnicodeCharacters.mjs index ad5ef3ea8..cbefd8c39 100644 --- a/src/core/operations/EscapeUnicodeCharacters.mjs +++ b/src/core/operations/EscapeUnicodeCharacters.mjs @@ -44,23 +44,27 @@ class EscapeUnicodeCharacters extends Operation { "value": true } ]; - this.patterns = [ - { - match: "\\\\u(?:[\\da-f]{4,6})", - flags: "i", - args: ["\\u"] - }, - { - match: "%u(?:[\\da-f]{4,6})", - flags: "i", - args: ["%u"] - }, - { - match: "U\\+(?:[\\da-f]{4,6})", - flags: "i", - args: ["U+"] - }, - ]; + this.checks = { + input: { + regex: [ + { + match: "\\\\u(?:[\\da-f]{4,6})", + flags: "i", + args: ["\\u"] + }, + { + match: "%u(?:[\\da-f]{4,6})", + flags: "i", + args: ["%u"] + }, + { + match: "U\\+(?:[\\da-f]{4,6})", + flags: "i", + args: ["U+"] + } + ] + } + }; } /** diff --git a/src/core/operations/FromBinary.mjs b/src/core/operations/FromBinary.mjs index e7ca50457..b7f4cc2ce 100644 --- a/src/core/operations/FromBinary.mjs +++ b/src/core/operations/FromBinary.mjs @@ -33,43 +33,47 @@ class FromBinary extends Operation { "value": BIN_DELIM_OPTIONS } ]; - this.patterns = [ - { - match: "^(?:[01]{8})+$", - flags: "", - args: ["None"] - }, - { - match: "^(?:[01]{8})(?: [01]{8})*$", - flags: "", - args: ["Space"] - }, - { - match: "^(?:[01]{8})(?:,[01]{8})*$", - flags: "", - args: ["Comma"] - }, - { - match: "^(?:[01]{8})(?:;[01]{8})*$", - flags: "", - args: ["Semi-colon"] - }, - { - match: "^(?:[01]{8})(?::[01]{8})*$", - flags: "", - args: ["Colon"] - }, - { - match: "^(?:[01]{8})(?:\\n[01]{8})*$", - flags: "", - args: ["Line feed"] - }, - { - match: "^(?:[01]{8})(?:\\r\\n[01]{8})*$", - flags: "", - args: ["CRLF"] - }, - ]; + this.checks = { + input: { + regex: [ + { + match: "^(?:[01]{8})+$", + flags: "", + args: ["None"] + }, + { + match: "^(?:[01]{8})(?: [01]{8})*$", + flags: "", + args: ["Space"] + }, + { + match: "^(?:[01]{8})(?:,[01]{8})*$", + flags: "", + args: ["Comma"] + }, + { + match: "^(?:[01]{8})(?:;[01]{8})*$", + flags: "", + args: ["Semi-colon"] + }, + { + match: "^(?:[01]{8})(?::[01]{8})*$", + flags: "", + args: ["Colon"] + }, + { + match: "^(?:[01]{8})(?:\\n[01]{8})*$", + flags: "", + args: ["Line feed"] + }, + { + match: "^(?:[01]{8})(?:\\r\\n[01]{8})*$", + flags: "", + args: ["CRLF"] + }, + ] + } + }; } /** diff --git a/src/core/operations/ParseQRCode.mjs b/src/core/operations/ParseQRCode.mjs index 6f34a6d01..a0691fed9 100644 --- a/src/core/operations/ParseQRCode.mjs +++ b/src/core/operations/ParseQRCode.mjs @@ -33,14 +33,18 @@ class ParseQRCode extends Operation { "value": false } ]; - this.patterns = [ - { - "match": "^(?:\\xff\\xd8\\xff|\\x89\\x50\\x4e\\x47|\\x47\\x49\\x46|.{8}\\x57\\x45\\x42\\x50|\\x42\\x4d)", - "flags": "", - "args": [false], - "useful": true + this.checks = { + input: { + regex: [ + { + "match": "^(?:\\xff\\xd8\\xff|\\x89\\x50\\x4e\\x47|\\x47\\x49\\x46|.{8}\\x57\\x45\\x42\\x50|\\x42\\x4d)", + "flags": "", + "args": [false], + "useful": true + } + ] } - ]; + }; } /** From 20d0ae53049cf4664a902e3e060ca4463803c5c2 Mon Sep 17 00:00:00 2001 From: n1073645 Date: Tue, 25 Feb 2020 11:35:39 +0000 Subject: [PATCH 03/10] Linting corrections --- src/core/operations/RawInflate.mjs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/core/operations/RawInflate.mjs b/src/core/operations/RawInflate.mjs index b897a1782..a50ffeb85 100644 --- a/src/core/operations/RawInflate.mjs +++ b/src/core/operations/RawInflate.mjs @@ -8,7 +8,6 @@ import Operation from "../Operation.mjs"; import {INFLATE_BUFFER_TYPE} from "../lib/Zlib.mjs"; import rawinflate from "zlibjs/bin/rawinflate.min.js"; import OperationError from "../errors/OperationError.mjs"; -import * as criteria from "../lib/MagicCriteria.mjs"; const Zlib = rawinflate.Zlib; From 99415359d0871719824affb29a36a665297fbb45 Mon Sep 17 00:00:00 2001 From: n1073645 Date: Tue, 10 Mar 2020 09:39:13 +0000 Subject: [PATCH 04/10] Extra Magic Tests --- tests/operations/tests/Magic.mjs | 55 ++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/tests/operations/tests/Magic.mjs b/tests/operations/tests/Magic.mjs index d168c92de..d07d8ec88 100644 --- a/tests/operations/tests/Magic.mjs +++ b/tests/operations/tests/Magic.mjs @@ -54,4 +54,59 @@ TestRegister.addTests([ } ], }, + { + name: "Magic Chain of Base64", + input: "WkVkV2VtUkRRbnBrU0Vwd1ltMWpQUT09", + expectedMatch: /From_Base64\('A-Za-z0-9\+\/=',true\)\nFrom_Base64\('A-Za-z0-9\+\/=',true\)\nFrom_Base64\('A-Za-z0-9\+\/=',true\)/, + recipeConfig: [ + { + op: "Magic", + args: [3, true, false] + } + ], + }, + { + name: "Magic Chain of Hex to Hexdump to Base64", + input: "MDAwMDAwMDAgIDM3IDM0IDIwIDM2IDM1IDIwIDM3IDMzIDIwIDM3IDM0IDIwIDMyIDMwIDIwIDM3ICB8NzQgNjUgNzMgNzQgMjAgN3wKMDAwMDAwMTAgIDMzIDIwIDM3IDM0IDIwIDM3IDMyIDIwIDM2IDM5IDIwIDM2IDY1IDIwIDM2IDM3ICB8MyA3NCA3MiA2OSA2ZSA2N3w=", + expectedMatch: /From_Base64\('A-Za-z0-9\+\/=',true\)\nFrom_Hexdump\(\)\nFrom_Hex\('Space'\)/, + recipeConfig: [ + { + op: "Magic", + args: [3, true, false] + } + ], + }, + { + name: "Magic Chain of Charcode to Octal to Base32", + input: "GY3SANRUEA2DAIBWGYQDMNJAGQYCANRXEA3DGIBUGAQDMNZAGY2CANBQEA3DEIBWGAQDIMBAGY3SANRTEA2DAIBWG4QDMNBAGQYCANRXEA3DEIBUGAQDMNRAG4YSANBQEA3DMIBRGQ2SANBQEA3DMIBWG4======", + expectedMatch: /From_Base32\('A-Z2-7=',false\)\nFrom_Octal\('Space'\)\nFrom_Hex\('Space'\)/, + recipeConfig: [ + { + op: "Magic", + args: [3, true, false] + } + ], + }, + { + name: "Magic Chain of Base64 Output Check", + input: "WkVkV2VtUkRRbnBrU0Vwd1ltMWpQUT09", + expectedMatch: /test string/, + recipeConfig: [ + { + op: "Magic", + args: [3, true, false] + } + ], + }, + { + name: "Magic Chain of Decimal to Base32 to Base32", + input: "I5CVSVCNJFBFER2BLFJUCTKKKJDVKUKEINGUUV2FIFNFIRKJIJJEORJSKNAU2SSSI5MVCRCDJVFFKRKBLFKECTSKIFDUKWKUIFEUEUSHIFNFCPJ5HU6Q====", + expectedMatch: /test string/, + recipeConfig: [ + { + op: "Magic", + args: [3, true, false] + } + ], + }, ]); From 3f3a7cd4f603224796050a85da8211099e91b461 Mon Sep 17 00:00:00 2001 From: n1073645 Date: Tue, 10 Mar 2020 11:12:43 +0000 Subject: [PATCH 05/10] From Hex Regexes --- src/core/Utils.mjs | 1 + src/core/operations/FromHex.mjs | 9 +++++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/core/Utils.mjs b/src/core/Utils.mjs index c99eccc93..634068c3f 100755 --- a/src/core/Utils.mjs +++ b/src/core/Utils.mjs @@ -1182,6 +1182,7 @@ class Utils { "CRLF": /\r\n/g, "Forward slash": /\//g, "Backslash": /\\/g, + "0x with comma": /,?0x/g, "0x": /0x/g, "\\x": /\\x/g, "None": /\s+/g // Included here to remove whitespace when there shouldn't be any diff --git a/src/core/operations/FromHex.mjs b/src/core/operations/FromHex.mjs index 94d540092..cccd78129 100644 --- a/src/core/operations/FromHex.mjs +++ b/src/core/operations/FromHex.mjs @@ -71,12 +71,17 @@ class FromHex extends Operation { args: ["CRLF"] }, { - match: "^[\\dA-F]{2}(?:0x[\\dA-F]{2})*$", + match: "^(?:0x[\\dA-F]{2})+$", flags: "i", args: ["0x"] }, { - match: "^[\\dA-F]{2}(?:\\\\x[\\dA-F]{2})*$", + match: "^0x[\\dA-F]{2}(?:,0x[\\dA-F]{2})*$", + flags: "i", + args: ["0x with comma"] + }, + { + match: "^(?:\\\\x[\\dA-F]{2})+$", flags: "i", args: ["\\x"] } From 0a064726397bc29c19a08e022964f394a4b1e7e1 Mon Sep 17 00:00:00 2001 From: n1073645 Date: Tue, 10 Mar 2020 11:23:14 +0000 Subject: [PATCH 06/10] Test added for From Hex --- tests/operations/tests/Hex.mjs | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/tests/operations/tests/Hex.mjs b/tests/operations/tests/Hex.mjs index 91520d886..3bb895440 100644 --- a/tests/operations/tests/Hex.mjs +++ b/tests/operations/tests/Hex.mjs @@ -92,6 +92,19 @@ TestRegister.addTests([ ] } ] + }, + { + name: "0x with Comma to Ascii", + input: "0x74,0x65,0x73,0x74,0x20,0x73,0x74,0x72,0x69,0x6e,0x67", + expectedOutput: "test string", + recipeConfig: [ + { + "op": "From Hex", + "args": [ + "0x with comma" + ] + } + ] - } + }, ]); From fd7176a445b7645afbd175448614d42432be1093 Mon Sep 17 00:00:00 2001 From: n1073645 Date: Wed, 11 Mar 2020 12:51:46 +0000 Subject: [PATCH 07/10] Extra Magic Tests --- tests/operations/tests/Magic.mjs | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/operations/tests/Magic.mjs b/tests/operations/tests/Magic.mjs index d07d8ec88..e608c3f64 100644 --- a/tests/operations/tests/Magic.mjs +++ b/tests/operations/tests/Magic.mjs @@ -109,4 +109,26 @@ TestRegister.addTests([ } ], }, + { + name: "Raw Inflate from Entropy", + input: "\x4d\x52\xb1\x6e\xdc\x30\x0c\xdd\xf3\x15\x44\x80\x6e\xae\x91\x02\x4d\x80\x8e\x4d\x9a\x21\x53\x8b\xa6\x43\x56\x5a\xe2\x9d\x84\x93\x25\x43\x94\xed\xf8\xef\xf3\xe8\x6b\x0e\xb7\x1c\xce\xd4\x7b\x8f\x8f\x7c\x7c\xda\x06\xa9\x4f\x41\x0e\x14\x95\x98\x34\x8e\x53\x92\x8e\x62\x6e\x73\x6c\x71\x11\x5a\x65\x20\x9e\x26\x3a\x94\x4a\x8e\x6b\xdd\x62\x3e\x52\x99\x1b\x71\x4a\x34\x72\xce\x52\xa9\x1c\xe8\xd6\x99\xd0\x2d\x95\x49\x2a\xb7\x58\xb2\xd2\x1a\x5b\x88\x19\xa2\x26\x31\xd4\xb2\xaa\xd4\x9e\xfe\x05\x51\xb9\x86\xc5\xec\xd2\xec\xe5\x7f\x6b\x92\xec\x8a\xb7\x1e\x29\x9e\x84\xde\x7e\xff\x25\x34\x7e\x64\x95\x87\xef\x1d\x8d\xa5\x0a\xb9\x62\xc0\x77\x43\xd6\x6d\x32\x91\x33\xf6\xe7\xf3\x6b\x47\xbf\x9e\x5f\x89\xb3\xa7\xc7\x54\xd6\x43\xd4\xd0\x91\xab\x82\x4e\x10\x1c\x62\xe6\xba\xed\xaf\x41\xde\xfd\x3c\x4e\x8a\x57\x88\x55\x51\x35\x15\x7b\xf1\x72\x5d\xc1\x60\x9e\x1b\x03\xc6\xc9\xcd\xe9\xac\x13\x58\x31\xc3\x8e\x76\x41\xdc\x49\xe7\x11\x42\x2f\x7f\x96\x87\xbd\xf6\xd6\xdf\xdf\xfd\xa0\x89\xab\x02\x0c\x66\xe0\x7c\x34\x1a\xfe\x54\x76\x0d\xeb\xfa\x1c\x11\x2c\x23\x8c\xb3\x0b\xfb\x64\xfd\xcd\x0d\xb6\x43\xad\x94\x64\x69\x78\xd1\x78\xcc\xe2\x51\x00\x85\x07\x2c\x67\x28\x2d\x50\x13\x17\x72\x84\xa3\x9d\x9d\x4b\xfe\x7a\x5d\xe1\xb4\x69\x53\xe3\x20\x9c\x38\x99\x69\xd9\x87\xc0\xa2\x2f\xab\x5b\x79\x3b\xe7\x63\x41\x06\x5e\xcc\x1f\x18\x5e\x20\x61\xe5\x0b\xd0\xbc\xa8\x25\xc0\xe9\x58\x2a\x5e\x46\xed\xe9\xa5\x41\x40\x81\xc9\x4e\x70\x22\xbe\xbb\x58\xed\x68\x98\x63\xc2\x6d\xc0\x18\x72\xad\x32\x4a\x6e\x38\x94\x8d\x10\x6e\x2d\xc0\xd2\x60\x09\x7c\xfa\x34\x4f\x2d\x48\xac\xf4\xed\xee\x0b\x3e\x72\x59\xf6\xab\xa0\x16\x47\x1c\xc9\x82\x65\xa9\xe0\x17\xb6\x36\xc1\x46\xfb\x0f", + expectedMatch: /#recipe=Raw_Inflate/, + recipeConfig: [ + { + op: "Magic", + args: [3, true, false] + } + ] + }, + { + name: "Raw Inflate Result Text", + input: "\x4d\x52\xb1\x6e\xdc\x30\x0c\xdd\xf3\x15\x44\x80\x6e\xae\x91\x02\x4d\x80\x8e\x4d\x9a\x21\x53\x8b\xa6\x43\x56\x5a\xe2\x9d\x84\x93\x25\x43\x94\xed\xf8\xef\xf3\xe8\x6b\x0e\xb7\x1c\xce\xd4\x7b\x8f\x8f\x7c\x7c\xda\x06\xa9\x4f\x41\x0e\x14\x95\x98\x34\x8e\x53\x92\x8e\x62\x6e\x73\x6c\x71\x11\x5a\x65\x20\x9e\x26\x3a\x94\x4a\x8e\x6b\xdd\x62\x3e\x52\x99\x1b\x71\x4a\x34\x72\xce\x52\xa9\x1c\xe8\xd6\x99\xd0\x2d\x95\x49\x2a\xb7\x58\xb2\xd2\x1a\x5b\x88\x19\xa2\x26\x31\xd4\xb2\xaa\xd4\x9e\xfe\x05\x51\xb9\x86\xc5\xec\xd2\xec\xe5\x7f\x6b\x92\xec\x8a\xb7\x1e\x29\x9e\x84\xde\x7e\xff\x25\x34\x7e\x64\x95\x87\xef\x1d\x8d\xa5\x0a\xb9\x62\xc0\x77\x43\xd6\x6d\x32\x91\x33\xf6\xe7\xf3\x6b\x47\xbf\x9e\x5f\x89\xb3\xa7\xc7\x54\xd6\x43\xd4\xd0\x91\xab\x82\x4e\x10\x1c\x62\xe6\xba\xed\xaf\x41\xde\xfd\x3c\x4e\x8a\x57\x88\x55\x51\x35\x15\x7b\xf1\x72\x5d\xc1\x60\x9e\x1b\x03\xc6\xc9\xcd\xe9\xac\x13\x58\x31\xc3\x8e\x76\x41\xdc\x49\xe7\x11\x42\x2f\x7f\x96\x87\xbd\xf6\xd6\xdf\xdf\xfd\xa0\x89\xab\x02\x0c\x66\xe0\x7c\x34\x1a\xfe\x54\x76\x0d\xeb\xfa\x1c\x11\x2c\x23\x8c\xb3\x0b\xfb\x64\xfd\xcd\x0d\xb6\x43\xad\x94\x64\x69\x78\xd1\x78\xcc\xe2\x51\x00\x85\x07\x2c\x67\x28\x2d\x50\x13\x17\x72\x84\xa3\x9d\x9d\x4b\xfe\x7a\x5d\xe1\xb4\x69\x53\xe3\x20\x9c\x38\x99\x69\xd9\x87\xc0\xa2\x2f\xab\x5b\x79\x3b\xe7\x63\x41\x06\x5e\xcc\x1f\x18\x5e\x20\x61\xe5\x0b\xd0\xbc\xa8\x25\xc0\xe9\x58\x2a\x5e\x46\xed\xe9\xa5\x41\x40\x81\xc9\x4e\x70\x22\xbe\xbb\x58\xed\x68\x98\x63\xc2\x6d\xc0\x18\x72\xad\x32\x4a\x6e\x38\x94\x8d\x10\x6e\x2d\xc0\xd2\x60\x09\x7c\xfa\x34\x4f\x2d\x48\xac\xf4\xed\xee\x0b\x3e\x72\x59\xf6\xab\xa0\x16\x47\x1c\xc9\x82\x65\xa9\xe0\x17\xb6\x36\xc1\x46\xfb\x0f", + expectedMatch: /CyberChef is a simple, intuitive web app for carrying out all manner of /, + recipeConfig: [ + { + op: "Magic", + args: [3, true, false] + } + ] + } ]); From 570a84b67a28aadeb1c893263d38b927eff02e97 Mon Sep 17 00:00:00 2001 From: n1073645 Date: Wed, 11 Mar 2020 16:27:37 +0000 Subject: [PATCH 08/10] More Magic tests --- tests/operations/tests/Magic.mjs | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/tests/operations/tests/Magic.mjs b/tests/operations/tests/Magic.mjs index e608c3f64..28950f4ee 100644 --- a/tests/operations/tests/Magic.mjs +++ b/tests/operations/tests/Magic.mjs @@ -109,25 +109,14 @@ TestRegister.addTests([ } ], }, - { - name: "Raw Inflate from Entropy", - input: "\x4d\x52\xb1\x6e\xdc\x30\x0c\xdd\xf3\x15\x44\x80\x6e\xae\x91\x02\x4d\x80\x8e\x4d\x9a\x21\x53\x8b\xa6\x43\x56\x5a\xe2\x9d\x84\x93\x25\x43\x94\xed\xf8\xef\xf3\xe8\x6b\x0e\xb7\x1c\xce\xd4\x7b\x8f\x8f\x7c\x7c\xda\x06\xa9\x4f\x41\x0e\x14\x95\x98\x34\x8e\x53\x92\x8e\x62\x6e\x73\x6c\x71\x11\x5a\x65\x20\x9e\x26\x3a\x94\x4a\x8e\x6b\xdd\x62\x3e\x52\x99\x1b\x71\x4a\x34\x72\xce\x52\xa9\x1c\xe8\xd6\x99\xd0\x2d\x95\x49\x2a\xb7\x58\xb2\xd2\x1a\x5b\x88\x19\xa2\x26\x31\xd4\xb2\xaa\xd4\x9e\xfe\x05\x51\xb9\x86\xc5\xec\xd2\xec\xe5\x7f\x6b\x92\xec\x8a\xb7\x1e\x29\x9e\x84\xde\x7e\xff\x25\x34\x7e\x64\x95\x87\xef\x1d\x8d\xa5\x0a\xb9\x62\xc0\x77\x43\xd6\x6d\x32\x91\x33\xf6\xe7\xf3\x6b\x47\xbf\x9e\x5f\x89\xb3\xa7\xc7\x54\xd6\x43\xd4\xd0\x91\xab\x82\x4e\x10\x1c\x62\xe6\xba\xed\xaf\x41\xde\xfd\x3c\x4e\x8a\x57\x88\x55\x51\x35\x15\x7b\xf1\x72\x5d\xc1\x60\x9e\x1b\x03\xc6\xc9\xcd\xe9\xac\x13\x58\x31\xc3\x8e\x76\x41\xdc\x49\xe7\x11\x42\x2f\x7f\x96\x87\xbd\xf6\xd6\xdf\xdf\xfd\xa0\x89\xab\x02\x0c\x66\xe0\x7c\x34\x1a\xfe\x54\x76\x0d\xeb\xfa\x1c\x11\x2c\x23\x8c\xb3\x0b\xfb\x64\xfd\xcd\x0d\xb6\x43\xad\x94\x64\x69\x78\xd1\x78\xcc\xe2\x51\x00\x85\x07\x2c\x67\x28\x2d\x50\x13\x17\x72\x84\xa3\x9d\x9d\x4b\xfe\x7a\x5d\xe1\xb4\x69\x53\xe3\x20\x9c\x38\x99\x69\xd9\x87\xc0\xa2\x2f\xab\x5b\x79\x3b\xe7\x63\x41\x06\x5e\xcc\x1f\x18\x5e\x20\x61\xe5\x0b\xd0\xbc\xa8\x25\xc0\xe9\x58\x2a\x5e\x46\xed\xe9\xa5\x41\x40\x81\xc9\x4e\x70\x22\xbe\xbb\x58\xed\x68\x98\x63\xc2\x6d\xc0\x18\x72\xad\x32\x4a\x6e\x38\x94\x8d\x10\x6e\x2d\xc0\xd2\x60\x09\x7c\xfa\x34\x4f\x2d\x48\xac\xf4\xed\xee\x0b\x3e\x72\x59\xf6\xab\xa0\x16\x47\x1c\xc9\x82\x65\xa9\xe0\x17\xb6\x36\xc1\x46\xfb\x0f", - expectedMatch: /#recipe=Raw_Inflate/, - recipeConfig: [ - { - op: "Magic", - args: [3, true, false] - } - ] - }, { name: "Raw Inflate Result Text", input: "\x4d\x52\xb1\x6e\xdc\x30\x0c\xdd\xf3\x15\x44\x80\x6e\xae\x91\x02\x4d\x80\x8e\x4d\x9a\x21\x53\x8b\xa6\x43\x56\x5a\xe2\x9d\x84\x93\x25\x43\x94\xed\xf8\xef\xf3\xe8\x6b\x0e\xb7\x1c\xce\xd4\x7b\x8f\x8f\x7c\x7c\xda\x06\xa9\x4f\x41\x0e\x14\x95\x98\x34\x8e\x53\x92\x8e\x62\x6e\x73\x6c\x71\x11\x5a\x65\x20\x9e\x26\x3a\x94\x4a\x8e\x6b\xdd\x62\x3e\x52\x99\x1b\x71\x4a\x34\x72\xce\x52\xa9\x1c\xe8\xd6\x99\xd0\x2d\x95\x49\x2a\xb7\x58\xb2\xd2\x1a\x5b\x88\x19\xa2\x26\x31\xd4\xb2\xaa\xd4\x9e\xfe\x05\x51\xb9\x86\xc5\xec\xd2\xec\xe5\x7f\x6b\x92\xec\x8a\xb7\x1e\x29\x9e\x84\xde\x7e\xff\x25\x34\x7e\x64\x95\x87\xef\x1d\x8d\xa5\x0a\xb9\x62\xc0\x77\x43\xd6\x6d\x32\x91\x33\xf6\xe7\xf3\x6b\x47\xbf\x9e\x5f\x89\xb3\xa7\xc7\x54\xd6\x43\xd4\xd0\x91\xab\x82\x4e\x10\x1c\x62\xe6\xba\xed\xaf\x41\xde\xfd\x3c\x4e\x8a\x57\x88\x55\x51\x35\x15\x7b\xf1\x72\x5d\xc1\x60\x9e\x1b\x03\xc6\xc9\xcd\xe9\xac\x13\x58\x31\xc3\x8e\x76\x41\xdc\x49\xe7\x11\x42\x2f\x7f\x96\x87\xbd\xf6\xd6\xdf\xdf\xfd\xa0\x89\xab\x02\x0c\x66\xe0\x7c\x34\x1a\xfe\x54\x76\x0d\xeb\xfa\x1c\x11\x2c\x23\x8c\xb3\x0b\xfb\x64\xfd\xcd\x0d\xb6\x43\xad\x94\x64\x69\x78\xd1\x78\xcc\xe2\x51\x00\x85\x07\x2c\x67\x28\x2d\x50\x13\x17\x72\x84\xa3\x9d\x9d\x4b\xfe\x7a\x5d\xe1\xb4\x69\x53\xe3\x20\x9c\x38\x99\x69\xd9\x87\xc0\xa2\x2f\xab\x5b\x79\x3b\xe7\x63\x41\x06\x5e\xcc\x1f\x18\x5e\x20\x61\xe5\x0b\xd0\xbc\xa8\x25\xc0\xe9\x58\x2a\x5e\x46\xed\xe9\xa5\x41\x40\x81\xc9\x4e\x70\x22\xbe\xbb\x58\xed\x68\x98\x63\xc2\x6d\xc0\x18\x72\xad\x32\x4a\x6e\x38\x94\x8d\x10\x6e\x2d\xc0\xd2\x60\x09\x7c\xfa\x34\x4f\x2d\x48\xac\xf4\xed\xee\x0b\x3e\x72\x59\xf6\xab\xa0\x16\x47\x1c\xc9\x82\x65\xa9\xe0\x17\xb6\x36\xc1\x46\xfb\x0f", - expectedMatch: /CyberChef is a simple, intuitive web app for carrying out all manner of /, + expectedMatch: /#recipe=Raw_Inflate(.|\n)+CyberChef is a simple, intuitive web app for carrying out all manner of /, recipeConfig: [ { op: "Magic", - args: [3, true, false] + args: [1, true, false] } ] } From 5b5105c86404ce11e7e646d61a16bf76f1cbd23b Mon Sep 17 00:00:00 2001 From: n1073645 Date: Thu, 12 Mar 2020 14:45:40 +0000 Subject: [PATCH 09/10] Caching added for Magic regexes --- src/core/lib/Magic.mjs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/core/lib/Magic.mjs b/src/core/lib/Magic.mjs index 6de148eb2..8ad5291f7 100644 --- a/src/core/lib/Magic.mjs +++ b/src/core/lib/Magic.mjs @@ -36,10 +36,10 @@ class Magic { const matches = []; for (let i = 0; i < opPatterns.length; i++) { - const pattern = opPatterns[i], - regex = new RegExp(pattern.match, pattern.flags); + const pattern = opPatterns[i]; - if (regex.test(this.inputStr)) { + + if (pattern.match.test(this.inputStr)) { matches.push(pattern); } } @@ -522,8 +522,7 @@ class Magic { OperationConfig[op].input.regex.forEach(pattern => { opCriteria.regex.push({ op: op, - match: pattern.match, - flags: pattern.flags, + match: new RegExp(pattern.match, pattern.flags), args: pattern.args, useful: pattern.useful || false }); From 5b6a53be3e98351648797b205e581b5f7c280b78 Mon Sep 17 00:00:00 2001 From: n1073645 Date: Thu, 12 Mar 2020 14:55:19 +0000 Subject: [PATCH 10/10] Docstrings added for Magic functions --- src/core/lib/Magic.mjs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/core/lib/Magic.mjs b/src/core/lib/Magic.mjs index 8ad5291f7..b4decbe98 100644 --- a/src/core/lib/Magic.mjs +++ b/src/core/lib/Magic.mjs @@ -29,8 +29,11 @@ class Magic { } /** + * Finds operations that claim to be able to decode the input based on + * regular expression matches. * - * @param opPatterns + * @param {[Object]} opPatterns + * @returns {Array} */ inputRegexMatch(opPatterns) { const matches = []; @@ -48,7 +51,11 @@ class Magic { } /** + * Finds operations that claim to be able to decode the input based on entropy + * matches. * + * @param {[Object]} opPatterns + * @returns {Array} */ entropyInputMatch(opPatterns) { const matches = []; @@ -64,8 +71,7 @@ class Magic { } /** - * Finds operations that claim to be able to decode the input based on regular - * expression matches. + * Finds operations that claim to be able to decode the input based on criteria. * * @returns {Object[]} */