Skip to content

Commit

Permalink
Merge branch 'newMagic' of https://github.com/n1073645/CyberChef into…
Browse files Browse the repository at this point in the history 
… n1073645-newMagic
  • Loading branch information
@n1474335
n1474335 committed Mar 20, 2020
2 parents 26b1935 + 5b6a53b commit 26fa66e
Show file tree
Hide file tree
Showing 44 changed files with 907 additions and 408 deletions.
1 change: 1 addition & 0 deletions src/core/Utils.mjs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1182,6 +1182,7 @@ class Utils {
"CRLF": /\r\n/g,
"Forward slash": /\//g,
"Backslash": /\\/g,
"0x with comma": /,?0x/g,
"0x": /0x/g,
"\\x": /\\x/g,
"None": /\s+/g // Included here to remove whitespace when there shouldn't be any
Expand Down
27 changes: 23 additions & 4 deletions src/core/config/scripts/generateConfig.mjs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,13 +42,32 @@ for (const opObj in Ops) {
outputType: op.presentType,
flowControl: op.flowControl,
manualBake: op.manualBake,
args: op.args
args: op.args,
};

if ("patterns" in op) {
operationConfig[op.name].patterns = op.patterns;
if ("checks" in op) {
if ("input" in op.checks) {
operationConfig[op.name].input = {};
if ("regex" in op.checks.input) {
operationConfig[op.name].input.regex = op.checks.input.regex;
}
if ("entropy" in op.checks.input) {
operationConfig[op.name].input.entropy = op.checks.input.entropy;
}
}
if ("output" in op.checks) {
operationConfig[op.name].output = {};
if ("regex" in op.checks.output) {
operationConfig[op.name].output.regex = op.checks.output.regex;
}
if ("entropy" in op.checks.output) {
operationConfig[op.name].output.entropy = op.checks.output.entropy;
}
if ("mime" in op.checks.output) {
operationConfig[op.name].output.mime = op.checks.output.mime;
}
}
}

if (!(op.module in modules))
modules[op.module] = {};
modules[op.module][op.name] = opObj;
Expand Down
8 changes: 4 additions & 4 deletions src/core/lib/IP.mjs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ export function ipv4CidrRange(cidr, includeNetworkInfo, enumerateAddresses, allo
let output = "";

if (cidrRange < 0 || cidrRange > 31) {
return "IPv4 CIDR must be less than 32";
throw new OperationError("IPv4 CIDR must be less than 32");
}

const mask = ~(0xFFFFFFFF >>> cidrRange),
Expand Down Expand Up @@ -64,7 +64,7 @@ export function ipv6CidrRange(cidr, includeNetworkInfo) {
cidrRange = parseInt(cidr[cidr.length-1], 10);

if (cidrRange < 0 || cidrRange > 127) {
return "IPv6 CIDR must be less than 128";
throw new OperationError("IPv6 CIDR must be less than 128");
}

const ip1 = new Array(8),
Expand Down Expand Up @@ -211,7 +211,7 @@ export function ipv4ListedRange(match, includeNetworkInfo, enumerateAddresses, a
const network = strToIpv4(ipv4CidrList[i].split("/")[0]);
const cidrRange = parseInt(ipv4CidrList[i].split("/")[1], 10);
if (cidrRange < 0 || cidrRange > 31) {
return "IPv4 CIDR must be less than 32";
throw new OperationError("IPv4 CIDR must be less than 32");
}
const mask = ~(0xFFFFFFFF >>> cidrRange),
cidrIp1 = network & mask,
Expand Down Expand Up @@ -254,7 +254,7 @@ export function ipv6ListedRange(match, includeNetworkInfo) {
const cidrRange = parseInt(ipv6CidrList[i].split("/")[1], 10);

if (cidrRange < 0 || cidrRange > 127) {
return "IPv6 CIDR must be less than 128";
throw new OperationError("IPv6 CIDR must be less than 128");
}

const cidrIp1 = new Array(8),
Expand Down
141 changes: 110 additions & 31 deletions src/core/lib/Magic.mjs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import OperationConfig from "../config/OperationConfig.json";
import Utils, { isWorkerEnvironment } from "../Utils.mjs";
import Recipe from "../Recipe.mjs";
import Dish from "../Dish.mjs";
import {detectFileType} from "./FileType.mjs";
import {detectFileType, isType} from "./FileType.mjs";
import chiSquared from "chi-squared";

/**
Expand All @@ -19,35 +19,68 @@ class Magic {
* Magic constructor.
*
* @param {ArrayBuffer} buf
* @param {Object[]} [opPatterns]
* @param {Object} prevOp
*/
constructor(buf, opPatterns) {
constructor(buf, opPatterns, prevOp) {
this.inputBuffer = new Uint8Array(buf);
this.inputStr = Utils.arrayBufferToStr(buf);
this.opPatterns = opPatterns || Magic._generateOpPatterns();
this.opPatterns = opPatterns || Magic._generateOpCriteria();
this.prevOp = prevOp;
}

/**
* Finds operations that claim to be able to decode the input based on regular
* expression matches.
* Finds operations that claim to be able to decode the input based on
* regular expression matches.
*
* @returns {Object[]}
* @param {[Object]} opPatterns
* @returns {Array}
*/
findMatchingOps() {
inputRegexMatch(opPatterns) {
const matches = [];

for (let i = 0; i < this.opPatterns.length; i++) {
const pattern = this.opPatterns[i],
regex = new RegExp(pattern.match, pattern.flags);
for (let i = 0; i < opPatterns.length; i++) {
const pattern = opPatterns[i];


if (regex.test(this.inputStr)) {
if (pattern.match.test(this.inputStr)) {
matches.push(pattern);
}
}

return matches;
}

/**
* Finds operations that claim to be able to decode the input based on entropy
* matches.
*
* @param {[Object]} opPatterns
* @returns {Array}
*/
entropyInputMatch(opPatterns) {
const matches = [];

const entropyOfInput = this.calcEntropy();

for (let i = 0; i < opPatterns.length; i++) {
const currOp = opPatterns[i];
if ((entropyOfInput > currOp.entropy[0]) && (entropyOfInput < currOp.entropy[1]))
matches.push(currOp);
}
return matches;
}

/**
* Finds operations that claim to be able to decode the input based on criteria.
*
* @returns {Object[]}
*/
findMatchingInputOps() {
let matches = this.inputRegexMatch(this.opPatterns.regex);
matches = matches.concat(this.entropyInputMatch(this.opPatterns.entropy));
return [...new Set(matches)];
}

/**
* Attempts to detect the language of the input by comparing its byte frequency
* to that of several known languages.
Expand Down Expand Up @@ -264,6 +297,35 @@ class Magic {
return results;
}

/**
*
*/
checkRegexes(regexes) {
for (const elem of regexes) {
const regex = new RegExp(elem.match, elem.flags);
if (regex.test(this.inputStr))
return true;
}
return false;
}
/**
*
*/
checkOutputFromPrevious() {
let score = 0;
if ("regex" in this.prevOp.output) {
if (this.checkRegexes(this.prevOp.output.regex)) score++;
}
if ("entropy" in this.prevOp.output) {
const inputEntropy = this.calcEntropy();
if ((inputEntropy > this.prevOp.output.entropy[0]) && (inputEntropy < this.prevOp.output.entropy[1])) score++;
}
if ("mime" in this.prevOp.output) {
if (isType(this.prevOp.output.mime, this.inputBuffer)) score++;
}
return score > 0;
}

/**
* Speculatively executes matching operations, recording metadata of each result.
*
Expand All @@ -281,8 +343,15 @@ class Magic {
if (depth < 0) return [];

// Find any operations that can be run on this data
const matchingOps = this.findMatchingOps();

if (this.prevOp) {
if ("output" in this.prevOp) {
if (!(this.checkOutputFromPrevious())) {
return [];
}
}
}
const matchingOps = this.findMatchingInputOps();
let results = [];

// Record the properties of the current data
Expand All @@ -305,8 +374,7 @@ class Magic {
const opConfig = {
op: op.op,
args: op.args
},
output = await this._runRecipe([opConfig]);
}, output = await this._runRecipe([opConfig]);

// If the recipe is repeating and returning the same data, do not continue
if (prevOp && op.op === prevOp.op && _buffersEqual(output, this.inputBuffer)) {
Expand All @@ -318,7 +386,8 @@ class Magic {
return;
}

const magic = new Magic(output, this.opPatterns),

const magic = new Magic(output, this.opPatterns, OperationConfig[op.op]),
speculativeResults = await magic.speculativeExecution(
depth-1, extLang, intensive, [...recipeConfig, opConfig], op.useful, crib);

Expand All @@ -330,7 +399,7 @@ class Magic {
const bfEncodings = await this.bruteForce();

await Promise.all(bfEncodings.map(async enc => {
const magic = new Magic(enc.data, this.opPatterns),
const magic = new Magic(enc.data, this.opPatterns, undefined),
bfResults = await magic.speculativeExecution(
depth-1, extLang, false, [...recipeConfig, enc.conf], false, crib);

Expand Down Expand Up @@ -447,24 +516,34 @@ class Magic {
* @private
* @returns {Object[]}
*/
static _generateOpPatterns() {
const opPatterns = [];
static _generateOpCriteria() {
const opCriteria = {
regex: [],
entropy: []
};

for (const op in OperationConfig) {
if (!("patterns" in OperationConfig[op])) continue;

OperationConfig[op].patterns.forEach(pattern => {
opPatterns.push({
op: op,
match: pattern.match,
flags: pattern.flags,
args: pattern.args,
useful: pattern.useful || false
});
});
if ("input" in OperationConfig[op]) {
if ("regex" in OperationConfig[op].input)
OperationConfig[op].input.regex.forEach(pattern => {
opCriteria.regex.push({
op: op,
match: new RegExp(pattern.match, pattern.flags),
args: pattern.args,
useful: pattern.useful || false
});
});
if ("entropy" in OperationConfig[op].input) {
opCriteria.entropy.push({
op: op,
entropy: OperationConfig[op].input.entropy.input,
args: OperationConfig[op].input.entropy.args
});
}
}
}

return opPatterns;
return opCriteria;
}

/**
Expand Down
12 changes: 12 additions & 0 deletions src/core/lib/MagicCriteria.mjs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
/**
* Constants for the entropy of text.
*
* @author n1073645 [n1073645@gmail.com]
* @copyright Crown Copyright 2020
* @license Apache-2.0
*/
export const compressedToDecompressed = [6.5, 8];

export const binary = [1, 1.5];

export const entropyOfText = [3.5, 6];
36 changes: 36 additions & 0 deletions src/core/operations/A1Z26CipherDecode.mjs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,42 @@ class A1Z26CipherDecode extends Operation {
value: DELIM_OPTIONS
}
];
this.checks = {
input: {
regex: [
{
match: "^\\s*([12]?[0-9] )+[12]?[0-9]\\s*$",
flags: "",
args: ["Space"]
},
{
match: "^\\s*([12]?[0-9],)+[12]?[0-9]\\s*$",
flags: "",
args: ["Comma"]
},
{
match: "^\\s*([12]?[0-9];)+[12]?[0-9]\\s*$",
flags: "",
args: ["Semi-colon"]
},
{
match: "^\\s*([12]?[0-9]:)+[12]?[0-9]\\s*$",
flags: "",
args: ["Colon"]
},
{
match: "^\\s*([12]?[0-9]\\n)+[12]?[0-9]\\s*$",
flags: "",
args: ["Line feed"]
},
{
match: "^\\s*([12]?[0-9]\\r\\n)+[12]?[0-9]\\s*$",
flags: "",
args: ["CRLF"]
}
]
}
};
}

/**
Expand Down
Loading

0 comments on commit 26fa66e

Please sign in to comment.