diff --git a/package-lock.json b/package-lock.json index 6fd9822b0..e2936c2a0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5353,8 +5353,7 @@ "ansi-regex": { "version": "2.1.1", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "aproba": { "version": "1.2.0", @@ -5375,14 +5374,12 @@ "balanced-match": { "version": "1.0.0", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "brace-expansion": { "version": "1.1.11", "bundled": true, "dev": true, - "optional": true, "requires": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -5397,20 +5394,17 @@ "code-point-at": { "version": "1.1.0", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "concat-map": { "version": "0.0.1", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "console-control-strings": { "version": "1.1.0", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "core-util-is": { "version": "1.0.2", @@ -5527,8 +5521,7 @@ "inherits": { "version": "2.0.3", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "ini": { "version": "1.3.5", @@ -5540,7 +5533,6 @@ "version": "1.0.0", "bundled": true, "dev": true, - "optional": true, "requires": { "number-is-nan": "^1.0.0" } @@ -5555,7 +5547,6 @@ "version": "3.0.4", "bundled": true, "dev": true, - "optional": true, "requires": { "brace-expansion": "^1.1.7" } @@ -5563,14 +5554,12 @@ "minimist": { "version": "0.0.8", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "minipass": { "version": "2.3.5", "bundled": true, "dev": true, - "optional": true, "requires": { "safe-buffer": "^5.1.2", "yallist": "^3.0.0" @@ -5589,7 +5578,6 @@ "version": "0.5.1", "bundled": true, "dev": true, - "optional": true, "requires": { "minimist": "0.0.8" } @@ -5670,8 +5658,7 @@ "number-is-nan": { "version": "1.0.1", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "object-assign": { "version": "4.1.1", @@ -5683,7 +5670,6 @@ "version": "1.4.0", "bundled": true, "dev": true, - "optional": true, "requires": { "wrappy": "1" } @@ -5769,8 +5755,7 @@ "safe-buffer": { "version": "5.1.2", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "safer-buffer": { "version": "2.1.2", @@ -5806,7 +5791,6 @@ "version": "1.0.2", "bundled": true, "dev": true, - "optional": true, "requires": { "code-point-at": "^1.0.0", "is-fullwidth-code-point": "^1.0.0", @@ -5826,7 +5810,6 @@ "version": "3.0.1", "bundled": true, "dev": true, - "optional": true, "requires": { "ansi-regex": "^2.0.0" } @@ -5870,14 +5853,12 @@ "wrappy": { "version": "1.0.2", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "yallist": { "version": "3.0.3", "bundled": true, - "dev": true, - "optional": true + "dev": true } } }, diff --git a/src/core/operations/Entropy.mjs b/src/core/operations/Entropy.mjs index 868178fcc..850b8d1a8 100644 --- a/src/core/operations/Entropy.mjs +++ b/src/core/operations/Entropy.mjs @@ -4,9 +4,15 @@ * @license Apache-2.0 */ +import * as d3temp from "d3"; +import * as nodomtemp from "nodom"; + import Operation from "../Operation"; import Utils from "../Utils"; +const d3 = d3temp.default ? d3temp.default : d3temp; +const nodom = nodomtemp.default ? nodomtemp.default: nodomtemp; + /** * Entropy operation */ @@ -23,22 +29,28 @@ class Entropy extends Operation { this.description = "Shannon Entropy, in the context of information theory, is a measure of the rate at which information is produced by a source of data. It can be used, in a broad sense, to detect whether data is likely to be structured or unstructured. 8 is the maximum, representing highly unstructured, 'random' data. English language text usually falls somewhere between 3.5 and 5. Properly encrypted or compressed data should have an entropy of over 7.5."; this.infoURL = "https://wikipedia.org/wiki/Entropy_(information_theory)"; this.inputType = "byteArray"; - this.outputType = "number"; - this.presentType = "html"; - this.args = []; + this.outputType = "html"; + this.args = [ + { + "name": "Visualisation", + "type": "option", + "value": ["Shannon", "Histogram (Bar)", "Histogram (Line)", "Curve", "Image"] + } + ]; } /** + * Calculates the frequency of bytes in the input. + * * @param {byteArray} input - * @param {Object[]} args - * @returns {number} + * @returns {frequency} */ - run(input, args) { + calculateShannonEntropy(input) { const prob = [], uniques = input.unique(), str = Utils.byteArrayToChars(input); - let i; + let i; for (i = 0; i < uniques.length; i++) { prob.push(str.count(Utils.chr(uniques[i])) / input.length); } @@ -54,44 +66,335 @@ class Entropy extends Operation { return -entropy; } + /** + * + * @param inputBytes + * @returns {entropyData} + */ + calculateScanningEntropy(inputBytes, binWidth) { + const entropyData = []; + + if (inputBytes.length < 256) binWidth = 8; + else binWidth = 256; + + for (let bytePos = 0; bytePos < inputBytes.length; bytePos+=binWidth) { + const block = inputBytes.slice(bytePos, bytePos+binWidth); + entropyData.push(this.calculateShannonEntropy(block)); + } + + return { entropyData, binWidth }; + } + + /** + * Calculates the frequency of bytes in the input. + * + * @param {object} svg + * @param {function} xScale + * @param {function} yScale + * @param {integer} svgHeight + * @param {integer} svgWidth + * @param {object} margins + * @param {string} xTitle + * @param {string} yTitle + * @returns {undefined} + */ + createAxes(svg, xScale, yScale, svgHeight, svgWidth, margins, title, xTitle, yTitle) { + // Axes + const yAxis = d3.axisLeft() + .scale(yScale); + + const xAxis = d3.axisBottom() + .scale(xScale); + + svg.append("g") + .attr("transform", `translate(0, ${svgHeight - margins.bottom})`) + .call(xAxis); + + svg.append("g") + .attr("transform", `translate(${margins.left},0)`) + .call(yAxis); + + // Axes labels + svg.append("text") + .attr("transform", "rotate(-90)") + .attr("y", 0 - margins.left) + .attr("x", 0 - (svgHeight / 2)) + .attr("dy", "1em") + .style("text-anchor", "middle") + .text(yTitle); + + svg.append("text") + .attr("transform", `translate(${svgWidth / 2}, ${svgHeight - margins.bottom + 40})`) + .style("text-anchor", "middle") + .text(xTitle); + + // Add title + svg.append("text") + .attr("transform", `translate(${svgWidth / 2}, ${margins.top - 10})`) + .style("text-anchor", "middle") + .text(title); + } + + /** + * Calculates the frequency of bytes in the input. + * + * @param {byteArray} inputBytes + * @returns {frequency} + */ + calculateByteFrequency(inputBytes) { + const byteFrequency = []; + for (let i = 0; i < 256; i++) { + if (inputBytes.length > 0) { + let count = 0; + for (const byte of inputBytes) { + if (byte === i) count++; + } + byteFrequency.push(count / inputBytes.length); + } else { + byteFrequency.push(0); + } + } + + return byteFrequency; + } + + /** + * Calculates the frequency of bytes in the input. + * + * @param {byteArray} byteFrequency + * @returns {frequency} + */ + createByteFrequencyLineHistogram(byteFrequency) { + const margins = { top: 30, right: 20, bottom: 50, left: 30 }; + + const svgWidth = 500, + svgHeight = 500; + + const document = new nodom.Document(); + let svg = document.createElement("svg"); + + svg = d3.select(svg) + .attr("width", "100%") + .attr("height", "100%") + .attr("viewBox", `0 0 ${svgWidth} ${svgHeight}`); + + const yScale = d3.scaleLinear() + .domain([0, d3.max(byteFrequency, d => d)]) + .range([svgHeight - margins.bottom, margins.top]); + + const xScale = d3.scaleLinear() + .domain([0, byteFrequency.length - 1]) + .range([margins.left, svgWidth - margins.right]); + + const line = d3.line() + .x((_, i) => xScale(i)) + .y(d => yScale(d)) + .curve(d3.curveMonotoneX); + + svg.append("path") + .datum(byteFrequency) + .attr("fill", "none") + .attr("stroke", "steelblue") + .attr("d", line); + + this.createAxes(svg, xScale, yScale, svgHeight, svgWidth, margins, "", "Byte", "Byte Frequency"); + + return svg._groups[0][0].outerHTML; + } + + /** + * Creates a byte frequency histogram + * + * @param {byteArray} byteFrequency + * @returns {HTML} + */ + createByteFrequencyBarHistogram(byteFrequency) { + const margins = { top: 30, right: 20, bottom: 50, left: 30 }; + + const svgWidth = 500, + svgHeight = 500, + binWidth = 1; + + const document = new nodom.Document(); + let svg = document.createElement("svg"); + svg = d3.select(svg) + .attr("width", "100%") + .attr("height", "100%") + .attr("viewBox", `0 0 ${svgWidth} ${svgHeight}`); + + const yExtent = d3.extent(byteFrequency, d => d); + const yScale = d3.scaleLinear() + .domain(yExtent) + .range([svgHeight - margins.bottom, margins.top]); + + const xScale = d3.scaleLinear() + .domain([0, byteFrequency.length - 1]) + .range([margins.left - binWidth, svgWidth - margins.right]); + + svg.selectAll("rect") + .data(byteFrequency) + .enter().append("rect") + .attr("x", (_, i) => xScale(i) + binWidth) + .attr("y", dataPoint => yScale(dataPoint)) + .attr("width", binWidth) + .attr("height", dataPoint => yScale(yExtent[0]) - yScale(dataPoint)) + .attr("fill", "blue"); + + this.createAxes(svg, xScale, yScale, svgHeight, svgWidth, margins, "", "Byte", "Byte Frequency"); + + return svg._groups[0][0].outerHTML; + } + + /** + * Creates a byte frequency histogram + * + * @param {byteArray} input + * @param {number} blockSize + * @returns {HTML} + */ + createEntropyCurve(entropyData) { + const margins = { top: 30, right: 20, bottom: 50, left: 30 }; + + const svgWidth = 500, + svgHeight = 500; + + const document = new nodom.Document(); + let svg = document.createElement("svg"); + svg = d3.select(svg) + .attr("width", "100%") + .attr("height", "100%") + .attr("viewBox", `0 0 ${svgWidth} ${svgHeight}`); + + const yScale = d3.scaleLinear() + .domain([0, d3.max(entropyData, d => d)]) + .range([svgHeight - margins.bottom, margins.top]); + + const xScale = d3.scaleLinear() + .domain([0, entropyData.length]) + .range([margins.left, svgWidth - margins.right]); + + const line = d3.line() + .x((_, i) => xScale(i)) + .y(d => yScale(d)) + .curve(d3.curveMonotoneX); + + if (entropyData.length > 0) { + svg.append("path") + .datum(entropyData) + .attr("d", line); + + svg.selectAll("path").attr("fill", "none").attr("stroke", "steelblue"); + } + + this.createAxes(svg, xScale, yScale, svgHeight, svgWidth, margins, "Scanning Entropy", "Block", "Entropy"); + + return svg._groups[0][0].outerHTML; + } + + /** + * Creates an image representation of the entropy + * + * @param {byteArray} input + * @param {number} blockSize + * @returns {HTML} + */ + createEntropyImage(entropyData) { + const svgHeight = 100, + svgWidth = 100, + cellSize = 1, + nodes = []; + + for (let i = 0; i < entropyData.length; i++) { + nodes.push({ + x: i % svgWidth, + y: Math.floor(i / svgWidth), + entropy: entropyData[i] + }); + } + + const document = new nodom.Document(); + let svg = document.createElement("svg"); + svg = d3.select(svg) + .attr("width", "100%") + .attr("height", "100%") + .attr("viewBox", `0 0 ${svgWidth} ${svgHeight}`); + + const greyScale = d3.scaleLinear() + .domain([0, d3.max(entropyData, d => d)]) + .range(["#000000", "#FFFFFF"]) + .interpolate(d3.interpolateRgb); + + svg + .selectAll("rect") + .data(nodes) + .enter().append("rect") + .attr("x", d => d.x * cellSize) + .attr("y", d => d.y * cellSize) + .attr("width", cellSize) + .attr("height", cellSize) + .style("fill", d => greyScale(d.entropy)); + + return svg._groups[0][0].outerHTML; + } + /** * Displays the entropy as a scale bar for web apps. * * @param {number} entropy * @returns {html} */ - present(entropy) { + createShannonEntropyVisualization(entropy) { return `Shannon entropy: ${entropy} -

-- 0 represents no randomness (i.e. all the bytes in the data have the same value) whereas 8, the maximum, represents a completely random string. -- Standard English text usually falls somewhere between 3.5 and 5. -- Properly encrypted or compressed data of a reasonable length should have an entropy of over 7.5. - -The following results show the entropy of chunks of the input data. Chunks with particularly high entropy could suggest encrypted or compressed sections. - -
`; +

+ - 0 represents no randomness (i.e. all the bytes in the data have the same value) whereas 8, the maximum, represents a completely random string. + - Standard English text usually falls somewhere between 3.5 and 5. + - Properly encrypted or compressed data of a reasonable length should have an entropy of over 7.5. + + The following results show the entropy of chunks of the input data. Chunks with particularly high entropy could suggest encrypted or compressed sections. + +
`; } + /** + * @param {byteArray} input + * @param {Object[]} args + * @returns {html} + */ + run(input, args) { + const visualizationType = args[0]; + + if (visualizationType === "Shannon") { + return this.createShannonEntropyVisualization(this.calculateShannonEntropy(input)); + } else if (visualizationType === "Histogram (Bar)") { + return this.createByteFrequencyBarHistogram(this.calculateByteFrequency(input)); + } else if (visualizationType === "Histogram (Line)") { + return this.createByteFrequencyLineHistogram(this.calculateByteFrequency(input)); + } else if (visualizationType === "Curve") { + return this.createEntropyCurve(this.calculateScanningEntropy(input).entropyData); + } else if (visualizationType === "Image") { + return this.createEntropyImage(this.calculateScanningEntropy(input).entropyData); + } + } } export default Entropy;