diff --git a/package-lock.json b/package-lock.json
index 6fd9822b0..e2936c2a0 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -5353,8 +5353,7 @@
"ansi-regex": {
"version": "2.1.1",
"bundled": true,
- "dev": true,
- "optional": true
+ "dev": true
},
"aproba": {
"version": "1.2.0",
@@ -5375,14 +5374,12 @@
"balanced-match": {
"version": "1.0.0",
"bundled": true,
- "dev": true,
- "optional": true
+ "dev": true
},
"brace-expansion": {
"version": "1.1.11",
"bundled": true,
"dev": true,
- "optional": true,
"requires": {
"balanced-match": "^1.0.0",
"concat-map": "0.0.1"
@@ -5397,20 +5394,17 @@
"code-point-at": {
"version": "1.1.0",
"bundled": true,
- "dev": true,
- "optional": true
+ "dev": true
},
"concat-map": {
"version": "0.0.1",
"bundled": true,
- "dev": true,
- "optional": true
+ "dev": true
},
"console-control-strings": {
"version": "1.1.0",
"bundled": true,
- "dev": true,
- "optional": true
+ "dev": true
},
"core-util-is": {
"version": "1.0.2",
@@ -5527,8 +5521,7 @@
"inherits": {
"version": "2.0.3",
"bundled": true,
- "dev": true,
- "optional": true
+ "dev": true
},
"ini": {
"version": "1.3.5",
@@ -5540,7 +5533,6 @@
"version": "1.0.0",
"bundled": true,
"dev": true,
- "optional": true,
"requires": {
"number-is-nan": "^1.0.0"
}
@@ -5555,7 +5547,6 @@
"version": "3.0.4",
"bundled": true,
"dev": true,
- "optional": true,
"requires": {
"brace-expansion": "^1.1.7"
}
@@ -5563,14 +5554,12 @@
"minimist": {
"version": "0.0.8",
"bundled": true,
- "dev": true,
- "optional": true
+ "dev": true
},
"minipass": {
"version": "2.3.5",
"bundled": true,
"dev": true,
- "optional": true,
"requires": {
"safe-buffer": "^5.1.2",
"yallist": "^3.0.0"
@@ -5589,7 +5578,6 @@
"version": "0.5.1",
"bundled": true,
"dev": true,
- "optional": true,
"requires": {
"minimist": "0.0.8"
}
@@ -5670,8 +5658,7 @@
"number-is-nan": {
"version": "1.0.1",
"bundled": true,
- "dev": true,
- "optional": true
+ "dev": true
},
"object-assign": {
"version": "4.1.1",
@@ -5683,7 +5670,6 @@
"version": "1.4.0",
"bundled": true,
"dev": true,
- "optional": true,
"requires": {
"wrappy": "1"
}
@@ -5769,8 +5755,7 @@
"safe-buffer": {
"version": "5.1.2",
"bundled": true,
- "dev": true,
- "optional": true
+ "dev": true
},
"safer-buffer": {
"version": "2.1.2",
@@ -5806,7 +5791,6 @@
"version": "1.0.2",
"bundled": true,
"dev": true,
- "optional": true,
"requires": {
"code-point-at": "^1.0.0",
"is-fullwidth-code-point": "^1.0.0",
@@ -5826,7 +5810,6 @@
"version": "3.0.1",
"bundled": true,
"dev": true,
- "optional": true,
"requires": {
"ansi-regex": "^2.0.0"
}
@@ -5870,14 +5853,12 @@
"wrappy": {
"version": "1.0.2",
"bundled": true,
- "dev": true,
- "optional": true
+ "dev": true
},
"yallist": {
"version": "3.0.3",
"bundled": true,
- "dev": true,
- "optional": true
+ "dev": true
}
}
},
diff --git a/src/core/operations/Entropy.mjs b/src/core/operations/Entropy.mjs
index 868178fcc..850b8d1a8 100644
--- a/src/core/operations/Entropy.mjs
+++ b/src/core/operations/Entropy.mjs
@@ -4,9 +4,15 @@
* @license Apache-2.0
*/
+import * as d3temp from "d3";
+import * as nodomtemp from "nodom";
+
import Operation from "../Operation";
import Utils from "../Utils";
+const d3 = d3temp.default ? d3temp.default : d3temp;
+const nodom = nodomtemp.default ? nodomtemp.default: nodomtemp;
+
/**
* Entropy operation
*/
@@ -23,22 +29,28 @@ class Entropy extends Operation {
this.description = "Shannon Entropy, in the context of information theory, is a measure of the rate at which information is produced by a source of data. It can be used, in a broad sense, to detect whether data is likely to be structured or unstructured. 8 is the maximum, representing highly unstructured, 'random' data. English language text usually falls somewhere between 3.5 and 5. Properly encrypted or compressed data should have an entropy of over 7.5.";
this.infoURL = "https://wikipedia.org/wiki/Entropy_(information_theory)";
this.inputType = "byteArray";
- this.outputType = "number";
- this.presentType = "html";
- this.args = [];
+ this.outputType = "html";
+ this.args = [
+ {
+ "name": "Visualisation",
+ "type": "option",
+ "value": ["Shannon", "Histogram (Bar)", "Histogram (Line)", "Curve", "Image"]
+ }
+ ];
}
/**
+ * Calculates the frequency of bytes in the input.
+ *
* @param {byteArray} input
- * @param {Object[]} args
- * @returns {number}
+ * @returns {frequency}
*/
- run(input, args) {
+ calculateShannonEntropy(input) {
const prob = [],
uniques = input.unique(),
str = Utils.byteArrayToChars(input);
- let i;
+ let i;
for (i = 0; i < uniques.length; i++) {
prob.push(str.count(Utils.chr(uniques[i])) / input.length);
}
@@ -54,44 +66,335 @@ class Entropy extends Operation {
return -entropy;
}
+ /**
+ *
+ * @param inputBytes
+ * @returns {entropyData}
+ */
+ calculateScanningEntropy(inputBytes, binWidth) {
+ const entropyData = [];
+
+ if (inputBytes.length < 256) binWidth = 8;
+ else binWidth = 256;
+
+ for (let bytePos = 0; bytePos < inputBytes.length; bytePos+=binWidth) {
+ const block = inputBytes.slice(bytePos, bytePos+binWidth);
+ entropyData.push(this.calculateShannonEntropy(block));
+ }
+
+ return { entropyData, binWidth };
+ }
+
+ /**
+ * Calculates the frequency of bytes in the input.
+ *
+ * @param {object} svg
+ * @param {function} xScale
+ * @param {function} yScale
+ * @param {integer} svgHeight
+ * @param {integer} svgWidth
+ * @param {object} margins
+ * @param {string} xTitle
+ * @param {string} yTitle
+ * @returns {undefined}
+ */
+ createAxes(svg, xScale, yScale, svgHeight, svgWidth, margins, title, xTitle, yTitle) {
+ // Axes
+ const yAxis = d3.axisLeft()
+ .scale(yScale);
+
+ const xAxis = d3.axisBottom()
+ .scale(xScale);
+
+ svg.append("g")
+ .attr("transform", `translate(0, ${svgHeight - margins.bottom})`)
+ .call(xAxis);
+
+ svg.append("g")
+ .attr("transform", `translate(${margins.left},0)`)
+ .call(yAxis);
+
+ // Axes labels
+ svg.append("text")
+ .attr("transform", "rotate(-90)")
+ .attr("y", 0 - margins.left)
+ .attr("x", 0 - (svgHeight / 2))
+ .attr("dy", "1em")
+ .style("text-anchor", "middle")
+ .text(yTitle);
+
+ svg.append("text")
+ .attr("transform", `translate(${svgWidth / 2}, ${svgHeight - margins.bottom + 40})`)
+ .style("text-anchor", "middle")
+ .text(xTitle);
+
+ // Add title
+ svg.append("text")
+ .attr("transform", `translate(${svgWidth / 2}, ${margins.top - 10})`)
+ .style("text-anchor", "middle")
+ .text(title);
+ }
+
+ /**
+ * Calculates the frequency of bytes in the input.
+ *
+ * @param {byteArray} inputBytes
+ * @returns {frequency}
+ */
+ calculateByteFrequency(inputBytes) {
+ const byteFrequency = [];
+ for (let i = 0; i < 256; i++) {
+ if (inputBytes.length > 0) {
+ let count = 0;
+ for (const byte of inputBytes) {
+ if (byte === i) count++;
+ }
+ byteFrequency.push(count / inputBytes.length);
+ } else {
+ byteFrequency.push(0);
+ }
+ }
+
+ return byteFrequency;
+ }
+
+ /**
+ * Calculates the frequency of bytes in the input.
+ *
+ * @param {byteArray} byteFrequency
+ * @returns {frequency}
+ */
+ createByteFrequencyLineHistogram(byteFrequency) {
+ const margins = { top: 30, right: 20, bottom: 50, left: 30 };
+
+ const svgWidth = 500,
+ svgHeight = 500;
+
+ const document = new nodom.Document();
+ let svg = document.createElement("svg");
+
+ svg = d3.select(svg)
+ .attr("width", "100%")
+ .attr("height", "100%")
+ .attr("viewBox", `0 0 ${svgWidth} ${svgHeight}`);
+
+ const yScale = d3.scaleLinear()
+ .domain([0, d3.max(byteFrequency, d => d)])
+ .range([svgHeight - margins.bottom, margins.top]);
+
+ const xScale = d3.scaleLinear()
+ .domain([0, byteFrequency.length - 1])
+ .range([margins.left, svgWidth - margins.right]);
+
+ const line = d3.line()
+ .x((_, i) => xScale(i))
+ .y(d => yScale(d))
+ .curve(d3.curveMonotoneX);
+
+ svg.append("path")
+ .datum(byteFrequency)
+ .attr("fill", "none")
+ .attr("stroke", "steelblue")
+ .attr("d", line);
+
+ this.createAxes(svg, xScale, yScale, svgHeight, svgWidth, margins, "", "Byte", "Byte Frequency");
+
+ return svg._groups[0][0].outerHTML;
+ }
+
+ /**
+ * Creates a byte frequency histogram
+ *
+ * @param {byteArray} byteFrequency
+ * @returns {HTML}
+ */
+ createByteFrequencyBarHistogram(byteFrequency) {
+ const margins = { top: 30, right: 20, bottom: 50, left: 30 };
+
+ const svgWidth = 500,
+ svgHeight = 500,
+ binWidth = 1;
+
+ const document = new nodom.Document();
+ let svg = document.createElement("svg");
+ svg = d3.select(svg)
+ .attr("width", "100%")
+ .attr("height", "100%")
+ .attr("viewBox", `0 0 ${svgWidth} ${svgHeight}`);
+
+ const yExtent = d3.extent(byteFrequency, d => d);
+ const yScale = d3.scaleLinear()
+ .domain(yExtent)
+ .range([svgHeight - margins.bottom, margins.top]);
+
+ const xScale = d3.scaleLinear()
+ .domain([0, byteFrequency.length - 1])
+ .range([margins.left - binWidth, svgWidth - margins.right]);
+
+ svg.selectAll("rect")
+ .data(byteFrequency)
+ .enter().append("rect")
+ .attr("x", (_, i) => xScale(i) + binWidth)
+ .attr("y", dataPoint => yScale(dataPoint))
+ .attr("width", binWidth)
+ .attr("height", dataPoint => yScale(yExtent[0]) - yScale(dataPoint))
+ .attr("fill", "blue");
+
+ this.createAxes(svg, xScale, yScale, svgHeight, svgWidth, margins, "", "Byte", "Byte Frequency");
+
+ return svg._groups[0][0].outerHTML;
+ }
+
+ /**
+ * Creates a byte frequency histogram
+ *
+ * @param {byteArray} input
+ * @param {number} blockSize
+ * @returns {HTML}
+ */
+ createEntropyCurve(entropyData) {
+ const margins = { top: 30, right: 20, bottom: 50, left: 30 };
+
+ const svgWidth = 500,
+ svgHeight = 500;
+
+ const document = new nodom.Document();
+ let svg = document.createElement("svg");
+ svg = d3.select(svg)
+ .attr("width", "100%")
+ .attr("height", "100%")
+ .attr("viewBox", `0 0 ${svgWidth} ${svgHeight}`);
+
+ const yScale = d3.scaleLinear()
+ .domain([0, d3.max(entropyData, d => d)])
+ .range([svgHeight - margins.bottom, margins.top]);
+
+ const xScale = d3.scaleLinear()
+ .domain([0, entropyData.length])
+ .range([margins.left, svgWidth - margins.right]);
+
+ const line = d3.line()
+ .x((_, i) => xScale(i))
+ .y(d => yScale(d))
+ .curve(d3.curveMonotoneX);
+
+ if (entropyData.length > 0) {
+ svg.append("path")
+ .datum(entropyData)
+ .attr("d", line);
+
+ svg.selectAll("path").attr("fill", "none").attr("stroke", "steelblue");
+ }
+
+ this.createAxes(svg, xScale, yScale, svgHeight, svgWidth, margins, "Scanning Entropy", "Block", "Entropy");
+
+ return svg._groups[0][0].outerHTML;
+ }
+
+ /**
+ * Creates an image representation of the entropy
+ *
+ * @param {byteArray} input
+ * @param {number} blockSize
+ * @returns {HTML}
+ */
+ createEntropyImage(entropyData) {
+ const svgHeight = 100,
+ svgWidth = 100,
+ cellSize = 1,
+ nodes = [];
+
+ for (let i = 0; i < entropyData.length; i++) {
+ nodes.push({
+ x: i % svgWidth,
+ y: Math.floor(i / svgWidth),
+ entropy: entropyData[i]
+ });
+ }
+
+ const document = new nodom.Document();
+ let svg = document.createElement("svg");
+ svg = d3.select(svg)
+ .attr("width", "100%")
+ .attr("height", "100%")
+ .attr("viewBox", `0 0 ${svgWidth} ${svgHeight}`);
+
+ const greyScale = d3.scaleLinear()
+ .domain([0, d3.max(entropyData, d => d)])
+ .range(["#000000", "#FFFFFF"])
+ .interpolate(d3.interpolateRgb);
+
+ svg
+ .selectAll("rect")
+ .data(nodes)
+ .enter().append("rect")
+ .attr("x", d => d.x * cellSize)
+ .attr("y", d => d.y * cellSize)
+ .attr("width", cellSize)
+ .attr("height", cellSize)
+ .style("fill", d => greyScale(d.entropy));
+
+ return svg._groups[0][0].outerHTML;
+ }
+
/**
* Displays the entropy as a scale bar for web apps.
*
* @param {number} entropy
* @returns {html}
*/
- present(entropy) {
+ createShannonEntropyVisualization(entropy) {
return `Shannon entropy: ${entropy}
-
-- 0 represents no randomness (i.e. all the bytes in the data have the same value) whereas 8, the maximum, represents a completely random string.
-- Standard English text usually falls somewhere between 3.5 and 5.
-- Properly encrypted or compressed data of a reasonable length should have an entropy of over 7.5.
-
-The following results show the entropy of chunks of the input data. Chunks with particularly high entropy could suggest encrypted or compressed sections.
-
-
`;
+
+ - 0 represents no randomness (i.e. all the bytes in the data have the same value) whereas 8, the maximum, represents a completely random string.
+ - Standard English text usually falls somewhere between 3.5 and 5.
+ - Properly encrypted or compressed data of a reasonable length should have an entropy of over 7.5.
+
+ The following results show the entropy of chunks of the input data. Chunks with particularly high entropy could suggest encrypted or compressed sections.
+
+
`;
}
+ /**
+ * @param {byteArray} input
+ * @param {Object[]} args
+ * @returns {html}
+ */
+ run(input, args) {
+ const visualizationType = args[0];
+
+ if (visualizationType === "Shannon") {
+ return this.createShannonEntropyVisualization(this.calculateShannonEntropy(input));
+ } else if (visualizationType === "Histogram (Bar)") {
+ return this.createByteFrequencyBarHistogram(this.calculateByteFrequency(input));
+ } else if (visualizationType === "Histogram (Line)") {
+ return this.createByteFrequencyLineHistogram(this.calculateByteFrequency(input));
+ } else if (visualizationType === "Curve") {
+ return this.createEntropyCurve(this.calculateScanningEntropy(input).entropyData);
+ } else if (visualizationType === "Image") {
+ return this.createEntropyImage(this.calculateScanningEntropy(input).entropyData);
+ }
+ }
}
export default Entropy;