From fa89713f194e820e5cb4d4996f2a00cc80208853 Mon Sep 17 00:00:00 2001 From: toby Date: Tue, 30 May 2017 14:50:56 -0400 Subject: [PATCH 01/11] Add d3 as a dependency --- package.json | 2 ++ 1 file changed, 2 insertions(+) diff --git a/package.json b/package.json index 4ad375658..09a080365 100644 --- a/package.json +++ b/package.json @@ -68,6 +68,8 @@ "bootstrap-switch": "^3.3.4", "crypto-api": "^0.6.2", "crypto-js": "^3.1.9-1", + "d3": "^4.9.1", + "d3-hexbin": "^0.2.2", "diff": "^3.2.0", "escodegen": "^1.8.1", "esmangle": "^1.0.1", From 281d558111c5094b828583109fe8417f94abfb1c Mon Sep 17 00:00:00 2001 From: toby Date: Tue, 30 May 2017 14:53:32 -0400 Subject: [PATCH 02/11] Add hex density chart --- src/core/Utils.js | 1 + src/core/config/OperationConfig.js | 39 ++++++ src/core/operations/Charts.js | 205 +++++++++++++++++++++++++++++ 3 files changed, 245 insertions(+) create mode 100755 src/core/operations/Charts.js diff --git a/src/core/Utils.js b/src/core/Utils.js index 9b0d2a30b..bb05ec3dd 100755 --- a/src/core/Utils.js +++ b/src/core/Utils.js @@ -1021,6 +1021,7 @@ const Utils = { "Comma": ",", "Semi-colon": ";", "Colon": ":", + "Tab": "\t", "Line feed": "\n", "CRLF": "\r\n", "Forward slash": "/", diff --git a/src/core/config/OperationConfig.js b/src/core/config/OperationConfig.js index 5fd5a9eea..f11809ad8 100755 --- a/src/core/config/OperationConfig.js +++ b/src/core/config/OperationConfig.js @@ -5,6 +5,7 @@ import Base64 from "../operations/Base64.js"; import BitwiseOp from "../operations/BitwiseOp.js"; import ByteRepr from "../operations/ByteRepr.js"; import CharEnc from "../operations/CharEnc.js"; +import Charts from "../operations/Charts.js"; import Checksum from "../operations/Checksum.js"; import Cipher from "../operations/Cipher.js"; import Code from "../operations/Code.js"; @@ -3388,6 +3389,44 @@ const OperationConfig = { } ] }, + "Hex Density chart": { + description: [].join("\n"), + run: Charts.runHexDensityChart, + inputType: "string", + outputType: "html", + args: [ + { + name: "Record delimiter", + type: "option", + value: Charts.RECORD_DELIMITER_OPTIONS, + }, + { + name: "Field delimiter", + type: "option", + value: Charts.FIELD_DELIMITER_OPTIONS, + }, + { + name: "Radius", + type: "number", + value: 25, + }, + { + name: "Use column headers as labels", + type: "boolean", + value: true, + }, + { + name: "X label", + type: "string", + value: "", + }, + { + name: "Y label", + type: "string", + value: "", + }, + ] + } }; export default OperationConfig; diff --git a/src/core/operations/Charts.js b/src/core/operations/Charts.js new file mode 100755 index 000000000..a1ab9725b --- /dev/null +++ b/src/core/operations/Charts.js @@ -0,0 +1,205 @@ +import * as d3 from "d3"; +import {hexbin as d3hexbin} from "d3-hexbin"; +import Utils from "../Utils.js"; + +/** + * Charting operations. + * + * @author tlwr [toby@toby.com] + * @copyright Crown Copyright 2016 + * @license Apache-2.0 + * + * @namespace + */ +const Charts = { + /** + * @constant + * @default + */ + RECORD_DELIMITER_OPTIONS: ["Line feed", "CRLF"], + + + /** + * @constant + * @default + */ + FIELD_DELIMITER_OPTIONS: ["Space", "Comma", "Semi-colon", "Colon", "Tab"], + + + /** + * Gets values from input for a scatter plot. + * + * @param {string} input + * @param {string} recordDelimiter + * @param {string} fieldDelimiter + * @param {boolean} columnHeadingsAreIncluded - whether we should skip the first record + * @returns {Object[]} + */ + _getScatterValues(input, recordDelimiter, fieldDelimiter, columnHeadingsAreIncluded) { + let headings; + const values = []; + + input + .split(recordDelimiter) + .forEach((row, rowIndex) => { + let split = row.split(fieldDelimiter); + + if (split.length !== 2) throw "Each row must have length 2."; + + if (columnHeadingsAreIncluded && rowIndex === 0) { + headings = {}; + headings.x = split[0]; + headings.y = split[1]; + } else { + let x = split[0], + y = split[1]; + + x = parseFloat(x, 10); + if (Number.isNaN(x)) throw "Values must be numbers in base 10."; + + y = parseFloat(y, 10); + if (Number.isNaN(y)) throw "Values must be numbers in base 10."; + + values.push([x, y]); + } + }); + + return { headings, values}; + }, + + + /** + * Hex Bin chart operation. + * + * @param {string} input + * @param {Object[]} args + * @returns {html} + */ + runHexDensityChart: function (input, args) { + const recordDelimiter = Utils.charRep[args[0]], + fieldDelimiter = Utils.charRep[args[1]], + radius = args[2], + columnHeadingsAreIncluded = args[3], + dimension = 500; + + let xLabel = args[4], + yLabel = args[5], + { headings, values } = Charts._getScatterValues( + input, + recordDelimiter, + fieldDelimiter, + columnHeadingsAreIncluded + ); + + if (headings) { + xLabel = headings.x; + yLabel = headings.y; + } + + let svg = document.createElement("svg"); + svg = d3.select(svg) + .attr("width", "100%") + .attr("height", "100%") + .attr("viewBox", `0 0 ${dimension} ${dimension}`); + + let margin = { + top: 0, + right: 0, + bottom: 30, + left: 30, + }, + width = dimension - margin.left - margin.right, + height = dimension - margin.top - margin.bottom, + marginedSpace = svg.append("g") + .attr("transform", "translate(" + margin.left + "," + margin.top + ")"); + + let hexbin = d3hexbin() + .radius(radius) + .extent([0, 0], [width, height]); + + let hexPoints = hexbin(values), + maxCount = Math.max(...hexPoints.map(b => b.length)); + + let xExtent = d3.extent(hexPoints, d => d.x), + yExtent = d3.extent(hexPoints, d => d.y); + xExtent[0] -= 2 * radius; + xExtent[1] += 2 * radius; + yExtent[0] -= 2 * radius; + yExtent[1] += 2 * radius; + + let xAxis = d3.scaleLinear() + .domain(xExtent) + .range([0, width]); + let yAxis = d3.scaleLinear() + .domain(yExtent) + .range([height, 0]); + + let color = d3.scaleSequential(d3.interpolateLab("white", "steelblue")) + .domain([0, maxCount]); + + marginedSpace.append("clipPath") + .attr("id", "clip") + .append("rect") + .attr("width", width) + .attr("height", height); + + marginedSpace.append("g") + .attr("class", "hexagon") + .attr("clip-path", "url(#clip)") + .selectAll("path") + .data(hexPoints) + .enter() + .append("path") + .attr("d", d => { + return `M${xAxis(d.x)},${yAxis(d.y)} ${hexbin.hexagon(radius * 0.75)}`; + }) + .attr("fill", (d) => color(d.length)) + .append("title") + .text(d => { + let count = d.length, + perc = 100.0 * d.length / values.length, + CX = d.x, + CY = d.y, + xMin = Math.min(...d.map(d => d[0])), + xMax = Math.max(...d.map(d => d[0])), + yMin = Math.min(...d.map(d => d[1])), + yMax = Math.max(...d.map(d => d[1])), + tooltip = `Count: ${count}\n + Percentage: ${perc.toFixed(2)}%\n + Center: ${CX.toFixed(2)}, ${CY.toFixed(2)}\n + Min X: ${xMin.toFixed(2)}\n + Max X: ${xMax.toFixed(2)}\n + Min Y: ${yMin.toFixed(2)}\n + Max Y: ${yMax.toFixed(2)} + `.replace(/\s{2,}/g, "\n"); + return tooltip; + }); + + marginedSpace.append("g") + .attr("class", "axis axis--y") + .call(d3.axisLeft(yAxis).tickSizeOuter(-width)); + + svg.append("text") + .attr("transform", "rotate(-90)") + .attr("y", -margin.left) + .attr("x", -(height / 2)) + .attr("dy", "1em") + .style("text-anchor", "middle") + .text(yLabel); + + marginedSpace.append("g") + .attr("class", "axis axis--x") + .attr("transform", "translate(0," + height + ")") + .call(d3.axisBottom(xAxis).tickSizeOuter(-height)); + + svg.append("text") + .attr("x", width / 2) + .attr("y", dimension) + .style("text-anchor", "middle") + .text(xLabel); + + return svg._groups[0][0].outerHTML; + }, +}; + +export default Charts; From 6cdc7d3966e19443c5d4d595ff1218eb04e6fc79 Mon Sep 17 00:00:00 2001 From: toby Date: Tue, 30 May 2017 15:24:23 -0400 Subject: [PATCH 03/11] Hex density: split radius into draw & pack radii --- src/core/config/OperationConfig.js | 7 ++++++- src/core/operations/Charts.js | 21 +++++++++++---------- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/src/core/config/OperationConfig.js b/src/core/config/OperationConfig.js index f11809ad8..db7f58373 100755 --- a/src/core/config/OperationConfig.js +++ b/src/core/config/OperationConfig.js @@ -3406,10 +3406,15 @@ const OperationConfig = { value: Charts.FIELD_DELIMITER_OPTIONS, }, { - name: "Radius", + name: "Pack radius", type: "number", value: 25, }, + { + name: "Draw radius", + type: "number", + value: 15, + }, { name: "Use column headers as labels", type: "boolean", diff --git a/src/core/operations/Charts.js b/src/core/operations/Charts.js index a1ab9725b..1c026fb7e 100755 --- a/src/core/operations/Charts.js +++ b/src/core/operations/Charts.js @@ -78,12 +78,13 @@ const Charts = { runHexDensityChart: function (input, args) { const recordDelimiter = Utils.charRep[args[0]], fieldDelimiter = Utils.charRep[args[1]], - radius = args[2], - columnHeadingsAreIncluded = args[3], + packRadius = args[2], + drawRadius = args[3], + columnHeadingsAreIncluded = args[4], dimension = 500; - let xLabel = args[4], - yLabel = args[5], + let xLabel = args[5], + yLabel = args[6], { headings, values } = Charts._getScatterValues( input, recordDelimiter, @@ -114,7 +115,7 @@ const Charts = { .attr("transform", "translate(" + margin.left + "," + margin.top + ")"); let hexbin = d3hexbin() - .radius(radius) + .radius(packRadius) .extent([0, 0], [width, height]); let hexPoints = hexbin(values), @@ -122,10 +123,10 @@ const Charts = { let xExtent = d3.extent(hexPoints, d => d.x), yExtent = d3.extent(hexPoints, d => d.y); - xExtent[0] -= 2 * radius; - xExtent[1] += 2 * radius; - yExtent[0] -= 2 * radius; - yExtent[1] += 2 * radius; + xExtent[0] -= 2 * packRadius; + xExtent[1] += 2 * packRadius; + yExtent[0] -= 2 * packRadius; + yExtent[1] += 2 * packRadius; let xAxis = d3.scaleLinear() .domain(xExtent) @@ -151,7 +152,7 @@ const Charts = { .enter() .append("path") .attr("d", d => { - return `M${xAxis(d.x)},${yAxis(d.y)} ${hexbin.hexagon(radius * 0.75)}`; + return `M${xAxis(d.x)},${yAxis(d.y)} ${hexbin.hexagon(drawRadius)}`; }) .attr("fill", (d) => color(d.length)) .append("title") From dc642be1f53b270f8107b09405f79e5ecd012ef2 Mon Sep 17 00:00:00 2001 From: toby Date: Tue, 30 May 2017 15:49:22 -0400 Subject: [PATCH 04/11] Hex plot: add edge drawing & changing colour opts --- src/core/config/OperationConfig.js | 15 +++++++++++++++ src/core/operations/Charts.js | 22 ++++++++++++++++++++-- 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/src/core/config/OperationConfig.js b/src/core/config/OperationConfig.js index db7f58373..ffb75a073 100755 --- a/src/core/config/OperationConfig.js +++ b/src/core/config/OperationConfig.js @@ -3430,6 +3430,21 @@ const OperationConfig = { type: "string", value: "", }, + { + name: "Draw hexagon edges", + type: "boolean", + value: false, + }, + { + name: "Min colour value", + type: "string", + value: Charts.COLOURS.min, + }, + { + name: "Max colour value", + type: "string", + value: Charts.COLOURS.max, + }, ] } }; diff --git a/src/core/operations/Charts.js b/src/core/operations/Charts.js index 1c026fb7e..eb8c7efec 100755 --- a/src/core/operations/Charts.js +++ b/src/core/operations/Charts.js @@ -68,6 +68,19 @@ const Charts = { }, + /** + * Default from colour + * + * @constant + * @default + */ + COLOURS: { + min: "white", + max: "black", + }, + + + /** * Hex Bin chart operation. * @@ -81,6 +94,9 @@ const Charts = { packRadius = args[2], drawRadius = args[3], columnHeadingsAreIncluded = args[4], + drawEdges = args[7], + minColour = args[8], + maxColour = args[9], dimension = 500; let xLabel = args[5], @@ -135,7 +151,7 @@ const Charts = { .domain(yExtent) .range([height, 0]); - let color = d3.scaleSequential(d3.interpolateLab("white", "steelblue")) + let colour = d3.scaleSequential(d3.interpolateLab(minColour, maxColour)) .domain([0, maxCount]); marginedSpace.append("clipPath") @@ -154,7 +170,9 @@ const Charts = { .attr("d", d => { return `M${xAxis(d.x)},${yAxis(d.y)} ${hexbin.hexagon(drawRadius)}`; }) - .attr("fill", (d) => color(d.length)) + .attr("fill", (d) => colour(d.length)) + .attr("stroke", drawEdges ? "black" : "none") + .attr("stroke-width", drawEdges ? "0.5" : "none") .append("title") .text(d => { let count = d.length, From b4188db671ec1451c089b0f5416a9aeaf13805ec Mon Sep 17 00:00:00 2001 From: toby Date: Wed, 31 May 2017 14:56:03 -0400 Subject: [PATCH 05/11] Hexagon density: allow dense plotting of hexagons --- src/core/config/OperationConfig.js | 5 +++ src/core/operations/Charts.js | 56 ++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+) diff --git a/src/core/config/OperationConfig.js b/src/core/config/OperationConfig.js index ffb75a073..ab38b7cf7 100755 --- a/src/core/config/OperationConfig.js +++ b/src/core/config/OperationConfig.js @@ -3445,6 +3445,11 @@ const OperationConfig = { type: "string", value: Charts.COLOURS.max, }, + { + name: "Draw empty hexagons within data boundaries", + type: "boolean", + value: false, + }, ] } }; diff --git a/src/core/operations/Charts.js b/src/core/operations/Charts.js index eb8c7efec..447d47b21 100755 --- a/src/core/operations/Charts.js +++ b/src/core/operations/Charts.js @@ -80,6 +80,36 @@ const Charts = { }, + /** + * Hex Bin chart operation. + * + * @param {Object[]} - centres + * @param {number} - radius + * @returns {Object[]} + */ + _getEmptyHexagons(centres, radius) { + const emptyCentres = []; + let boundingRect = [d3.extent(centres, d => d.x), d3.extent(centres, d => d.y)], + indent = false, + hexagonCenterToEdge = Math.cos(2 * Math.PI / 12) * radius, + hexagonEdgeLength = Math.sin(2 * Math.PI / 12) * radius; + + for (let y = boundingRect[1][0]; y <= boundingRect[1][1] + radius; y += hexagonEdgeLength + radius) { + for (let x = boundingRect[0][0]; x <= boundingRect[0][1] + radius; x += 2 * hexagonCenterToEdge) { + let cx = x, + cy = y; + + if (indent && x >= boundingRect[0][1]) break; + if (indent) cx += hexagonCenterToEdge; + + emptyCentres.push({x: cx, y: cy}); + } + indent = !indent; + } + + return emptyCentres; + }, + /** * Hex Bin chart operation. @@ -97,6 +127,7 @@ const Charts = { drawEdges = args[7], minColour = args[8], maxColour = args[9], + drawEmptyHexagons = args[10], dimension = 500; let xLabel = args[5], @@ -160,6 +191,31 @@ const Charts = { .attr("width", width) .attr("height", height); + if (drawEmptyHexagons) { + marginedSpace.append("g") + .attr("class", "empty-hexagon") + .selectAll("path") + .data(Charts._getEmptyHexagons(hexPoints, packRadius)) + .enter() + .append("path") + .attr("d", d => { + return `M${xAxis(d.x)},${yAxis(d.y)} ${hexbin.hexagon(drawRadius)}`; + }) + .attr("fill", (d) => colour(0)) + .attr("stroke", drawEdges ? "black" : "none") + .attr("stroke-width", drawEdges ? "0.5" : "none") + .append("title") + .text(d => { + let count = 0, + perc = 0, + tooltip = `Count: ${count}\n + Percentage: ${perc.toFixed(2)}%\n + Center: ${d.x.toFixed(2)}, ${d.y.toFixed(2)}\n + `.replace(/\s{2,}/g, "\n"); + return tooltip; + }); + } + marginedSpace.append("g") .attr("class", "hexagon") .attr("clip-path", "url(#clip)") From 1c87707a76652642b544ed993a6289b2fc9a4053 Mon Sep 17 00:00:00 2001 From: toby Date: Mon, 5 Jun 2017 10:24:06 -0400 Subject: [PATCH 06/11] Add heatmap chart operation --- src/core/config/OperationConfig.js | 58 ++++++++++ src/core/operations/Charts.js | 176 +++++++++++++++++++++++++++++ 2 files changed, 234 insertions(+) diff --git a/src/core/config/OperationConfig.js b/src/core/config/OperationConfig.js index ab38b7cf7..62ba46e58 100755 --- a/src/core/config/OperationConfig.js +++ b/src/core/config/OperationConfig.js @@ -3451,6 +3451,64 @@ const OperationConfig = { value: false, }, ] + }, + "Heatmap chart": { + description: [].join("\n"), + run: Charts.runHeatmapChart, + inputType: "string", + outputType: "html", + args: [ + { + name: "Record delimiter", + type: "option", + value: Charts.RECORD_DELIMITER_OPTIONS, + }, + { + name: "Field delimiter", + type: "option", + value: Charts.FIELD_DELIMITER_OPTIONS, + }, + { + name: "Number of vertical bins", + type: "number", + value: 25, + }, + { + name: "Number of horizontal bins", + type: "number", + value: 25, + }, + { + name: "Use column headers as labels", + type: "boolean", + value: true, + }, + { + name: "X label", + type: "string", + value: "", + }, + { + name: "Y label", + type: "string", + value: "", + }, + { + name: "Draw bin edges", + type: "boolean", + value: false, + }, + { + name: "Min colour value", + type: "string", + value: Charts.COLOURS.min, + }, + { + name: "Max colour value", + type: "string", + value: Charts.COLOURS.max, + }, + ] } }; diff --git a/src/core/operations/Charts.js b/src/core/operations/Charts.js index 447d47b21..5a927ce30 100755 --- a/src/core/operations/Charts.js +++ b/src/core/operations/Charts.js @@ -275,6 +275,182 @@ const Charts = { return svg._groups[0][0].outerHTML; }, + + + /** + * Packs a list of x, y coordinates into a number of bins for use in a heatmap. + * + * @param {Object[]} points + * @param {number} number of vertical bins + * @param {number} number of horizontal bins + * @returns {Object[]} a list of bins (each bin is an Array) with x y coordinates, filled with the points + */ + _getHeatmapPacking(values, vBins, hBins) { + const xBounds = d3.extent(values, d => d[0]), + yBounds = d3.extent(values, d => d[1]), + bins = []; + + if (xBounds[0] === xBounds[1]) throw "Cannot pack points. There is no difference between the minimum and maximum X coordinate."; + if (yBounds[0] === yBounds[1]) throw "Cannot pack points. There is no difference between the minimum and maximum Y coordinate."; + + for (let y = 0; y < vBins; y++) { + bins.push([]); + for (let x = 0; x < hBins; x++) { + let item = []; + item.y = y; + item.x = x; + + bins[y].push(item); + } // x + } // y + + let epsilon = 0.000000001; // This is to clamp values that are exactly the maximum; + + values.forEach(v => { + let fractionOfY = (v[1] - yBounds[0]) / ((yBounds[1] + epsilon) - yBounds[0]), + fractionOfX = (v[0] - xBounds[0]) / ((xBounds[1] + epsilon) - xBounds[0]); + let y = Math.floor(vBins * fractionOfY), + x = Math.floor(hBins * fractionOfX); + + bins[y][x].push({x: v[0], y: v[1]}); + }); + + return bins; + }, + + + /** + * Heatmap chart operation. + * + * @param {string} input + * @param {Object[]} args + * @returns {html} + */ + runHeatmapChart: function (input, args) { + const recordDelimiter = Utils.charRep[args[0]], + fieldDelimiter = Utils.charRep[args[1]], + vBins = args[2], + hBins = args[3], + columnHeadingsAreIncluded = args[4], + drawEdges = args[7], + minColour = args[8], + maxColour = args[9], + dimension = 500; + + if (vBins <= 0) throw "Number of vertical bins must be greater than 0"; + if (hBins <= 0) throw "Number of horizontal bins must be greater than 0"; + + let xLabel = args[5], + yLabel = args[6], + { headings, values } = Charts._getScatterValues( + input, + recordDelimiter, + fieldDelimiter, + columnHeadingsAreIncluded + ); + + if (headings) { + xLabel = headings.x; + yLabel = headings.y; + } + + let svg = document.createElement("svg"); + svg = d3.select(svg) + .attr("width", "100%") + .attr("height", "100%") + .attr("viewBox", `0 0 ${dimension} ${dimension}`); + + let margin = { + top: 10, + right: 0, + bottom: 40, + left: 30, + }, + width = dimension - margin.left - margin.right, + height = dimension - margin.top - margin.bottom, + binWidth = width / hBins, + binHeight = height/ vBins, + marginedSpace = svg.append("g") + .attr("transform", "translate(" + margin.left + "," + margin.top + ")"); + + let bins = Charts._getHeatmapPacking(values, vBins, hBins), + maxCount = Math.max(...bins.map(row => { + let lengths = row.map(cell => cell.length); + return Math.max(...lengths); + })); + + let xExtent = d3.extent(values, d => d[0]), + yExtent = d3.extent(values, d => d[1]); + + let xAxis = d3.scaleLinear() + .domain(xExtent) + .range([0, width]); + let yAxis = d3.scaleLinear() + .domain(yExtent) + .range([height, 0]); + + let colour = d3.scaleSequential(d3.interpolateLab(minColour, maxColour)) + .domain([0, maxCount]); + + marginedSpace.append("clipPath") + .attr("id", "clip") + .append("rect") + .attr("width", width) + .attr("height", height); + + marginedSpace.append("g") + .attr("class", "bins") + .attr("clip-path", "url(#clip)") + .selectAll("g") + .data(bins) + .enter() + .append("g") + .selectAll("rect") + .data(d => d) + .enter() + .append("rect") + .attr("x", (d) => binWidth * d.x) + .attr("y", (d) => (height - binHeight * (d.y + 1))) + .attr("width", binWidth) + .attr("height", binHeight) + .attr("fill", (d) => colour(d.length)) + .attr("stroke", drawEdges ? "rgba(0, 0, 0, 0.5)" : "none") + .attr("stroke-width", drawEdges ? "0.5" : "none") + .append("title") + .text(d => { + let count = d.length, + perc = 100.0 * d.length / values.length, + tooltip = `Count: ${count}\n + Percentage: ${perc.toFixed(2)}%\n + `.replace(/\s{2,}/g, "\n"); + return tooltip; + }); + + marginedSpace.append("g") + .attr("class", "axis axis--y") + .call(d3.axisLeft(yAxis).tickSizeOuter(-width)); + + svg.append("text") + .attr("transform", "rotate(-90)") + .attr("y", -margin.left) + .attr("x", -(height / 2)) + .attr("dy", "1em") + .style("text-anchor", "middle") + .text(yLabel); + + marginedSpace.append("g") + .attr("class", "axis axis--x") + .attr("transform", "translate(0," + height + ")") + .call(d3.axisBottom(xAxis).tickSizeOuter(-height)); + + svg.append("text") + .attr("x", width / 2) + .attr("y", dimension) + .style("text-anchor", "middle") + .text(xLabel); + + return svg._groups[0][0].outerHTML; + }, }; export default Charts; From 594456856592d936711f52a5a6cde5cd937694d5 Mon Sep 17 00:00:00 2001 From: toby Date: Mon, 5 Jun 2017 10:24:15 -0400 Subject: [PATCH 07/11] Change margins in hex density chart --- src/core/operations/Charts.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/operations/Charts.js b/src/core/operations/Charts.js index 5a927ce30..2202e0f15 100755 --- a/src/core/operations/Charts.js +++ b/src/core/operations/Charts.js @@ -151,9 +151,9 @@ const Charts = { .attr("viewBox", `0 0 ${dimension} ${dimension}`); let margin = { - top: 0, + top: 10, right: 0, - bottom: 30, + bottom: 40, left: 30, }, width = dimension - margin.left - margin.right, From 247e9bfbdeaa113b37ff1bea35c1db624a71a720 Mon Sep 17 00:00:00 2001 From: toby Date: Mon, 5 Jun 2017 21:47:32 -0400 Subject: [PATCH 08/11] Add "HTML to Text" operation --- src/core/config/OperationConfig.js | 8 ++++++++ src/core/operations/HTML.js | 10 ++++++++++ 2 files changed, 18 insertions(+) diff --git a/src/core/config/OperationConfig.js b/src/core/config/OperationConfig.js index 62ba46e58..cf8363f82 100755 --- a/src/core/config/OperationConfig.js +++ b/src/core/config/OperationConfig.js @@ -3509,6 +3509,14 @@ const OperationConfig = { value: Charts.COLOURS.max, }, ] + }, + "HTML to Text": { + description: [].join("\n"), + run: HTML.runHTMLToText, + inputType: "html", + outputType: "string", + args: [ + ] } }; diff --git a/src/core/operations/HTML.js b/src/core/operations/HTML.js index 601d61029..457124bed 100755 --- a/src/core/operations/HTML.js +++ b/src/core/operations/HTML.js @@ -851,6 +851,16 @@ const HTML = { "diams" : 9830, }, + /** + * HTML to text operation + * + * @param {string} input + * @param {Object[]} args + * @returns {string} + */ + runHTMLToText(input, args) { + return input; + }, }; export default HTML; From 49ea532cdc36cb6a7a52ede3cc04b40e771a3d24 Mon Sep 17 00:00:00 2001 From: toby Date: Tue, 6 Jun 2017 09:46:46 -0400 Subject: [PATCH 09/11] Tweak extent of hex density charts --- src/core/operations/Charts.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/operations/Charts.js b/src/core/operations/Charts.js index 2202e0f15..e47d26e26 100755 --- a/src/core/operations/Charts.js +++ b/src/core/operations/Charts.js @@ -171,7 +171,7 @@ const Charts = { let xExtent = d3.extent(hexPoints, d => d.x), yExtent = d3.extent(hexPoints, d => d.y); xExtent[0] -= 2 * packRadius; - xExtent[1] += 2 * packRadius; + xExtent[1] += 3 * packRadius; yExtent[0] -= 2 * packRadius; yExtent[1] += 2 * packRadius; From 39ab60088774f5375206209d59cadfbf2e2a84e8 Mon Sep 17 00:00:00 2001 From: toby Date: Tue, 6 Jun 2017 14:01:23 -0400 Subject: [PATCH 10/11] Add scatter plot operation --- src/core/config/OperationConfig.js | 48 +++++++ src/core/operations/Charts.js | 223 ++++++++++++++++++++++++++--- 2 files changed, 249 insertions(+), 22 deletions(-) diff --git a/src/core/config/OperationConfig.js b/src/core/config/OperationConfig.js index cf8363f82..d0565e7c2 100755 --- a/src/core/config/OperationConfig.js +++ b/src/core/config/OperationConfig.js @@ -3510,6 +3510,54 @@ const OperationConfig = { }, ] }, + "Scatter chart": { + description: [].join("\n"), + run: Charts.runScatterChart, + inputType: "string", + outputType: "html", + args: [ + { + name: "Record delimiter", + type: "option", + value: Charts.RECORD_DELIMITER_OPTIONS, + }, + { + name: "Field delimiter", + type: "option", + value: Charts.FIELD_DELIMITER_OPTIONS, + }, + { + name: "Use column headers as labels", + type: "boolean", + value: true, + }, + { + name: "X label", + type: "string", + value: "", + }, + { + name: "Y label", + type: "string", + value: "", + }, + { + name: "Colour", + type: "string", + value: Charts.COLOURS.max, + }, + { + name: "Point radius", + type: "number", + value: 10, + }, + { + name: "Use colour from third column", + type: "boolean", + value: false, + }, + ] + }, "HTML to Text": { description: [].join("\n"), run: HTML.runHTMLToText, diff --git a/src/core/operations/Charts.js b/src/core/operations/Charts.js index e47d26e26..06a3cb623 100755 --- a/src/core/operations/Charts.js +++ b/src/core/operations/Charts.js @@ -27,7 +27,19 @@ const Charts = { /** - * Gets values from input for a scatter plot. + * Default from colour + * + * @constant + * @default + */ + COLOURS: { + min: "white", + max: "black", + }, + + + /** + * Gets values from input for a plot. * * @param {string} input * @param {string} recordDelimiter @@ -35,7 +47,7 @@ const Charts = { * @param {boolean} columnHeadingsAreIncluded - whether we should skip the first record * @returns {Object[]} */ - _getScatterValues(input, recordDelimiter, fieldDelimiter, columnHeadingsAreIncluded) { + _getValues(input, recordDelimiter, fieldDelimiter, columnHeadingsAreIncluded, length) { let headings; const values = []; @@ -44,23 +56,12 @@ const Charts = { .forEach((row, rowIndex) => { let split = row.split(fieldDelimiter); - if (split.length !== 2) throw "Each row must have length 2."; + if (split.length !== length) throw `Each row must have length ${length}.`; if (columnHeadingsAreIncluded && rowIndex === 0) { - headings = {}; - headings.x = split[0]; - headings.y = split[1]; + headings = split; } else { - let x = split[0], - y = split[1]; - - x = parseFloat(x, 10); - if (Number.isNaN(x)) throw "Values must be numbers in base 10."; - - y = parseFloat(y, 10); - if (Number.isNaN(y)) throw "Values must be numbers in base 10."; - - values.push([x, y]); + values.push(split); } }); @@ -69,14 +70,73 @@ const Charts = { /** - * Default from colour + * Gets values from input for a scatter plot. * - * @constant - * @default + * @param {string} input + * @param {string} recordDelimiter + * @param {string} fieldDelimiter + * @param {boolean} columnHeadingsAreIncluded - whether we should skip the first record + * @returns {Object[]} */ - COLOURS: { - min: "white", - max: "black", + _getScatterValues(input, recordDelimiter, fieldDelimiter, columnHeadingsAreIncluded) { + let { headings, values } = Charts._getValues( + input, + recordDelimiter, fieldDelimiter, + columnHeadingsAreIncluded, + 2 + ); + + if (headings) { + headings = {x: headings[0], y: headings[1]}; + } + + values = values.map(row => { + let x = parseFloat(row[0], 10), + y = parseFloat(row[1], 10); + + if (Number.isNaN(x)) throw "Values must be numbers in base 10."; + if (Number.isNaN(y)) throw "Values must be numbers in base 10."; + + return [x, y]; + }); + + return { headings, values }; + }, + + + /** + * Gets values from input for a scatter plot with colour from the third column. + * + * @param {string} input + * @param {string} recordDelimiter + * @param {string} fieldDelimiter + * @param {boolean} columnHeadingsAreIncluded - whether we should skip the first record + * @returns {Object[]} + */ + _getScatterValuesWithColour(input, recordDelimiter, fieldDelimiter, columnHeadingsAreIncluded) { + let { headings, values } = Charts._getValues( + input, + recordDelimiter, fieldDelimiter, + columnHeadingsAreIncluded, + 3 + ); + + if (headings) { + headings = {x: headings[0], y: headings[1]}; + } + + values = values.map(row => { + let x = parseFloat(row[0], 10), + y = parseFloat(row[1], 10), + colour = row[2]; + + if (Number.isNaN(x)) throw "Values must be numbers in base 10."; + if (Number.isNaN(y)) throw "Values must be numbers in base 10."; + + return [x, y, colour]; + }); + + return { headings, values }; }, @@ -451,6 +511,125 @@ const Charts = { return svg._groups[0][0].outerHTML; }, + + + /** + * Scatter chart operation. + * + * @param {string} input + * @param {Object[]} args + * @returns {html} + */ + runScatterChart: function (input, args) { + const recordDelimiter = Utils.charRep[args[0]], + fieldDelimiter = Utils.charRep[args[1]], + columnHeadingsAreIncluded = args[2], + fillColour = args[5], + radius = args[6], + colourInInput = args[7], + dimension = 500; + + let xLabel = args[3], + yLabel = args[4]; + + let dataFunction = colourInInput ? Charts._getScatterValuesWithColour : Charts._getScatterValues; + + let { headings, values } = dataFunction( + input, + recordDelimiter, + fieldDelimiter, + columnHeadingsAreIncluded + ); + + if (headings) { + xLabel = headings.x; + yLabel = headings.y; + } + + let svg = document.createElement("svg"); + svg = d3.select(svg) + .attr("width", "100%") + .attr("height", "100%") + .attr("viewBox", `0 0 ${dimension} ${dimension}`); + + let margin = { + top: 10, + right: 0, + bottom: 40, + left: 30, + }, + width = dimension - margin.left - margin.right, + height = dimension - margin.top - margin.bottom, + marginedSpace = svg.append("g") + .attr("transform", "translate(" + margin.left + "," + margin.top + ")"); + + let xExtent = d3.extent(values, d => d[0]), + xDelta = xExtent[1] - xExtent[0], + yExtent = d3.extent(values, d => d[1]), + yDelta = yExtent[1] - yExtent[0], + xAxis = d3.scaleLinear() + .domain([xExtent[0] - (0.1 * xDelta), xExtent[1] + (0.1 * xDelta)]) + .range([0, width]), + yAxis = d3.scaleLinear() + .domain([yExtent[0] - (0.1 * yDelta), yExtent[1] + (0.1 * yDelta)]) + .range([height, 0]); + + marginedSpace.append("clipPath") + .attr("id", "clip") + .append("rect") + .attr("width", width) + .attr("height", height); + + marginedSpace.append("g") + .attr("class", "points") + .attr("clip-path", "url(#clip)") + .selectAll("circle") + .data(values) + .enter() + .append("circle") + .attr("cx", (d) => xAxis(d[0])) + .attr("cy", (d) => yAxis(d[1])) + .attr("r", d => radius) + .attr("fill", d => { + return colourInInput ? d[2] : fillColour; + }) + .attr("stroke", "rgba(0, 0, 0, 0.5)") + .attr("stroke-width", "0.5") + .append("title") + .text(d => { + let x = d[0], + y = d[1], + tooltip = `X: ${x}\n + Y: ${y}\n + `.replace(/\s{2,}/g, "\n"); + return tooltip; + }); + + marginedSpace.append("g") + .attr("class", "axis axis--y") + .call(d3.axisLeft(yAxis).tickSizeOuter(-width)); + + svg.append("text") + .attr("transform", "rotate(-90)") + .attr("y", -margin.left) + .attr("x", -(height / 2)) + .attr("dy", "1em") + .style("text-anchor", "middle") + .text(yLabel); + + marginedSpace.append("g") + .attr("class", "axis axis--x") + .attr("transform", "translate(0," + height + ")") + .call(d3.axisBottom(xAxis).tickSizeOuter(-height)); + + svg.append("text") + .attr("x", width / 2) + .attr("y", dimension) + .style("text-anchor", "middle") + .text(xLabel); + + return svg._groups[0][0].outerHTML; + }, }; export default Charts; From 6784a1c0276c83e7e35020f3953a6b839c67239d Mon Sep 17 00:00:00 2001 From: toby Date: Tue, 20 Jun 2017 15:25:16 -0400 Subject: [PATCH 11/11] Add Series chart operation --- src/core/config/OperationConfig.js | 33 +++++ src/core/operations/Charts.js | 208 ++++++++++++++++++++++++++++- 2 files changed, 240 insertions(+), 1 deletion(-) diff --git a/src/core/config/OperationConfig.js b/src/core/config/OperationConfig.js index d0565e7c2..f9b5937d2 100755 --- a/src/core/config/OperationConfig.js +++ b/src/core/config/OperationConfig.js @@ -3558,6 +3558,39 @@ const OperationConfig = { }, ] }, + "Series chart": { + description: [].join("\n"), + run: Charts.runSeriesChart, + inputType: "string", + outputType: "html", + args: [ + { + name: "Record delimiter", + type: "option", + value: Charts.RECORD_DELIMITER_OPTIONS, + }, + { + name: "Field delimiter", + type: "option", + value: Charts.FIELD_DELIMITER_OPTIONS, + }, + { + name: "X label", + type: "string", + value: "", + }, + { + name: "Point radius", + type: "number", + value: 1, + }, + { + name: "Series colours", + type: "string", + value: "mediumseagreen, dodgerblue, tomato", + }, + ] + }, "HTML to Text": { description: [].join("\n"), run: HTML.runHTMLToText, diff --git a/src/core/operations/Charts.js b/src/core/operations/Charts.js index 06a3cb623..2ce084d04 100755 --- a/src/core/operations/Charts.js +++ b/src/core/operations/Charts.js @@ -103,7 +103,7 @@ const Charts = { return { headings, values }; }, - + /** * Gets values from input for a scatter plot with colour from the third column. * @@ -140,6 +140,50 @@ const Charts = { }, + /** + * Gets values from input for a time series plot. + * + * @param {string} input + * @param {string} recordDelimiter + * @param {string} fieldDelimiter + * @param {boolean} columnHeadingsAreIncluded - whether we should skip the first record + * @returns {Object[]} + */ + _getSeriesValues(input, recordDelimiter, fieldDelimiter, columnHeadingsAreIncluded) { + let { headings, values } = Charts._getValues( + input, + recordDelimiter, fieldDelimiter, + false, + 3 + ); + + let xValues = new Set(), + series = {}; + + values = values.forEach(row => { + let serie = row[0], + xVal = row[1], + val = parseFloat(row[2], 10); + + if (Number.isNaN(val)) throw "Values must be numbers in base 10."; + + xValues.add(xVal); + if (typeof series[serie] === "undefined") series[serie] = {}; + series[serie][xVal] = val; + }); + + xValues = new Array(...xValues); + + const seriesList = []; + for (let seriesName in series) { + let serie = series[seriesName]; + seriesList.push({name: seriesName, data: serie}); + } + + return { xValues, series: seriesList }; + }, + + /** * Hex Bin chart operation. * @@ -630,6 +674,168 @@ const Charts = { return svg._groups[0][0].outerHTML; }, + + + /** + * Series chart operation. + * + * @param {string} input + * @param {Object[]} args + * @returns {html} + */ + runSeriesChart(input, args) { + const recordDelimiter = Utils.charRep[args[0]], + fieldDelimiter = Utils.charRep[args[1]], + xLabel = args[2], + pipRadius = args[3], + seriesColours = args[4].split(","), + svgWidth = 500, + interSeriesPadding = 20, + xAxisHeight = 50, + seriesLabelWidth = 50, + seriesHeight = 100, + seriesWidth = svgWidth - seriesLabelWidth - interSeriesPadding; + + let { xValues, series } = Charts._getSeriesValues(input, recordDelimiter, fieldDelimiter), + allSeriesHeight = Object.keys(series).length * (interSeriesPadding + seriesHeight), + svgHeight = allSeriesHeight + xAxisHeight + interSeriesPadding; + + let svg = document.createElement("svg"); + svg = d3.select(svg) + .attr("width", "100%") + .attr("height", "100%") + .attr("viewBox", `0 0 ${svgWidth} ${svgHeight}`); + + let xAxis = d3.scalePoint() + .domain(xValues) + .range([0, seriesWidth]); + + svg.append("g") + .attr("class", "axis axis--x") + .attr("transform", `translate(${seriesLabelWidth}, ${xAxisHeight})`) + .call( + d3.axisTop(xAxis).tickValues(xValues.filter((x, i) => { + return [0, Math.round(xValues.length / 2), xValues.length -1].indexOf(i) >= 0; + })) + ); + + svg.append("text") + .attr("x", svgWidth / 2) + .attr("y", xAxisHeight / 2) + .style("text-anchor", "middle") + .text(xLabel); + + let tooltipText = {}, + tooltipAreaWidth = seriesWidth / xValues.length; + + xValues.forEach(x => { + let tooltip = []; + + series.forEach(serie => { + let y = serie.data[x]; + if (typeof y === "undefined") return; + + tooltip.push(`${serie.name}: ${y}`); + }); + + tooltipText[x] = tooltip.join("\n"); + }); + + let chartArea = svg.append("g") + .attr("transform", `translate(${seriesLabelWidth}, ${xAxisHeight})`); + + chartArea + .append("g") + .selectAll("rect") + .data(xValues) + .enter() + .append("rect") + .attr("x", x => { + return xAxis(x) - (tooltipAreaWidth / 2); + }) + .attr("y", 0) + .attr("width", tooltipAreaWidth) + .attr("height", allSeriesHeight) + .attr("stroke", "none") + .attr("fill", "transparent") + .append("title") + .text(x => { + return `${x}\n + --\n + ${tooltipText[x]}\n + `.replace(/\s{2,}/g, "\n"); + }); + + let yAxesArea = svg.append("g") + .attr("transform", `translate(0, ${xAxisHeight})`); + + series.forEach((serie, seriesIndex) => { + let yExtent = d3.extent(Object.values(serie.data)), + yAxis = d3.scaleLinear() + .domain(yExtent) + .range([seriesHeight, 0]); + + let seriesGroup = chartArea + .append("g") + .attr("transform", `translate(0, ${seriesHeight * seriesIndex + interSeriesPadding * (seriesIndex + 1)})`); + + let path = ""; + xValues.forEach((x, xIndex) => { + let nextX = xValues[xIndex + 1], + y = serie.data[x], + nextY= serie.data[nextX]; + + if (typeof y === "undefined" || typeof nextY === "undefined") return; + + x = xAxis(x); nextX = xAxis(nextX); + y = yAxis(y); nextY = yAxis(nextY); + + path += `M ${x} ${y} L ${nextX} ${nextY} z `; + }); + + seriesGroup + .append("path") + .attr("d", path) + .attr("fill", "none") + .attr("stroke", seriesColours[seriesIndex % seriesColours.length]) + .attr("stroke-width", "1"); + + xValues.forEach(x => { + let y = serie.data[x]; + if (typeof y === "undefined") return; + + seriesGroup + .append("circle") + .attr("cx", xAxis(x)) + .attr("cy", yAxis(y)) + .attr("r", pipRadius) + .attr("fill", seriesColours[seriesIndex % seriesColours.length]) + .append("title") + .text(d => { + return `${x}\n + --\n + ${tooltipText[x]}\n + `.replace(/\s{2,}/g, "\n"); + }); + }); + + yAxesArea + .append("g") + .attr("transform", `translate(${seriesLabelWidth - interSeriesPadding}, ${seriesHeight * seriesIndex + interSeriesPadding * (seriesIndex + 1)})`) + .attr("class", "axis axis--y") + .call(d3.axisLeft(yAxis).ticks(5)); + + yAxesArea + .append("g") + .attr("transform", `translate(0, ${seriesHeight / 2 + seriesHeight * seriesIndex + interSeriesPadding * (seriesIndex + 1)})`) + .append("text") + .style("text-anchor", "middle") + .attr("transform", "rotate(-90)") + .text(serie.name); + }); + + return svg._groups[0][0].outerHTML; + }, }; export default Charts;