diff --git a/CHANGELOG.md b/CHANGELOG.md index 6eaec2432..1204742fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ All major and minor version changes will be documented in this file. Details of patch-level version changes can be found in [commit messages](https://github.com/gchq/CyberChef/commits/master). +### [9.8.0] - 2019-10-31 +- 'Avro to JSON' operation added [@jarrodconnolly] | [#865] + ### [9.7.0] - 2019-09-13 - 'Optical Character Recognition' operation added [@MShwed] [@n1474335] | [#632] @@ -185,6 +188,7 @@ All major and minor version changes will be documented in this file. Details of +[9.8.0]: https://github.com/gchq/CyberChef/releases/tag/v9.8.0 [9.7.0]: https://github.com/gchq/CyberChef/releases/tag/v9.7.0 [9.6.0]: https://github.com/gchq/CyberChef/releases/tag/v9.6.0 [9.5.0]: https://github.com/gchq/CyberChef/releases/tag/v9.5.0 @@ -262,6 +266,7 @@ All major and minor version changes will be documented in this file. Details of [@Ge0rg3]: https://github.com/Ge0rg3 [@MShwed]: https://github.com/MShwed [@kassi]: https://github.com/kassi +[@jarrodconnolly]: https://github.com/jarrodconnolly [#95]: https://github.com/gchq/CyberChef/pull/299 [#173]: https://github.com/gchq/CyberChef/pull/173 @@ -322,3 +327,4 @@ All major and minor version changes will be documented in this file. Details of [#625]: https://github.com/gchq/CyberChef/pull/625 [#627]: https://github.com/gchq/CyberChef/pull/627 [#632]: https://github.com/gchq/CyberChef/pull/632 +[#865]: https://github.com/gchq/CyberChef/pull/865 diff --git a/package-lock.json b/package-lock.json index b3c6f415e..dd1d3d79f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "cyberchef", - "version": "9.7.19", + "version": "9.8.0", "lockfileVersion": 1, "requires": true, "dependencies": { @@ -2149,6 +2149,11 @@ } } }, + "avsc": { + "version": "5.4.16", + "resolved": "https://registry.npmjs.org/avsc/-/avsc-5.4.16.tgz", + "integrity": "sha512-Z85B8ZaEU2PWNPRJYuMSp5Hg7Nw3KPKW47lW/Kus7AcwV7fr6uJG3UckagqIPLydIeO/Cm+yjnJG7g0tliICOg==" + }, "aws-sign2": { "version": "0.7.0", "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz", diff --git a/package.json b/package.json index 518fb29be..2419ae875 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "cyberchef", - "version": "9.7.19", + "version": "9.8.0", "description": "The Cyber Swiss Army Knife for encryption, encoding, compression and data analysis.", "author": "n1474335 ", "homepage": "https://gchq.github.io/CyberChef", @@ -86,6 +86,7 @@ "@babel/polyfill": "^7.4.4", "@babel/runtime": "^7.5.5", "arrive": "^2.4.1", + "avsc": "^5.4.16", "babel-plugin-transform-builtin-extend": "1.1.2", "bcryptjs": "^2.4.3", "bignumber.js": "^9.0.0", diff --git a/src/core/config/Categories.json b/src/core/config/Categories.json index 9867543d5..f663e16d0 100755 --- a/src/core/config/Categories.json +++ b/src/core/config/Categories.json @@ -59,7 +59,8 @@ "From Braille", "Parse TLV", "CSV to JSON", - "JSON to CSV" + "JSON to CSV", + "Avro to JSON" ] }, { diff --git a/src/core/operations/AvroToJSON.mjs b/src/core/operations/AvroToJSON.mjs new file mode 100644 index 000000000..497a3872b --- /dev/null +++ b/src/core/operations/AvroToJSON.mjs @@ -0,0 +1,76 @@ +/** + * @author jarrodconnolly [jarrod@nestedquotes.ca] + * @copyright Crown Copyright 2019 + * @license Apache-2.0 + */ + +import Operation from "../Operation.mjs"; +import OperationError from "../errors/OperationError.mjs"; +import avro from "avsc"; + +/** + * Avro to JSON operation + */ +class AvroToJSON extends Operation { + + /** + * AvroToJSON constructor + */ + constructor() { + super(); + + this.name = "Avro to JSON"; + this.module = "Serialise"; + this.description = "Converts Avro encoded data into JSON."; + this.infoURL = "https://wikipedia.org/wiki/Apache_Avro"; + this.inputType = "ArrayBuffer"; + this.outputType = "string"; + this.args = [ + { + name: "Force Valid JSON", + type: "boolean", + value: true + } + ]; + } + + /** + * @param {ArrayBuffer} input + * @param {Object[]} args + * @returns {string} + */ + run(input, args) { + if (input.byteLength <= 0) { + throw new OperationError("Please provide an input."); + } + + const forceJSON = args[0]; + + return new Promise((resolve, reject) => { + const result = []; + const inpArray = new Uint8Array(input); + const decoder = new avro.streams.BlockDecoder(); + + decoder + .on("data", function (obj) { + result.push(obj); + }) + .on("error", function () { + reject(new OperationError("Error parsing Avro file.")); + }) + .on("end", function () { + if (forceJSON) { + resolve(result.length === 1 ? JSON.stringify(result[0], null, 4) : JSON.stringify(result, null, 4)); + } else { + const data = result.reduce((result, current) => result + JSON.stringify(current) + "\n", ""); + resolve(data); + } + }); + + decoder.write(inpArray); + decoder.end(); + }); + } +} + +export default AvroToJSON; diff --git a/src/core/operations/BSONDeserialise.mjs b/src/core/operations/BSONDeserialise.mjs index a21eaadd3..cb46b357e 100644 --- a/src/core/operations/BSONDeserialise.mjs +++ b/src/core/operations/BSONDeserialise.mjs @@ -20,7 +20,7 @@ class BSONDeserialise extends Operation { super(); this.name = "BSON deserialise"; - this.module = "BSON"; + this.module = "Serialise"; this.description = "BSON is a computer data interchange format used mainly as a data storage and network transfer format in the MongoDB database. It is a binary form for representing simple data structures, associative arrays (called objects or documents in MongoDB), and various data types of specific interest to MongoDB. The name 'BSON' is based on the term JSON and stands for 'Binary JSON'.

Input data should be in a raw bytes format."; this.infoURL = "https://wikipedia.org/wiki/BSON"; this.inputType = "ArrayBuffer"; diff --git a/src/core/operations/BSONSerialise.mjs b/src/core/operations/BSONSerialise.mjs index 6d33c6be6..25eed876d 100644 --- a/src/core/operations/BSONSerialise.mjs +++ b/src/core/operations/BSONSerialise.mjs @@ -20,7 +20,7 @@ class BSONSerialise extends Operation { super(); this.name = "BSON serialise"; - this.module = "BSON"; + this.module = "Serialise"; this.description = "BSON is a computer data interchange format used mainly as a data storage and network transfer format in the MongoDB database. It is a binary form for representing simple data structures, associative arrays (called objects or documents in MongoDB), and various data types of specific interest to MongoDB. The name 'BSON' is based on the term JSON and stands for 'Binary JSON'.

Input data should be valid JSON."; this.infoURL = "https://wikipedia.org/wiki/BSON"; this.inputType = "string"; diff --git a/src/core/operations/Diff.mjs b/src/core/operations/Diff.mjs index 7adea178c..841072457 100644 --- a/src/core/operations/Diff.mjs +++ b/src/core/operations/Diff.mjs @@ -47,6 +47,11 @@ class Diff extends Operation { "type": "boolean", "value": true }, + { + "name": "Show subtraction", + "type": "boolean", + "value": false + }, { "name": "Ignore whitespace", "type": "boolean", @@ -67,6 +72,7 @@ class Diff extends Operation { diffBy, showAdded, showRemoved, + showSubtraction, ignoreWhitespace ] = args, samples = input.split(sampleDelim); @@ -116,7 +122,7 @@ class Diff extends Operation { if (showAdded) output += "" + Utils.escapeHtml(diff[i].value) + ""; } else if (diff[i].removed) { if (showRemoved) output += "" + Utils.escapeHtml(diff[i].value) + ""; - } else { + } else if (!showSubtraction) { output += Utils.escapeHtml(diff[i].value); } } diff --git a/src/web/HTMLIngredient.mjs b/src/web/HTMLIngredient.mjs index 03f4e9835..fd496f2aa 100755 --- a/src/web/HTMLIngredient.mjs +++ b/src/web/HTMLIngredient.mjs @@ -54,7 +54,9 @@ class HTMLIngredient { case "binaryString": case "byteArray": html += `
- + - ${this.hint ? "" + this.hint + "" : ""}
`; break; case "shortString": case "binaryShortString": html += `
- + - ${this.hint ? "" + this.hint + "" : ""}
`; break; case "toggleString": html += `
- + - ${this.hint ? "" + this.hint + "" : ""}
@@ -105,7 +108,9 @@ class HTMLIngredient { break; case "number": html += `
- + - ${this.hint ? "" + this.hint + "" : ""}
`; break; case "boolean": html += `
-
`; break; case "option": html += `
- + - ${this.hint ? "" + this.hint + "" : ""}
`; eventFn = this.type === "populateMultiOption" ? @@ -191,7 +196,9 @@ class HTMLIngredient { break; case "editableOption": html += `
- + - ${this.hint ? "" + this.hint + "" : ""}
`; break; case "argSelector": html += `
- + - ${this.hint ? "" + this.hint + "" : ""}
`; this.manager.addDynamicListener(".arg-selector", "change", this.argSelectorChange, this); diff --git a/src/web/stylesheets/components/_operation.css b/src/web/stylesheets/components/_operation.css index a4255fc33..39f53a077 100755 --- a/src/web/stylesheets/components/_operation.css +++ b/src/web/stylesheets/components/_operation.css @@ -159,10 +159,6 @@ div.toggle-string { width: calc(100% - 13px); } -.operation .bmd-form-group .bmd-help { - margin-top: -17px; -} - .input-group .form-control { border-top-left-radius: 4px !important; } diff --git a/src/web/waiters/RecipeWaiter.mjs b/src/web/waiters/RecipeWaiter.mjs index d198098b7..ba0e7b11c 100755 --- a/src/web/waiters/RecipeWaiter.mjs +++ b/src/web/waiters/RecipeWaiter.mjs @@ -399,6 +399,8 @@ class RecipeWaiter { this.buildRecipeOperation(item); document.getElementById("rec-list").appendChild(item); + $(item).find("[data-toggle='tooltip']").tooltip(); + item.dispatchEvent(this.manager.operationadd); return item; } diff --git a/tests/operations/index.mjs b/tests/operations/index.mjs index 8c39a179f..c54fa7ef0 100644 --- a/tests/operations/index.mjs +++ b/tests/operations/index.mjs @@ -91,8 +91,10 @@ import "./tests/Protobuf.mjs"; import "./tests/ParseSSHHostKey.mjs"; import "./tests/DefangIP.mjs"; import "./tests/ParseUDP.mjs"; +import "./tests/AvroToJSON.mjs"; import "./tests/Lorenz.mjs"; + // Cannot test operations that use the File type yet // import "./tests/SplitColourChannels.mjs"; diff --git a/tests/operations/tests/AvroToJSON.mjs b/tests/operations/tests/AvroToJSON.mjs new file mode 100644 index 000000000..045abddbc --- /dev/null +++ b/tests/operations/tests/AvroToJSON.mjs @@ -0,0 +1,66 @@ +/** + * Avro to JSON tests. + * + * @author jarrodconnolly [jarrod@nestedquotes.ca] + * @copyright Crown Copyright 2019 + * @license Apache-2.0 + */ + +import TestRegister from "../../lib/TestRegister"; + +TestRegister.addTests([ + { + name: "Avro to JSON: no input (force JSON true)", + input: "", + expectedOutput: "Please provide an input.", + recipeConfig: [ + { + op: "Avro to JSON", + args: [true] + } + ], + }, + { + name: "Avro to JSON: no input (force JSON false)", + input: "", + expectedOutput: "Please provide an input.", + recipeConfig: [ + { + op: "Avro to JSON", + args: [false] + } + ], + }, + { + name: "Avro to JSON: small (force JSON true)", + input: "\x4f\x62\x6a\x01\x04\x16\x61\x76\x72\x6f\x2e\x73\x63\x68\x65\x6d\x61\x96\x01\x7b\x22\x74\x79\x70\x65\x22\x3a\x22\x72\x65" + + "\x63\x6f\x72\x64\x22\x2c\x22\x6e\x61\x6d\x65\x22\x3a\x22\x73\x6d\x61\x6c\x6c\x22\x2c\x22\x66\x69\x65\x6c\x64\x73\x22\x3a" + + "\x5b\x7b\x22\x6e\x61\x6d\x65\x22\x3a\x22\x6e\x61\x6d\x65\x22\x2c\x22\x74\x79\x70\x65\x22\x3a\x22\x73\x74\x72\x69\x6e\x67" + + "\x22\x7d\x5d\x7d\x14\x61\x76\x72\x6f\x2e\x63\x6f\x64\x65\x63\x08\x6e\x75\x6c\x6c\x00\x4e\x02\x47\x63\x2e\x37\x02\xe5\xb7" + + "\x5c\xda\xb9\xa6\x2f\x15\x41\x02\x0e\x0c\x6d\x79\x6e\x61\x6d\x65\x4e\x02\x47\x63\x2e\x37\x02\xe5\xb7\x5c\xda\xb9\xa6\x2f" + + "\x15\x41", + expectedOutput: "{\n \"name\": \"myname\"\n}", + recipeConfig: [ + { + op: "Avro to JSON", + args: [true] + } + ], + }, + { + name: "Avro to JSON: small (force JSON false)", + input: "\x4f\x62\x6a\x01\x04\x16\x61\x76\x72\x6f\x2e\x73\x63\x68\x65\x6d\x61\x96\x01\x7b\x22\x74\x79\x70\x65\x22\x3a\x22\x72\x65" + + "\x63\x6f\x72\x64\x22\x2c\x22\x6e\x61\x6d\x65\x22\x3a\x22\x73\x6d\x61\x6c\x6c\x22\x2c\x22\x66\x69\x65\x6c\x64\x73\x22\x3a" + + "\x5b\x7b\x22\x6e\x61\x6d\x65\x22\x3a\x22\x6e\x61\x6d\x65\x22\x2c\x22\x74\x79\x70\x65\x22\x3a\x22\x73\x74\x72\x69\x6e\x67" + + "\x22\x7d\x5d\x7d\x14\x61\x76\x72\x6f\x2e\x63\x6f\x64\x65\x63\x08\x6e\x75\x6c\x6c\x00\x4e\x02\x47\x63\x2e\x37\x02\xe5\xb7" + + "\x5c\xda\xb9\xa6\x2f\x15\x41\x02\x0e\x0c\x6d\x79\x6e\x61\x6d\x65\x4e\x02\x47\x63\x2e\x37\x02\xe5\xb7\x5c\xda\xb9\xa6\x2f" + + "\x15\x41", + expectedOutput: "{\"name\":\"myname\"}\n", + recipeConfig: [ + { + op: "Avro to JSON", + args: [false] + } + ], + } +]); diff --git a/tests/operations/tests/StrUtils.mjs b/tests/operations/tests/StrUtils.mjs index 9098d2f1e..c78b6b23d 100644 --- a/tests/operations/tests/StrUtils.mjs +++ b/tests/operations/tests/StrUtils.mjs @@ -15,7 +15,29 @@ TestRegister.addTests([ recipeConfig: [ { "op": "Diff", - "args": ["\\n\\n", "Character", true, true, false] + "args": ["\\n\\n", "Character", true, true, false, false] + } + ], + }, + { + name: "Diff added with subtraction, basic usage", + input: "testing23\n\ntesting123", + expectedOutput: "1", + recipeConfig: [ + { + "op": "Diff", + "args": ["\\n\\n", "Character", true, true, true, false] + } + ], + }, + { + name: "Diff removed with subtraction, basic usage", + input: "testing123\n\ntesting3", + expectedOutput: "12", + recipeConfig: [ + { + "op": "Diff", + "args": ["\\n\\n", "Character", true, true, true, false] } ], },