From dc7760247be4610e561709d1d6ba50bb0ce810e3 Mon Sep 17 00:00:00 2001 From: Alice Date: Tue, 23 Jan 2024 13:52:50 -0500 Subject: [PATCH 1/5] Add MurmurHash3 --- src/core/config/Categories.json | 1 + src/core/operations/Murmurhash3.mjs | 113 ++++++++++++++++++++++++++++ 2 files changed, 114 insertions(+) create mode 100644 src/core/operations/Murmurhash3.mjs diff --git a/src/core/config/Categories.json b/src/core/config/Categories.json index c8f83f95e..28bbeefaf 100644 --- a/src/core/config/Categories.json +++ b/src/core/config/Categories.json @@ -369,6 +369,7 @@ "SHA2", "SHA3", "SM3", + "MurmurHash3", "Keccak", "Shake", "RIPEMD", diff --git a/src/core/operations/Murmurhash3.mjs b/src/core/operations/Murmurhash3.mjs new file mode 100644 index 000000000..c646a539e --- /dev/null +++ b/src/core/operations/Murmurhash3.mjs @@ -0,0 +1,113 @@ +/** + * @author AliceGrey [alice@grey.systems] + * @copyright Crown Copyright 2024 + * @license Apache-2.0 + */ + +import Operation from "../Operation.mjs"; + +/** + * MurmurHash3 operation + */ +class MurmurHash3 extends Operation { + + /** + * MurmurHash3 constructor + */ + constructor() { + super(); + + this.name = "MurmurHash3"; + this.module = "Default"; + this.description = "Generates a MurmurHash v3 for a string input and an optional seed input"; + this.infoURL = "https://wikipedia.org/wiki/MurmurHash"; + this.inputType = "string"; + this.outputType = "number"; + this.args = [ + { + name: "Seed", + type: "number", + value: 0 + } + ]; + } + + /** + * Calculates the MurmurHash3 hash of the input. + * Based on Gary Court's JS MurmurHash implementation + * @see http://github.com/garycourt/murmurhash-js + * @author AliceGrey [alice@grey.systems] + * @param {string} input ASCII only + * @param {number} seed Positive integer only + * @return {number} 32-bit positive integer hash + */ + mmh3(input, seed) { + let h1, h1b, k1, i; + const remainder = input.length & 3; // input.length % 4 + const bytes = input.length - remainder; + h1 = seed; + const c1 = 0xcc9e2d51; + const c2 = 0x1b873593; + i = 0; + + while (i < bytes) { + k1 = + ((input.charCodeAt(i) & 0xff)) | + ((input.charCodeAt(++i) & 0xff) << 8) | + ((input.charCodeAt(++i) & 0xff) << 16) | + ((input.charCodeAt(++i) & 0xff) << 24); + ++i; + + k1 = ((((k1 & 0xffff) * c1) + ((((k1 >>> 16) * c1) & 0xffff) << 16))) & 0xffffffff; + k1 = (k1 << 15) | (k1 >>> 17); + k1 = ((((k1 & 0xffff) * c2) + ((((k1 >>> 16) * c2) & 0xffff) << 16))) & 0xffffffff; + + h1 ^= k1; + h1 = (h1 << 13) | (h1 >>> 19); + h1b = ((((h1 & 0xffff) * 5) + ((((h1 >>> 16) * 5) & 0xffff) << 16))) & 0xffffffff; + h1 = (((h1b & 0xffff) + 0x6b64) + ((((h1b >>> 16) + 0xe654) & 0xffff) << 16)); + } + + k1 = 0; + + if (remainder === 3) { + k1 ^= (input.charCodeAt(i + 2) & 0xff) << 16; + } + if (remainder === 2) { + k1 ^= (input.charCodeAt(i + 1) & 0xff) << 8; + } + if (remainder === 1) { + k1 ^= (input.charCodeAt(i) & 0xff); + } + k1 = (((k1 & 0xffff) * c1) + ((((k1 >>> 16) * c1) & 0xffff) << 16)) & 0xffffffff; + k1 = (k1 << 15) | (k1 >>> 17); + k1 = (((k1 & 0xffff) * c2) + ((((k1 >>> 16) * c2) & 0xffff) << 16)) & 0xffffffff; + h1 ^= k1; + + h1 ^= input.length; + + h1 ^= h1 >>> 16; + h1 = (((h1 & 0xffff) * 0x85ebca6b) + ((((h1 >>> 16) * 0x85ebca6b) & 0xffff) << 16)) & 0xffffffff; + h1 ^= h1 >>> 13; + h1 = ((((h1 & 0xffff) * 0xc2b2ae35) + ((((h1 >>> 16) * 0xc2b2ae35) & 0xffff) << 16))) & 0xffffffff; + h1 ^= h1 >>> 16; + + return h1 >>> 0; + } + + /** + * @param {string} input + * @param {Object[]} args + * @returns {number} + */ + run(input, args) { + let seed; + if (args && args.length > 0) { + seed = args[0]; + } + return this.mmh3(input, seed); + } + +} + +export default MurmurHash3; From 20db43c0a88140dbc703e0a2ac40c507c0d6fc8b Mon Sep 17 00:00:00 2001 From: Alice Date: Tue, 6 Feb 2024 12:22:08 -0500 Subject: [PATCH 2/5] Add MIT License --- src/core/operations/Murmurhash3.mjs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/core/operations/Murmurhash3.mjs b/src/core/operations/Murmurhash3.mjs index c646a539e..83a008edb 100644 --- a/src/core/operations/Murmurhash3.mjs +++ b/src/core/operations/Murmurhash3.mjs @@ -1,4 +1,8 @@ /** + * Based on murmurhash-js (https://github.com/garycourt/murmurhash-js) + * @author Gary Court + * @license MIT + * * @author AliceGrey [alice@grey.systems] * @copyright Crown Copyright 2024 * @license Apache-2.0 From 59b97bfccbd3110a66979d77706ad159e6090e53 Mon Sep 17 00:00:00 2001 From: Alice Date: Tue, 6 Feb 2024 13:03:15 -0500 Subject: [PATCH 3/5] Add MurmurHash3 Tests and normalize filename --- tests/operations/index.mjs | 1 + tests/operations/tests/MurmurHash3.mjs | 55 ++++++++++++++++++++++++++ 2 files changed, 56 insertions(+) create mode 100644 tests/operations/tests/MurmurHash3.mjs diff --git a/tests/operations/index.mjs b/tests/operations/index.mjs index e61f542d9..9a04e4688 100644 --- a/tests/operations/index.mjs +++ b/tests/operations/index.mjs @@ -66,6 +66,7 @@ import "./tests/LZNT1Decompress.mjs"; import "./tests/MS.mjs"; import "./tests/Magic.mjs"; import "./tests/MorseCode.mjs"; +import "./tests/MurmurHash3.mjs"; import "./tests/NetBIOS.mjs"; import "./tests/NormaliseUnicode.mjs"; import "./tests/OTP.mjs"; diff --git a/tests/operations/tests/MurmurHash3.mjs b/tests/operations/tests/MurmurHash3.mjs new file mode 100644 index 000000000..d6490c7b5 --- /dev/null +++ b/tests/operations/tests/MurmurHash3.mjs @@ -0,0 +1,55 @@ +/** + * MurmurHash3 tests + * @author AliceGrey [alice@grey.systems] + * @copyright Crown Copyright 2024 + * @license Apache-2.0 + */ + +import TestRegister from "../../lib/TestRegister.mjs"; + +TestRegister.addTests([ + { + name: "To MurmurHash3: nothing", + input: "", + expectedOutput: "0", + recipeConfig: [ + { + op: "MurmurHash3", + args: [0], + }, + ], + }, + { + name: "To MurmurHash3: 1", + input: "1", + expectedOutput: "2484513939", + recipeConfig: [ + { + op: "MurmurHash3", + args: [0], + }, + ], + }, + { + name: "To MurmurHash3: Hello World!", + input: "Hello World!", + expectedOutput: "3691591037", + recipeConfig: [ + { + op: "MurmurHash3", + args: [0], + }, + ], + }, + { + name: "To MurmurHash3: Hello World! with seed", + input: "Hello World!", + expectedOutput: "1148600031", + recipeConfig: [ + { + op: "MurmurHash3", + args: [1337], + }, + ], + } +]); From afcf46561ae552d58afe6c3a194101a40c64b62e Mon Sep 17 00:00:00 2001 From: Allie Date: Tue, 6 Feb 2024 13:13:43 -0500 Subject: [PATCH 4/5] Rename Murmurhash3.mjs to MurmurHash3.mjs --- src/core/operations/{Murmurhash3.mjs => MurmurHash3.mjs} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/core/operations/{Murmurhash3.mjs => MurmurHash3.mjs} (100%) diff --git a/src/core/operations/Murmurhash3.mjs b/src/core/operations/MurmurHash3.mjs similarity index 100% rename from src/core/operations/Murmurhash3.mjs rename to src/core/operations/MurmurHash3.mjs From cfc8a506f7b13db136a62a9cec80ceda04c0dedc Mon Sep 17 00:00:00 2001 From: Alice Date: Tue, 6 Feb 2024 15:21:39 -0500 Subject: [PATCH 5/5] Fix calculation bug and add Convert to Signed A signed output is often needed for Shodan and other favicon searches. --- src/core/operations/MurmurHash3.mjs | 72 ++++++++++++++++++-------- tests/operations/tests/MurmurHash3.mjs | 22 ++++++++ 2 files changed, 71 insertions(+), 23 deletions(-) diff --git a/src/core/operations/MurmurHash3.mjs b/src/core/operations/MurmurHash3.mjs index 83a008edb..f4aea611e 100644 --- a/src/core/operations/MurmurHash3.mjs +++ b/src/core/operations/MurmurHash3.mjs @@ -32,27 +32,33 @@ class MurmurHash3 extends Operation { name: "Seed", type: "number", value: 0 + }, + { + name: "Convert to Signed", + type: "boolean", + value: false } ]; } - /** - * Calculates the MurmurHash3 hash of the input. - * Based on Gary Court's JS MurmurHash implementation - * @see http://github.com/garycourt/murmurhash-js - * @author AliceGrey [alice@grey.systems] - * @param {string} input ASCII only - * @param {number} seed Positive integer only - * @return {number} 32-bit positive integer hash - */ + /** + * Calculates the MurmurHash3 hash of the input. + * Based on Gary Court's JS MurmurHash implementation + * @see http://github.com/garycourt/murmurhash-js + * @author AliceGrey [alice@grey.systems] + * @param {string} input ASCII only + * @param {number} seed Positive integer only + * @return {number} 32-bit positive integer hash + */ mmh3(input, seed) { - let h1, h1b, k1, i; + let h1b; + let k1; const remainder = input.length & 3; // input.length % 4 const bytes = input.length - remainder; - h1 = seed; + let h1 = seed; const c1 = 0xcc9e2d51; const c2 = 0x1b873593; - i = 0; + let i = 0; while (i < bytes) { k1 = @@ -77,16 +83,19 @@ class MurmurHash3 extends Operation { if (remainder === 3) { k1 ^= (input.charCodeAt(i + 2) & 0xff) << 16; } - if (remainder === 2) { + + if (remainder === 3 || remainder === 2) { k1 ^= (input.charCodeAt(i + 1) & 0xff) << 8; } - if (remainder === 1) { + + if (remainder === 3 || remainder === 2 || remainder === 1) { k1 ^= (input.charCodeAt(i) & 0xff); + + k1 = (((k1 & 0xffff) * c1) + ((((k1 >>> 16) * c1) & 0xffff) << 16)) & 0xffffffff; + k1 = (k1 << 15) | (k1 >>> 17); + k1 = (((k1 & 0xffff) * c2) + ((((k1 >>> 16) * c2) & 0xffff) << 16)) & 0xffffffff; + h1 ^= k1; } - k1 = (((k1 & 0xffff) * c1) + ((((k1 >>> 16) * c1) & 0xffff) << 16)) & 0xffffffff; - k1 = (k1 << 15) | (k1 >>> 17); - k1 = (((k1 & 0xffff) * c2) + ((((k1 >>> 16) * c2) & 0xffff) << 16)) & 0xffffffff; - h1 ^= k1; h1 ^= input.length; @@ -99,19 +108,36 @@ class MurmurHash3 extends Operation { return h1 >>> 0; } + /** + * Converts an unsigned 32-bit integer to a signed 32-bit integer + * @author AliceGrey [alice@grey.systems] + * @param {value} 32-bit unsigned integer + * @return {number} 32-bit signed integer + */ + unsignedToSigned(value) { + if (value & 0x80000000) { + return -0x100000000 + value; + } else { + return value; + } + } + /** * @param {string} input * @param {Object[]} args * @returns {number} */ run(input, args) { - let seed; - if (args && args.length > 0) { - seed = args[0]; + if (args && args.length >= 1) { + const seed = args[0]; + const hash = this.mmh3(input, seed); + if (args.length > 1 && args[1]) { + return this.unsignedToSigned(hash); + } + return hash; } - return this.mmh3(input, seed); + return this.mmh3(input); } - } export default MurmurHash3; diff --git a/tests/operations/tests/MurmurHash3.mjs b/tests/operations/tests/MurmurHash3.mjs index d6490c7b5..278846102 100644 --- a/tests/operations/tests/MurmurHash3.mjs +++ b/tests/operations/tests/MurmurHash3.mjs @@ -51,5 +51,27 @@ TestRegister.addTests([ args: [1337], }, ], + }, + { + name: "To MurmurHash3: foo", + input: "foo", + expectedOutput: "4138058784", + recipeConfig: [ + { + op: "MurmurHash3", + args: [0], + }, + ], + }, + { + name: "To MurmurHash3: foo signed", + input: "foo", + expectedOutput: "-156908512", + recipeConfig: [ + { + op: "MurmurHash3", + args: [0, true], + }, + ], } ]);