diff --git a/package-lock.json b/package-lock.json index bf7e8528f..96d1ff250 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1743,7 +1743,7 @@ "dependencies": { "es6-promisify": { "version": "5.0.0", - "resolved": "https://registry.npmjs.org/es6-promisify/-/es6-promisify-5.0.0.tgz", + "resolved": "http://registry.npmjs.org/es6-promisify/-/es6-promisify-5.0.0.tgz", "integrity": "sha1-UQnWLz5W6pZ8S2NQWu8IKRyKUgM=", "dev": true, "requires": { @@ -3778,7 +3778,7 @@ }, "deep-eql": { "version": "0.1.3", - "resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-0.1.3.tgz", + "resolved": "http://registry.npmjs.org/deep-eql/-/deep-eql-0.1.3.tgz", "integrity": "sha1-71WKyrjeJSBs1xOQbXTlaTDrafI=", "dev": true, "requires": { @@ -4066,7 +4066,7 @@ }, "duplexer": { "version": "0.1.1", - "resolved": "https://registry.npmjs.org/duplexer/-/duplexer-0.1.1.tgz", + "resolved": "http://registry.npmjs.org/duplexer/-/duplexer-0.1.1.tgz", "integrity": "sha1-rOb/gIwc5mtX0ev5eXessCM0z8E=", "dev": true }, @@ -5263,8 +5263,7 @@ "ansi-regex": { "version": "2.1.1", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "aproba": { "version": "1.2.0", @@ -5285,14 +5284,12 @@ "balanced-match": { "version": "1.0.0", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "brace-expansion": { "version": "1.1.11", "bundled": true, "dev": true, - "optional": true, "requires": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -5307,20 +5304,17 @@ "code-point-at": { "version": "1.1.0", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "concat-map": { "version": "0.0.1", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "console-control-strings": { "version": "1.1.0", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "core-util-is": { "version": "1.0.2", @@ -5437,8 +5431,7 @@ "inherits": { "version": "2.0.3", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "ini": { "version": "1.3.5", @@ -5450,7 +5443,6 @@ "version": "1.0.0", "bundled": true, "dev": true, - "optional": true, "requires": { "number-is-nan": "^1.0.0" } @@ -5465,7 +5457,6 @@ "version": "3.0.4", "bundled": true, "dev": true, - "optional": true, "requires": { "brace-expansion": "^1.1.7" } @@ -5473,14 +5464,12 @@ "minimist": { "version": "0.0.8", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "minipass": { "version": "2.3.5", "bundled": true, "dev": true, - "optional": true, "requires": { "safe-buffer": "^5.1.2", "yallist": "^3.0.0" @@ -5499,7 +5488,6 @@ "version": "0.5.1", "bundled": true, "dev": true, - "optional": true, "requires": { "minimist": "0.0.8" } @@ -5580,8 +5568,7 @@ "number-is-nan": { "version": "1.0.1", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "object-assign": { "version": "4.1.1", @@ -5593,7 +5580,6 @@ "version": "1.4.0", "bundled": true, "dev": true, - "optional": true, "requires": { "wrappy": "1" } @@ -5679,8 +5665,7 @@ "safe-buffer": { "version": "5.1.2", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "safer-buffer": { "version": "2.1.2", @@ -5716,7 +5701,6 @@ "version": "1.0.2", "bundled": true, "dev": true, - "optional": true, "requires": { "code-point-at": "^1.0.0", "is-fullwidth-code-point": "^1.0.0", @@ -5736,7 +5720,6 @@ "version": "3.0.1", "bundled": true, "dev": true, - "optional": true, "requires": { "ansi-regex": "^2.0.0" } @@ -5780,14 +5763,12 @@ "wrappy": { "version": "1.0.2", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "yallist": { "version": "3.0.3", "bundled": true, - "dev": true, - "optional": true + "dev": true } } }, @@ -6669,7 +6650,7 @@ }, "html-webpack-plugin": { "version": "3.2.0", - "resolved": "https://registry.npmjs.org/html-webpack-plugin/-/html-webpack-plugin-3.2.0.tgz", + "resolved": "http://registry.npmjs.org/html-webpack-plugin/-/html-webpack-plugin-3.2.0.tgz", "integrity": "sha1-sBq71yOsqqeze2r0SS69oD2d03s=", "dev": true, "requires": { @@ -6690,7 +6671,7 @@ }, "json5": { "version": "0.5.1", - "resolved": "https://registry.npmjs.org/json5/-/json5-0.5.1.tgz", + "resolved": "http://registry.npmjs.org/json5/-/json5-0.5.1.tgz", "integrity": "sha1-Hq3nrMASA0rYTiOWdn6tn6VJWCE=", "dev": true }, @@ -7625,7 +7606,7 @@ }, "marked": { "version": "0.3.19", - "resolved": "https://registry.npmjs.org/marked/-/marked-0.3.19.tgz", + "resolved": "http://registry.npmjs.org/marked/-/marked-0.3.19.tgz", "integrity": "sha512-ea2eGWOqNxPcXv8dyERdSr/6FmzvWwzjMxpfGB/sbMccXoct+xY+YukPD+QTUZwyvK7BZwcr4m21WBOW41pAkg==", "dev": true }, @@ -7904,6 +7885,11 @@ "resolved": "https://registry.npmjs.org/lex-parser/-/lex-parser-0.1.4.tgz", "integrity": "sha1-ZMTwJfF/1Tv7RXY/rrFvAVp0dVA=" }, + "libbzip2-wasm": { + "version": "0.0.4", + "resolved": "https://registry.npmjs.org/libbzip2-wasm/-/libbzip2-wasm-0.0.4.tgz", + "integrity": "sha512-RqscTx95+RTKhFAyjedsboR0Lmo3zd8//EuRwQXkdWmsCwYlzarVRaiYg6kS1O8m10MCQkGdrnlK9L4eAmZUwA==" + }, "libyara-wasm": { "version": "0.0.12", "resolved": "https://registry.npmjs.org/libyara-wasm/-/libyara-wasm-0.0.12.tgz", diff --git a/package.json b/package.json index 9af179e3e..929e038d5 100644 --- a/package.json +++ b/package.json @@ -117,6 +117,7 @@ "jsqr": "^1.2.0", "jsrsasign": "8.0.12", "kbpgp": "2.1.0", + "libbzip2-wasm": "0.0.4", "libyara-wasm": "0.0.12", "lodash": "^4.17.11", "loglevel": "^1.6.1", diff --git a/src/core/operations/Bzip2Compress.mjs b/src/core/operations/Bzip2Compress.mjs new file mode 100644 index 000000000..5c029c61e --- /dev/null +++ b/src/core/operations/Bzip2Compress.mjs @@ -0,0 +1,70 @@ +/** + * @author Matt C [me@mitt.dev] + * @copyright Crown Copyright 2019 + * @license Apache-2.0 + */ + +import Operation from "../Operation"; +import OperationError from "../errors/OperationError"; +import Bzip2 from "libbzip2-wasm"; + +/** + * Bzip2 Compress operation + */ +class Bzip2Compress extends Operation { + + /** + * Bzip2Compress constructor + */ + constructor() { + super(); + + this.name = "Bzip2 Compress"; + this.module = "Compression"; + this.description = "Bzip2 is a compression library developed by Julian Seward (of GHC fame) that uses the Burrows-Wheeler algorithm. It only supports compressing single files and its compression is slow, however is more effective than Deflate (.gz & .zip)."; + this.infoURL = "https://wikipedia.org/wiki/Bzip2"; + this.inputType = "ArrayBuffer"; + this.outputType = "ArrayBuffer"; + this.args = [ + { + name: "Block size (100s of kb)", + type: "number", + value: 9 + }, + { + name: "Work factor", + type: "number", + value: 30 + } + ]; + } + + /** + * @param {ArrayBuffer} input + * @param {Object[]} args + * @returns {File} + */ + run(input, args) { + const [blockSize, workFactor] = args; + if (input.byteLength <= 0) { + throw new OperationError("Please provide an input."); + } + if (ENVIRONMENT_IS_WORKER()) self.sendStatusMessage("Loading Bzip2..."); + return new Promise((resolve, reject) => { + Bzip2().then(bzip2 => { + if (ENVIRONMENT_IS_WORKER()) self.sendStatusMessage("Compressing data..."); + const inpArray = new Uint8Array(input); + const bzip2cc = bzip2.compressBZ2(inpArray, blockSize, workFactor); + if (bzip2cc.error !== 0) { + reject(new OperationError(bzip2cc.error_msg)); + } else { + const output = bzip2cc.output; + resolve(output.buffer.slice(output.byteOffset, output.byteLength + output.byteOffset)); + } + }); + }); + } + +} + +export default Bzip2Compress; diff --git a/src/core/operations/Bzip2Decompress.mjs b/src/core/operations/Bzip2Decompress.mjs index 3b3574868..08a1f3d7e 100644 --- a/src/core/operations/Bzip2Decompress.mjs +++ b/src/core/operations/Bzip2Decompress.mjs @@ -1,12 +1,12 @@ /** - * @author n1474335 [n1474335@gmail.com] - * @copyright Crown Copyright 2016 + * @author Matt C [me@mitt.dev] + * @copyright Crown Copyright 2019 * @license Apache-2.0 */ import Operation from "../Operation"; -import bzip2 from "../vendor/bzip2"; import OperationError from "../errors/OperationError"; +import Bzip2 from "libbzip2-wasm"; /** * Bzip2 Decompress operation @@ -23,9 +23,15 @@ class Bzip2Decompress extends Operation { this.module = "Compression"; this.description = "Decompresses data using the Bzip2 algorithm."; this.infoURL = "https://wikipedia.org/wiki/Bzip2"; - this.inputType = "byteArray"; - this.outputType = "string"; - this.args = []; + this.inputType = "ArrayBuffer"; + this.outputType = "ArrayBuffer"; + this.args = [ + { + name: "Use low-memory, slower decompression algorithm", + type: "boolean", + value: false + } + ]; this.patterns = [ { "match": "^\\x42\\x5a\\x68", @@ -41,14 +47,24 @@ class Bzip2Decompress extends Operation { * @returns {string} */ run(input, args) { - const compressed = new Uint8Array(input); - - try { - const bzip2Reader = bzip2.array(compressed); - return bzip2.simple(bzip2Reader); - } catch (err) { - throw new OperationError(err); + const [small] = args; + if (input.byteLength <= 0) { + throw new OperationError("Please provide an input."); } + if (ENVIRONMENT_IS_WORKER()) self.sendStatusMessage("Loading Bzip2..."); + return new Promise((resolve, reject) => { + Bzip2().then(bzip2 => { + if (ENVIRONMENT_IS_WORKER()) self.sendStatusMessage("Decompressing data..."); + const inpArray = new Uint8Array(input); + const bzip2cc = bzip2.decompressBZ2(inpArray, small ? 1 : 0); + if (bzip2cc.error !== 0) { + reject(new OperationError(bzip2cc.error_msg)); + } else { + const output = bzip2cc.output; + resolve(output.buffer.slice(output.byteOffset, output.byteLength + output.byteOffset)); + } + }); + }); } } diff --git a/src/core/vendor/bzip2.mjs b/src/core/vendor/bzip2.mjs deleted file mode 100755 index df0573d83..000000000 --- a/src/core/vendor/bzip2.mjs +++ /dev/null @@ -1,265 +0,0 @@ -/** @license -======================================================================== - bzip2.js - a small bzip2 decompression implementation - - Copyright 2011 by antimatter15 (antimatter15@gmail.com) - - Based on micro-bunzip by Rob Landley (rob@landley.net). - - Copyright (c) 2011 by antimatter15 (antimatter15@gmail.com). - - Permission is hereby granted, free of charge, to any person obtaining a - copy of this software and associated documentation files (the "Software"), - to deal in the Software without restriction, including without limitation - the rights to use, copy, modify, merge, publish, distribute, sublicense, - and/or sell copies of the Software, and to permit persons to whom the - Software is furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included - in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, - DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, - TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH - THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -*/ -"use strict"; - -var bzip2 = {}; - -bzip2.array = function(bytes){ - var bit = 0, byte = 0; - var BITMASK = [0, 0x01, 0x03, 0x07, 0x0F, 0x1F, 0x3F, 0x7F, 0xFF ]; - return function(n){ - var result = 0; - while(n > 0){ - var left = 8 - bit; - if(n >= left){ - result <<= left; - result |= (BITMASK[left] & bytes[byte++]); - bit = 0; - n -= left; - }else{ - result <<= n; - result |= ((bytes[byte] & (BITMASK[n] << (8 - n - bit))) >> (8 - n - bit)); - bit += n; - n = 0; - } - } - return result - } -} - -bzip2.simple = function(bits){ - var size = bzip2.header(bits); - var all = '', chunk = ''; - do{ - all += chunk; - chunk = bzip2.decompress(bits, size); - }while(chunk != -1); - return all; -} - -bzip2.header = function(bits){ - if(bits(8*3) != 4348520) throw "No magic number found"; - var i = bits(8) - 48; - if(i < 1 || i > 9) throw "Not a BZIP archive"; - return i; -}; - - -//takes a function for reading the block data (starting with 0x314159265359) -//a block size (0-9) (optional, defaults to 9) -//a length at which to stop decompressing and return the output -bzip2.decompress = function(bits, size, len){ - var MAX_HUFCODE_BITS = 20; - var MAX_SYMBOLS = 258; - var SYMBOL_RUNA = 0; - var SYMBOL_RUNB = 1; - var GROUP_SIZE = 50; - - var bufsize = 100000 * size; - for(var h = '', i = 0; i < 6; i++) h += bits(8).toString(16); - if(h == "177245385090") return -1; //last block - if(h != "314159265359") throw "Not valid bzip data"; - bits(32); //ignore CRC codes - if(bits(1)) throw "Unsupported obsolete version"; - var origPtr = bits(24); - if(origPtr > bufsize) throw "Initial position larger than buffer size"; - var t = bits(16); - var symToByte = new Uint8Array(256), - symTotal = 0; - for (i = 0; i < 16; i++) { - if(t & (1 << (15 - i))) { - var k = bits(16); - for(j = 0; j < 16; j++){ - if(k & (1 << (15 - j))){ - symToByte[symTotal++] = (16 * i) + j; - } - } - } - } - - var groupCount = bits(3); - if(groupCount < 2 || groupCount > 6) throw "Error 1"; - var nSelectors = bits(15); - if(nSelectors == 0) throw "Error"; - var mtfSymbol = []; //TODO: possibly replace JS array with typed arrays - for(var i = 0; i < groupCount; i++) mtfSymbol[i] = i; - var selectors = new Uint8Array(32768); - - for(var i = 0; i < nSelectors; i++){ - for(var j = 0; bits(1); j++) if(j >= groupCount) throw "Error 2"; - var uc = mtfSymbol[j]; - mtfSymbol.splice(j, 1); //this is a probably inefficient MTF transform - mtfSymbol.splice(0, 0, uc); - selectors[i] = uc; - } - - var symCount = symTotal + 2; - var groups = []; - for(var j = 0; j < groupCount; j++){ - var length = new Uint8Array(MAX_SYMBOLS), - temp = new Uint8Array(MAX_HUFCODE_BITS+1); - t = bits(5); //lengths - for(var i = 0; i < symCount; i++){ - while(true){ - if (t < 1 || t > MAX_HUFCODE_BITS) throw "Error 3"; - if(!bits(1)) break; - if(!bits(1)) t++; - else t--; - } - length[i] = t; - } - var minLen, maxLen; - minLen = maxLen = length[0]; - for(var i = 1; i < symCount; i++){ - if(length[i] > maxLen) maxLen = length[i]; - else if(length[i] < minLen) minLen = length[i]; - } - var hufGroup; - hufGroup = groups[j] = {}; - hufGroup.permute = new Uint32Array(MAX_SYMBOLS); - hufGroup.limit = new Uint32Array(MAX_HUFCODE_BITS + 1); - hufGroup.base = new Uint32Array(MAX_HUFCODE_BITS + 1); - hufGroup.minLen = minLen; - hufGroup.maxLen = maxLen; - var base = hufGroup.base.subarray(1); - var limit = hufGroup.limit.subarray(1); - var pp = 0; - for(var i = minLen; i <= maxLen; i++) - for(var t = 0; t < symCount; t++) - if(length[t] == i) hufGroup.permute[pp++] = t; - for(i = minLen; i <= maxLen; i++) temp[i] = limit[i] = 0; - for(i = 0; i < symCount; i++) temp[length[i]]++; - pp = t = 0; - for(i = minLen; i < maxLen; i++) { - pp += temp[i]; - limit[i] = pp - 1; - pp <<= 1; - base[i+1] = pp - (t += temp[i]); - } - limit[maxLen]=pp+temp[maxLen]-1; - base[minLen]=0; - } - var byteCount = new Uint32Array(256); - for(var i = 0; i < 256; i++) mtfSymbol[i] = i; - var runPos, count, symCount, selector; - runPos = count = symCount = selector = 0; - var buf = new Uint32Array(bufsize); - while(true){ - if(!(symCount--)){ - symCount = GROUP_SIZE - 1; - if(selector >= nSelectors) throw "Error 4"; - hufGroup = groups[selectors[selector++]]; - base = hufGroup.base.subarray(1); - limit = hufGroup.limit.subarray(1); - } - i = hufGroup.minLen; - j = bits(i); - while(true){ - if(i > hufGroup.maxLen) throw "Error 5"; - if(j <= limit[i]) break; - i++; - j = (j << 1) | bits(1); - } - j -= base[i]; - if(j < 0 || j >= MAX_SYMBOLS) throw "Error 6"; - var nextSym = hufGroup.permute[j]; - if (nextSym == SYMBOL_RUNA || nextSym == SYMBOL_RUNB) { - if(!runPos){ - runPos = 1; - t = 0; - } - if(nextSym == SYMBOL_RUNA) t += runPos; - else t += 2 * runPos; - runPos <<= 1; - continue; - } - if(runPos){ - runPos = 0; - if(count + t >= bufsize) throw "Error 7"; - uc = symToByte[mtfSymbol[0]]; - byteCount[uc] += t; - while(t--) buf[count++] = uc; - } - if(nextSym > symTotal) break; - if(count >= bufsize) throw "Error 8"; - i = nextSym -1; - uc = mtfSymbol[i]; - mtfSymbol.splice(i, 1); - mtfSymbol.splice(0, 0, uc); - uc = symToByte[uc]; - byteCount[uc]++; - buf[count++] = uc; - } - if(origPtr < 0 || origPtr >= count) throw "Error 9"; - var j = 0; - for(var i = 0; i < 256; i++){ - k = j + byteCount[i]; - byteCount[i] = j; - j = k; - } - for(var i = 0; i < count; i++){ - uc = buf[i] & 0xff; - buf[byteCount[uc]] |= (i << 8); - byteCount[uc]++; - } - var pos = 0, current = 0, run = 0; - if(count) { - pos = buf[origPtr]; - current = (pos & 0xff); - pos >>= 8; - run = -1; - } - count = count; - var output = ''; - var copies, previous, outbyte; - if(!len) len = Infinity; - while(count){ - count--; - previous = current; - pos = buf[pos]; - current = pos & 0xff; - pos >>= 8; - if(run++ == 3){ - copies = current; - outbyte = previous; - current = -1; - }else{ - copies = 1; - outbyte = current; - } - while(copies--){ - output += (String.fromCharCode(outbyte)); - if(!--len) return output; - } - if(current != previous) run = 0; - } - return output; -} - -export default bzip2;