WIP: ESM Conversion

.babelrc

@@ -10,5 +10,10 @@
             "modules": false,
             "useBuiltIns": true
         }]
+    ],
+    "plugins": [
+        ["babel-plugin-transform-builtin-extend", {
+            "globals": ["Error"]
+        }]
     ]
 }

.editorconfig

@@ -0,0 +1,14 @@
+# top-most EditorConfig file
+root = true
+
+[*]
+charset = utf-8
+end_of_line = lf
+insert_final_newline = true
+trim_trailing_whitespace = true
+indent_style = space
+indent_size = 4
+
+[{package.json,.travis.yml}]
+indent_style = space
+indent_size = 2

.eslintignore

@@ -1,2 +1 @@
-src/core/lib/**
-src/core/config/MetaConfig.js
+src/core/vendor/**

.eslintrc.json

@@ -1,6 +1,6 @@
 {
     "parserOptions": {
-        "ecmaVersion": 8,
+        "ecmaVersion": 9,
         "ecmaFeatures": {
             "impliedStrict": true
         },
@@ -84,12 +84,12 @@
         "no-whitespace-before-property": "error",
         "operator-linebreak": ["error", "after"],
         "space-in-parens": "error",
-        "no-var": "error"
+        "no-var": "error",
+        "prefer-const": "error"
     },
     "globals": {
         "$": false,
         "jQuery": false,
-        "moment": false,
         "log": false,
 
         "COMPILE_TIME": false,

.gitignore

@@ -6,4 +6,7 @@ docs/*
 !docs/*.conf.json
 !docs/*.ico
 .vscode
-src/core/config/MetaConfig.js
+src/core/config/modules/*
+src/core/config/OperationConfig.json
+src/core/operations/index.mjs
+

.travis.yml

@@ -1,6 +1,6 @@
 language: node_js
 node_js:
-  - "8.4"
+  - node
 install: npm install
 before_script:
   - npm install -g grunt

CHANGELOG.md

@@ -0,0 +1,36 @@
+# Changelog
+All notable changes to this project will be documented in this file.
+
+## [8.0.0] - 2018-08-05
+- Codebase rewritten using [ES modules](https://hacks.mozilla.org/2018/03/es-modules-a-cartoon-deep-dive/) and [classes](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Classes) #284
+- Operation architecture restructured to make adding new operations a lot simpler #284
+- A script has been added to aid in the creation of new operations by running `npm run newop` @n1474335 #284
+- 'Magic' operation added - [automated detection of encoded data](https://github.com/gchq/CyberChef/wiki/Automatic-detection-of-encoded-data-using-CyberChef-Magic) @n1474335 #239
+- UI updated to use [Bootstrap Material Design](https://fezvrasta.github.io/bootstrap-material-design/) @n1474335 #248
+- `JSON`, `File` and `List<File>` Dish types added @n1474335 #284
+- `OperationError` type added for better handling of errors thrown by operations @d98762625 #296
+- A `present()` method has been added, allowing operations to pass machine-friendly data to subsequent operations whilst presenting human-friendly data to the user @n1474335 #284
+- Set operations added @d98762625 #281
+- 'To Table' operation added @JustAnotherMark #294
+- 'Haversine distance' operation added @Dachande663 #325
+- Started keeping a changelog @n1474335
+
+## [7.0.0] - 2017-12-28
+- Added support for loading, processing and downloading files up to 500MB @n1474335 #224
+
+## [6.0.0] - 2017-09-19
+- Added threading support, moving all recipe processing into a [Web Worker](https://developer.mozilla.org/en-US/docs/Web/API/Web_Workers_API/Using_web_workers) to increase performance and allow long-running operations to be cancelled @n1474335 #173
+- Created modules so that operations relying on large libraries can be downloaded separately as required, reducing the initial loading time for the app @n1474335 #173
+
+## [5.0.0] - 2017-03-30
+-  Configured Webpack build process, Babel transpilation and ES6 imports and exports @n1474335 #95
+
+## [4.0.0] - 2016-11-28
+-  Initial open source commit @n1474335
+
+
+[8.0.0]: https://github.com/gchq/CyberChef/releases/tag/v8.0.0
+[7.0.0]: https://github.com/gchq/CyberChef/releases/tag/v7.0.0
+[6.0.0]: https://github.com/gchq/CyberChef/releases/tag/v6.0.0
+[5.0.0]: https://github.com/gchq/CyberChef/releases/tag/v5.0.0
+[4.0.0]: https://github.com/gchq/CyberChef/commit/b1d73a725dc7ab9fb7eb789296efd2b7e4b08306

Gruntfile.js

@@ -4,7 +4,8 @@ const webpack = require("webpack");
 const HtmlWebpackPlugin = require("html-webpack-plugin");
 const NodeExternals = require("webpack-node-externals");
 const Inliner = require("web-resource-inliner");
-const fs = require("fs");
+const glob = require("glob");
+const path = require("path");
 
 /**
  * Grunt configuration for building the app in various formats.
@@ -21,31 +22,31 @@ module.exports = function (grunt) {
     // Tasks
     grunt.registerTask("dev",
         "A persistent task which creates a development build whenever source files are modified.",
-        ["clean:dev", "concurrent:dev"]);
+        ["clean:dev", "exec:generateConfig", "concurrent:dev"]);
 
     grunt.registerTask("node",
         "Compiles CyberChef into a single NodeJS module.",
-        ["clean:node", "webpack:metaConf", "webpack:node", "chmod:build"]);
+        ["clean:node", "clean:config", "exec:generateConfig", "webpack:node", "chmod:build"]);
 
     grunt.registerTask("test",
         "A task which runs all the tests in test/tests.",
-        ["clean:test", "webpack:metaConf", "webpack:tests", "execute:test"]);
+        ["exec:generateConfig", "exec:tests"]);
 
     grunt.registerTask("docs",
         "Compiles documentation in the /docs directory.",
         ["clean:docs", "jsdoc", "chmod:docs"]);
 
     grunt.registerTask("prod",
         "Creates a production-ready build. Use the --msg flag to add a compile message.",
-        ["eslint", "clean:prod", "webpack:metaConf", "webpack:web", "inline", "chmod"]);
+        ["eslint", "clean:prod", "exec:generateConfig", "webpack:web", "inline", "chmod"]);
 
     grunt.registerTask("default",
         "Lints the code base",
         ["eslint", "exec:repoSize"]);
 
     grunt.registerTask("inline",
         "Compiles a production build of CyberChef into a single, portable web page.",
-        ["webpack:webInline", "runInliner", "clean:inlineScripts"]);
+        ["exec:generateConfig", "webpack:webInline", "runInliner", "clean:inlineScripts"]);
 
 
     grunt.registerTask("runInliner", runInliner);
@@ -60,9 +61,9 @@ module.exports = function (grunt) {
     grunt.loadNpmTasks("grunt-jsdoc");
     grunt.loadNpmTasks("grunt-contrib-clean");
     grunt.loadNpmTasks("grunt-contrib-copy");
+    grunt.loadNpmTasks("grunt-contrib-watch");
     grunt.loadNpmTasks("grunt-chmod");
     grunt.loadNpmTasks("grunt-exec");
-    grunt.loadNpmTasks("grunt-execute");
     grunt.loadNpmTasks("grunt-accessibility");
     grunt.loadNpmTasks("grunt-concurrent");
 
@@ -118,12 +119,12 @@ module.exports = function (grunt) {
      * Generates an entry list for all the modules.
      */
     function listEntryModules() {
-        const path = "./src/core/config/modules/";
-        let entryModules = {};
+        const entryModules = {};
 
-        fs.readdirSync(path).forEach(file => {
-            if (file !== "Default.js" && file !== "OpModules.js")
-                entryModules[file.split(".js")[0]] = path + file;
+        glob.sync("./src/core/config/modules/*.mjs").forEach(file => {
+            const basename = path.basename(file);
+            if (basename !== "Default.mjs" && basename !== "OpModules.mjs")
+                entryModules[basename.split(".mjs")[0]] = path.resolve(file);
         });
 
         return entryModules;
@@ -132,9 +133,9 @@ module.exports = function (grunt) {
     grunt.initConfig({
         clean: {
             dev: ["build/dev/*"],
-            prod: ["build/prod/*", "src/core/config/MetaConfig.js"],
-            test: ["build/test/*", "src/core/config/MetaConfig.js"],
-            node: ["build/node/*", "src/core/config/MetaConfig.js"],
+            prod: ["build/prod/*"],
+            node: ["build/node/*"],
+            config: ["src/core/config/OperationConfig.json", "src/core/config/modules/*", "src/code/operations/index.mjs"],
             docs: ["docs/*", "!docs/*.conf.json", "!docs/*.ico", "!docs/*.png"],
             inlineScripts: ["build/prod/scripts.js"],
         },
@@ -143,10 +144,10 @@ module.exports = function (grunt) {
                 configFile: "./.eslintrc.json"
             },
             configs: ["Gruntfile.js"],
-            core: ["src/core/**/*.js", "!src/core/lib/**/*", "!src/core/config/MetaConfig.js"],
-            web: ["src/web/**/*.js"],
-            node: ["src/node/**/*.js"],
-            tests: ["test/**/*.js"],
+            core: ["src/core/**/*.{js,mjs}", "!src/core/vendor/**/*", "!src/core/operations/legacy/**/*"],
+            web: ["src/web/**/*.{js,mjs}"],
+            node: ["src/node/**/*.{js,mjs}"],
+            tests: ["test/**/*.{js,mjs}"],
         },
         jsdoc: {
             options: {
@@ -159,17 +160,11 @@ module.exports = function (grunt) {
             all: {
                 src: [
                     "src/**/*.js",
-                    "!src/core/lib/**/*",
-                    "!src/core/config/MetaConfig.js"
+                    "src/**/*.mjs",
+                    "!src/core/vendor/**/*"
                 ],
             }
         },
-        concurrent: {
-            options: {
-                logConcurrentOutput: true
-            },
-            dev: ["webpack:metaConfDev", "webpack-dev-server:start"]
-        },
         accessibility: {
             options: {
                 accessibilityLevel: "WCAG2A",
@@ -184,39 +179,6 @@ module.exports = function (grunt) {
         },
         webpack: {
             options: webpackConfig,
-            metaConf: {
-                mode: "production",
-                target: "node",
-                entry: [
-                    "babel-polyfill",
-                    "./src/core/config/OperationConfig.js"
-                ],
-                output: {
-                    filename: "MetaConfig.js",
-                    path: __dirname + "/src/core/config/",
-                    library: "MetaConfig",
-                    libraryTarget: "commonjs2",
-                    libraryExport: "default"
-                },
-                externals: [NodeExternals()],
-            },
-            metaConfDev: {
-                mode: "development",
-                target: "node",
-                entry: [
-                    "babel-polyfill",
-                    "./src/core/config/OperationConfig.js"
-                ],
-                output: {
-                    filename: "MetaConfig.js",
-                    path: __dirname + "/src/core/config/",
-                    library: "MetaConfig",
-                    libraryTarget: "commonjs2",
-                    libraryExport: "default"
-                },
-                externals: [NodeExternals()],
-                watch: true
-            },
             web: {
                 mode: "production",
                 target: "web",
@@ -229,7 +191,7 @@ module.exports = function (grunt) {
                 },
                 resolve: {
                     alias: {
-                        "./config/modules/OpModules.js": "./config/modules/Default.js"
+                        "./config/modules/OpModules": "./config/modules/Default"
                     }
                 },
                 plugins: [
@@ -279,7 +241,7 @@ module.exports = function (grunt) {
             tests: {
                 mode: "development",
                 target: "node",
-                entry: "./test/index.js",
+                entry: "./test/index.mjs",
                 externals: [NodeExternals()],
                 output: {
                     filename: "index.js",
@@ -292,7 +254,7 @@ module.exports = function (grunt) {
             node: {
                 mode: "production",
                 target: "node",
-                entry: "./src/node/index.js",
+                entry: "./src/node/index.mjs",
                 externals: [NodeExternals()],
                 output: {
                     filename: "CyberChef.js",
@@ -330,7 +292,7 @@ module.exports = function (grunt) {
                     }, moduleEntryPoints),
                     resolve: {
                         alias: {
-                            "./config/modules/OpModules.js": "./config/modules/Default.js"
+                            "./config/modules/OpModules": "./config/modules/Default"
                         }
                     },
                     plugins: [
@@ -388,6 +350,18 @@ module.exports = function (grunt) {
                 src: ["docs/**/*", "docs/"]
             }
         },
+        watch: {
+            config: {
+                files: ["src/core/operations/**/*", "!src/core/operations/index.mjs"],
+                tasks: ["exec:generateConfig"]
+            }
+        },
+        concurrent: {
+            dev: ["watch:config", "webpack-dev-server:start"],
+            options: {
+                logConcurrentOutput: true
+            }
+        },
         exec: {
             repoSize: {
                 command: [
@@ -401,10 +375,21 @@ module.exports = function (grunt) {
             },
             sitemap: {
                 command: "node build/prod/sitemap.js > build/prod/sitemap.xml"
+            },
+            generateConfig: {
+                command: [
+                    "echo '\n--- Regenerating config files. ---'",
+                    "mkdir -p src/core/config/modules",
+                    "echo 'export default {};\n' > src/core/config/modules/OpModules.mjs",
+                    "echo '[]\n' > src/core/config/OperationConfig.json",
+                    "node --experimental-modules src/core/config/scripts/generateOpsIndex.mjs",
+                    "node --experimental-modules src/core/config/scripts/generateConfig.mjs",
+                    "echo '--- Config scripts finished. ---\n'"
+                ].join(";")
+            },
+            tests: {
+                command: "node --experimental-modules test/index.mjs"
             }
         },
-        execute: {
-            test: "build/test/index.js"
-        },
     });
 };

README.md

@@ -12,7 +12,7 @@
 
 CyberChef is a simple, intuitive web app for carrying out all manner of "cyber" operations within a web browser. These operations include simple encoding like XOR or Base64, more complex encryption like AES, DES and Blowfish, creating binary and hexdumps, compression and decompression of data, calculating hashes and checksums, IPv6 and X.509 parsing, changing character encodings, and much more.
 
-The tool is designed to enable both technical and non-technical analysts to manipulate data in complex ways without having to deal with complex tools or algorithms. It was conceived, designed, built and incrementally improved by an analyst in their 10% innovation time over several years. Every effort has been made to structure the code in a readable and extendable format, however it should be noted that the analyst is not a professional developer.
+The tool is designed to enable both technical and non-technical analysts to manipulate data in complex ways without having to deal with complex tools or algorithms. It was conceived, designed, built and incrementally improved by an analyst in their 10% innovation time over several years.
 
 ## Live demo
 
@@ -43,16 +43,19 @@ You can use as many operations as you like in simple or complex ways. Some examp
  - [Carry out different operations on data of different types][8]
  - [Use parts of the input as arguments to operations][9]
  - [Perform AES decryption, extracting the IV from the beginning of the cipher stream][10]
+ - [Automagically detect several layers of nested encoding][12]
 
 
 ## Features
 
  - Drag and drop
      - Operations can be dragged in and out of the recipe list, or reorganised.
-     - Files can be dragged over the input box to load them directly into the browser.
+     - Files up to 500MB can be dragged over the input box to load them directly into the browser.
  - Auto Bake
      - Whenever you modify the input or the recipe, CyberChef will automatically "bake" for you and produce the output immediately.
      - This can be turned off and operated manually if it is affecting performance (if the input is very large, for instance).
+ - Automated encoding detection
+     - CyberChef uses [a number of techniques](https://github.com/gchq/CyberChef/wiki/Automatic-detection-of-encoded-data-using-CyberChef-Magic) to attempt to automatically detect which encodings your data is under. If it finds a suitable operation which can make sense of your data, it displays the 'magic' icon in the Output field which you can click to decode your data.
  - Breakpoints
      - You can set breakpoints on any operation in your recipe to pause execution before running it.
      - You can also step through the recipe one operation at a time to see what the data looks like at each stage.
@@ -81,6 +84,8 @@ CyberChef is built to support
 
 ## Contributing
 
+Contributing a new operation to CyberChef is super easy! There is a quickstart script which will walk you through the process. If you can write basic JavaScript, you can write a CyberChef operation.
+
 An installation walkthrough, how-to guides for adding new operations and themes, descriptions of the repository structure, available data types and coding conventions can all be found in the project [wiki pages](https://github.com/gchq/CyberChef/wiki).
 
  - Sign the [GCHQ Contributor Licence Agreement](https://github.com/gchq/Gaffer/wiki/GCHQ-OSS-Contributor-License-Agreement-V1.0)
@@ -104,3 +109,4 @@ CyberChef is released under the [Apache 2.0 Licence](https://www.apache.org/lice
   [9]: https://gchq.github.io/CyberChef/#recipe=Register('key%3D(%5B%5C%5Cda-f%5D*)',true,false)Find_/_Replace(%7B'option':'Regex','string':'.*data%3D(.*)'%7D,'$1',true,false,true)RC4(%7B'option':'Hex','string':'$R0'%7D,'Hex','Latin1')&input=aHR0cDovL21hbHdhcmV6LmJpei9iZWFjb24ucGhwP2tleT0wZTkzMmE1YyZkYXRhPThkYjdkNWViZTM4NjYzYTU0ZWNiYjMzNGUzZGIxMQ
   [10]: https://gchq.github.io/CyberChef/#recipe=Register('(.%7B32%7D)',true,false)Drop_bytes(0,32,false)AES_Decrypt(%7B'option':'Hex','string':'1748e7179bd56570d51fa4ba287cc3e5'%7D,%7B'option':'Hex','string':'$R0'%7D,'CTR','Hex','Raw',%7B'option':'Hex','string':''%7D)&input=NTFlMjAxZDQ2MzY5OGVmNWY3MTdmNzFmNWI0NzEyYWYyMGJlNjc0YjNiZmY1M2QzODU0NjM5NmVlNjFkYWFjNDkwOGUzMTljYTNmY2Y3MDg5YmZiNmIzOGVhOTllNzgxZDI2ZTU3N2JhOWRkNmYzMTFhMzk0MjBiODk3OGU5MzAxNGIwNDJkNDQ3MjZjYWVkZjU0MzZlYWY2NTI0MjljMGRmOTRiNTIxNjc2YzdjMmNlODEyMDk3YzI3NzI3M2M3YzcyY2Q4OWFlYzhkOWZiNGEyNzU4NmNjZjZhYTBhZWUyMjRjMzRiYTNiZmRmN2FlYjFkZGQ0Nzc2MjJiOTFlNzJjOWU3MDlhYjYwZjhkYWY3MzFlYzBjYzg1Y2UwZjc0NmZmMTU1NGE1YTNlYzI5MWNhNDBmOWU2MjlhODcyNTkyZDk4OGZkZDgzNDUzNGFiYTc5YzFhZDE2NzY3NjlhN2MwMTBiZjA0NzM5ZWNkYjY1ZDk1MzAyMzcxZDYyOWQ5ZTM3ZTdiNGEzNjFkYTQ2OGYxZWQ1MzU4OTIyZDJlYTc1MmRkMTFjMzY2ZjMwMTdiMTRhYTAxMWQyYWYwM2M0NGY5NTU3OTA5OGExNWUzY2Y5YjQ0ODZmOGZmZTljMjM5ZjM0ZGU3MTUxZjZjYTY1MDBmZTRiODUwYzNmMWMwMmU4MDFjYWYzYTI0NDY0NjE0ZTQyODAxNjE1YjhmZmFhMDdhYzgyNTE0OTNmZmRhN2RlNWRkZjMzNjg4ODBjMmI5NWIwMzBmNDFmOGYxNTA2NmFkZDA3MWE2NmNmNjBlNWY0NmYzYTIzMGQzOTdiNjUyOTYzYTIxYTUzZg
   [11]: https://gchq.github.io/CyberChef/#recipe=XOR(%7B'option':'Hex','string':'3a'%7D,'Standard',false)To_Hexdump(16,false,false)&input=VGhlIGFuc3dlciB0byB0aGUgdWx0aW1hdGUgcXVlc3Rpb24gb2YgbGlmZSwgdGhlIFVuaXZlcnNlLCBhbmQgZXZlcnl0aGluZyBpcyA0Mi4
+  [12]: https://gchq.github.io/CyberChef/#recipe=Magic(3,false,false)&input=V1VhZ3dzaWFlNm1QOGdOdENDTFVGcENwQ0IyNlJtQkRvREQ4UGFjZEFtekF6QlZqa0syUXN0RlhhS2hwQzZpVVM3UkhxWHJKdEZpc29SU2dvSjR3aGptMWFybTg2NHFhTnE0UmNmVW1MSHJjc0FhWmM1VFhDWWlmTmRnUzgzZ0RlZWpHWDQ2Z2FpTXl1QlY2RXNrSHQxc2NnSjg4eDJ0TlNvdFFEd2JHWTFtbUNvYjJBUkdGdkNLWU5xaU45aXBNcTFaVTFtZ2tkYk51R2NiNzZhUnRZV2hDR1VjOGc5M1VKdWRoYjhodHNoZVpud1RwZ3FoeDgzU1ZKU1pYTVhVakpUMnptcEM3dVhXdHVtcW9rYmRTaTg4WXRrV0RBYzFUb291aDJvSDRENGRkbU5LSldVRHBNd21uZ1VtSzE0eHdtb21jY1BRRTloTTE3MkFQblNxd3hkS1ExNzJSa2NBc3lzbm1qNWdHdFJtVk5OaDJzMzU5d3I2bVMyUVJQ