Implement Node API

.eslintrc.json

@@ -106,9 +106,6 @@
 
         "COMPILE_TIME": false,
         "COMPILE_MSG": false,
-        "PKG_VERSION": false,
-        "ENVIRONMENT_IS_WORKER": false,
-        "ENVIRONMENT_IS_NODE": false,
-        "ENVIRONMENT_IS_WEB": false
+        "PKG_VERSION": false
     }
 }

.github/CONTRIBUTING.md

@@ -22,15 +22,15 @@ Before your contributions can be accepted, you must:
 * Line endings: UNIX style (\n)
 
 
-## Design Principals
+## Design Principles
 
 1. If at all possible, all operations and features should be client-side and not rely on connections to an external server. This increases the utility of CyberChef on closed networks and in virtual machines that are not connected to the Internet. Calls to external APIs may be accepted if there is no other option, but not for critical components.
 2. Latency should be kept to a minimum to enhance the user experience. This means that all operation code should sit on the client, rather than being loaded dynamically from a server.
 3. Use Vanilla JS if at all possible to reduce the number of libraries required and relied upon. Frameworks like jQuery, although included, should not be used unless absolutely necessary.
 4. Minimise the use of large libraries, especially for niche operations that won't be used very often - these will be downloaded by everyone using the app, whether they use that operation or not (due to principal 2).
 
 
-With these principals in mind, any changes or additions to CyberChef should keep it:
+With these principles in mind, any changes or additions to CyberChef should keep it:
 
  - Standalone
  - Efficient

.gitignore

@@ -11,5 +11,8 @@ docs/*
 src/core/config/modules/*
 src/core/config/OperationConfig.json
 src/core/operations/index.mjs
+src/node/config/OperationConfig.json
+src/node/index.mjs
+**/*.DS_Store
 tests/browser/output/*
 

.travis.yml

@@ -1,6 +1,7 @@
 language: node_js
 node_js:
   - lts/*
+cache: npm
 addons:
   chrome: stable
 install: npm install
@@ -10,8 +11,9 @@ before_script:
 script:
   - grunt lint
   - grunt test
+  - grunt test-node
   - grunt docs
-  - grunt node
+  - npm run node-prod
   - grunt prod --msg="$COMPILE_MSG"
   - xvfb-run --server-args="-screen 0 1200x800x24" grunt testui
 before_deploy:

Gruntfile.js

@@ -15,6 +15,8 @@ const path = require("path");
  * @license Apache-2.0
  */
 
+const NODE_PROD = process.env.NODE_ENV === "production";
+
 module.exports = function (grunt) {
     grunt.file.defaultEncoding = "utf8";
     grunt.file.preserveBOM = false;
@@ -26,16 +28,20 @@ module.exports = function (grunt) {
 
     grunt.registerTask("node",
         "Compiles CyberChef into a single NodeJS module.",
-        ["clean:node", "clean:config", "exec:generateConfig", "webpack:node", "chmod:build"]);
+        ["clean", "exec:generateConfig", "exec:generateNodeIndex",  "webpack:node", "webpack:nodeRepl",  "chmod:build"]);
 
     grunt.registerTask("test",
         "A task which runs all the operation tests in the tests directory.",
-        ["exec:generateConfig", "exec:opTests"]);
+        ["clean", "exec:generateConfig", "exec:opTests"]);
 
     grunt.registerTask("testui",
         "A task which runs all the UI tests in the tests directory. The prod task must already have been run.",
         ["connect:prod", "exec:browserTests"]);
 
+    grunt.registerTask("test-node",
+        "Run all the node tests in the tests directory",
+        ["clean", "exec:generateConfig", "exec:generateNodeIndex", "exec:nodeTests"]);
+
     grunt.registerTask("docs",
         "Compiles documentation in the /docs directory.",
         ["clean:docs", "jsdoc", "chmod:docs"]);
@@ -80,15 +86,6 @@ module.exports = function (grunt) {
             COMPILE_TIME: JSON.stringify(compileTime),
             COMPILE_MSG: JSON.stringify(grunt.option("compile-msg") || grunt.option("msg") || ""),
             PKG_VERSION: JSON.stringify(pkg.version),
-            ENVIRONMENT_IS_WORKER: function() {
-                return typeof importScripts === "function";
-            },
-            ENVIRONMENT_IS_NODE: function() {
-                return typeof process === "object" && typeof require === "function";
-            },
-            ENVIRONMENT_IS_WEB: function() {
-                return typeof window === "object";
-            }
         },
         moduleEntryPoints = listEntryModules();
 
@@ -113,7 +110,7 @@ module.exports = function (grunt) {
             dev: ["build/dev/*"],
             prod: ["build/prod/*"],
             node: ["build/node/*"],
-            config: ["src/core/config/OperationConfig.json", "src/core/config/modules/*", "src/code/operations/index.mjs"],
+            config: ["src/core/config/OperationConfig.json", "src/core/config/modules/*", "src/code/operations/index.mjs", "src/node/index.mjs", "src/node/config/OperationConfig.json"],
             docs: ["docs/*", "!docs/*.conf.json", "!docs/*.ico", "!docs/*.png"],
             standalone: ["build/prod/CyberChef*.html"]
         },
@@ -201,19 +198,44 @@ module.exports = function (grunt) {
                 };
             },
             node: {
-                mode: "production",
+                mode: NODE_PROD ? "production" : "development",
                 target: "node",
                 entry: "./src/node/index.mjs",
-                externals: [NodeExternals()],
+                externals: [NodeExternals({
+                    whitelist: ["crypto-api/src/crypto-api"]
+                })],
                 output: {
                     filename: "CyberChef.js",
                     path: __dirname + "/build/node",
                     library: "CyberChef",
                     libraryTarget: "commonjs2"
                 },
                 plugins: [
-                    new webpack.DefinePlugin(BUILD_CONSTANTS)
-                ]
+                    new webpack.DefinePlugin(BUILD_CONSTANTS),
+                    new webpack.optimize.LimitChunkCountPlugin({
+                        maxChunks: 1
+                    })
+                ],
+            },
+            nodeRepl: {
+                mode: NODE_PROD ? "production" : "development",
+                target: "node",
+                entry: "./src/node/repl-index.mjs",
+                externals: [NodeExternals({
+                    whitelist: ["crypto-api/src/crypto-api"]
+                })],
+                output: {
+                    filename: "CyberChef-repl.js",
+                    path: __dirname + "/build/node",
+                    library: "CyberChef",
+                    libraryTarget: "commonjs2"
+                },
+                plugins: [
+                    new webpack.DefinePlugin(BUILD_CONSTANTS),
+                    new webpack.optimize.LimitChunkCountPlugin({
+                        maxChunks: 1
+                    })
+                ],
             }
         },
         "webpack-dev-server": {
@@ -249,9 +271,6 @@ module.exports = function (grunt) {
                             "./config/modules/OpModules": "./config/modules/Default"
                         }
                     },
-                    output: {
-                        globalObject: "this",
-                    },
                     plugins: [
                         new webpack.DefinePlugin(BUILD_CONSTANTS),
                         new HtmlWebpackPlugin({
@@ -358,7 +377,7 @@ module.exports = function (grunt) {
         watch: {
             config: {
                 files: ["src/core/operations/**/*", "!src/core/operations/index.mjs"],
-                tasks: ["exec:generateConfig"]
+                tasks: ["exec:generateNodeIndex", "exec:generateConfig"]
             }
         },
         concurrent: {
@@ -390,11 +409,21 @@ module.exports = function (grunt) {
                     "echo '--- Config scripts finished. ---\n'"
                 ].join(";")
             },
+            generateNodeIndex: {
+                command: [
+                    "echo '\n--- Regenerating node index ---'",
+                    "node --experimental-modules src/node/config/scripts/generateNodeIndex.mjs",
+                    "echo '--- Node index generated. ---\n'"
+                ].join(";"),
+            },
             opTests: {
                 command: "node --experimental-modules --no-warnings --no-deprecation tests/operations/index.mjs"
             },
             browserTests: {
                 command: "./node_modules/.bin/nightwatch --env prod"
+            },
+            nodeTests: {
+                command: "node --experimental-modules --no-warnings --no-deprecation tests/node/index.mjs"
             }
         },
     });

babel.config.js

@@ -16,7 +16,7 @@ module.exports = function(api) {
             }]
         ],
         "plugins": [
-            "babel-plugin-syntax-dynamic-import",
+            "dynamic-import-node",
             [
                 "babel-plugin-transform-builtin-extend", {
                     "globals": ["Error"]