Skip to content

Commit

Permalink
fix: base64 and unicode characters
Browse files Browse the repository at this point in the history
  • Loading branch information
alexander-akait authored Nov 4, 2021
1 parent a30b4b7 commit 02b1f3f
Show file tree
Hide file tree
Showing 7 changed files with 240 additions and 31 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/nodejs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ jobs:
run: yarn

- name: Run tests
run: yarn test
run: yarn test:only

- name: Submit coverage data to codecov
uses: codecov/codecov-action@v2
Expand Down
38 changes: 27 additions & 11 deletions lib/getHashDigest.js
Original file line number Diff line number Diff line change
Expand Up @@ -45,37 +45,54 @@ function encodeBufferToBase(buffer, base) {
let crypto = undefined;
let createXXHash64 = undefined;
let createMd4 = undefined;
let BatchedHash = undefined;
let BulkUpdateDecorator = undefined;

function getHashDigest(buffer, hashType, digestType, maxLength) {
hashType = hashType || "xxhash64";
function getHashDigest(buffer, algorithm, digestType, maxLength) {
algorithm = algorithm || "xxhash64";
maxLength = maxLength || 9999;

let hash;

if (hashType === "xxhash64") {
if (algorithm === "xxhash64") {
if (createXXHash64 === undefined) {
createXXHash64 = require("./hash/xxhash64");

if (BatchedHash === undefined) {
BatchedHash = require("./hash/BatchedHash");
}
}

hash = createXXHash64();
} else if (hashType === "md4") {
hash = new BatchedHash(createXXHash64());
} else if (algorithm === "md4") {
if (createMd4 === undefined) {
createMd4 = require("./hash/md4");
}

hash = createMd4();
} else if (hashType === "native-md4") {
hash = new BatchedHash(createMd4());
} else if (algorithm === "native-md4") {
if (typeof crypto === "undefined") {
crypto = require("crypto");

if (BulkUpdateDecorator === undefined) {
BulkUpdateDecorator = require("./hash/BulkUpdateDecorator");
}
}

hash = crypto.createHash("md4");
hash = new BulkUpdateDecorator(() => crypto.createHash("md4"), "md4");
} else {
if (typeof crypto === "undefined") {
crypto = require("crypto");

if (BulkUpdateDecorator === undefined) {
BulkUpdateDecorator = require("./hash/BulkUpdateDecorator");
}
}

hash = crypto.createHash(hashType);
hash = new BulkUpdateDecorator(
() => crypto.createHash(algorithm),
algorithm
);
}

hash.update(buffer);
Expand All @@ -87,8 +104,7 @@ function getHashDigest(buffer, hashType, digestType, maxLength) {
digestType === "base49" ||
digestType === "base52" ||
digestType === "base58" ||
digestType === "base62" ||
digestType === "base64"
digestType === "base62"
) {
return encodeBufferToBase(hash.digest(), digestType.substr(4)).substr(
0,
Expand Down
64 changes: 64 additions & 0 deletions lib/hash/BatchedHash.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
const MAX_SHORT_STRING = require("./wasm-hash").MAX_SHORT_STRING;

class BatchedHash {
constructor(hash) {
this.string = undefined;
this.encoding = undefined;
this.hash = hash;
}

/**
* Update hash {@link https://nodejs.org/api/crypto.html#crypto_hash_update_data_inputencoding}
* @param {string|Buffer} data data
* @param {string=} inputEncoding data encoding
* @returns {this} updated hash
*/
update(data, inputEncoding) {
if (this.string !== undefined) {
if (
typeof data === "string" &&
inputEncoding === this.encoding &&
this.string.length + data.length < MAX_SHORT_STRING
) {
this.string += data;

return this;
}

this.hash.update(this.string, this.encoding);
this.string = undefined;
}

if (typeof data === "string") {
if (
data.length < MAX_SHORT_STRING &&
// base64 encoding is not valid since it may contain padding chars
(!inputEncoding || !inputEncoding.startsWith("ba"))
) {
this.string = data;
this.encoding = inputEncoding;
} else {
this.hash.update(data, inputEncoding);
}
} else {
this.hash.update(data);
}

return this;
}

/**
* Calculates the digest {@link https://nodejs.org/api/crypto.html#crypto_hash_digest_encoding}
* @param {string=} encoding encoding of the return value
* @returns {string|Buffer} digest
*/
digest(encoding) {
if (this.string !== undefined) {
this.hash.update(this.string, this.encoding);
}

return this.hash.digest(encoding);
}
}

module.exports = BatchedHash;
107 changes: 107 additions & 0 deletions lib/hash/BulkUpdateDecorator.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
const BULK_SIZE = 2000;

// We are using an object instead of a Map as this will stay static during the runtime
// so access to it can be optimized by v8
const digestCaches = {};

class BulkUpdateDecorator {
/**
* @param {Hash | function(): Hash} hashOrFactory function to create a hash
* @param {string=} hashKey key for caching
*/
constructor(hashOrFactory, hashKey) {
this.hashKey = hashKey;

if (typeof hashOrFactory === "function") {
this.hashFactory = hashOrFactory;
this.hash = undefined;
} else {
this.hashFactory = undefined;
this.hash = hashOrFactory;
}

this.buffer = "";
}

/**
* Update hash {@link https://nodejs.org/api/crypto.html#crypto_hash_update_data_inputencoding}
* @param {string|Buffer} data data
* @param {string=} inputEncoding data encoding
* @returns {this} updated hash
*/
update(data, inputEncoding) {
if (
inputEncoding !== undefined ||
typeof data !== "string" ||
data.length > BULK_SIZE
) {
if (this.hash === undefined) {
this.hash = this.hashFactory();
}

if (this.buffer.length > 0) {
this.hash.update(this.buffer);
this.buffer = "";
}

this.hash.update(data, inputEncoding);
} else {
this.buffer += data;

if (this.buffer.length > BULK_SIZE) {
if (this.hash === undefined) {
this.hash = this.hashFactory();
}

this.hash.update(this.buffer);
this.buffer = "";
}
}

return this;
}

/**
* Calculates the digest {@link https://nodejs.org/api/crypto.html#crypto_hash_digest_encoding}
* @param {string=} encoding encoding of the return value
* @returns {string|Buffer} digest
*/
digest(encoding) {
let digestCache;

const buffer = this.buffer;

if (this.hash === undefined) {
// short data for hash, we can use caching
const cacheKey = `${this.hashKey}-${encoding}`;

digestCache = digestCaches[cacheKey];

if (digestCache === undefined) {
digestCache = digestCaches[cacheKey] = new Map();
}

const cacheEntry = digestCache.get(buffer);

if (cacheEntry !== undefined) {
return cacheEntry;
}

this.hash = this.hashFactory();
}

if (buffer.length > 0) {
this.hash.update(buffer);
}

const digestResult = this.hash.digest(encoding);

if (digestCache !== undefined) {
digestCache.set(buffer, digestResult);
}

return digestResult;
}
}

module.exports = BulkUpdateDecorator;
5 changes: 3 additions & 2 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,11 @@
"big.js": "^6.1.1"
},
"scripts": {
"lint": "prettier --list-different . && eslint lib test",
"lint": "prettier --list-different . && eslint .",
"pretest": "yarn lint",
"test": "jest",
"test:ci": "jest --coverage",
"test:only": "jest --coverage",
"test:ci": "yarn test:only",
"release": "yarn test && standard-version"
},
"license": "MIT",
Expand Down
43 changes: 32 additions & 11 deletions test/getHashDigest.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -4,36 +4,57 @@ const loaderUtils = require("../");

describe("getHashDigest()", () => {
[
["test string", "xxhash64", "hex", undefined, "e9e2c351e3c6b198"],
["test string", "xxhash64", "base64", undefined, "6eLDUePGsZg="],
["test string", "xxhash64", "base52", undefined, "byfYGDmnmyUr"],
["abc\\0♥", "xxhash64", "hex", undefined, "4b9a34297dc03d20"],
["abc\\0💩", "xxhash64", "hex", undefined, "86733ec125b93904"],
["abc\\0💩", "xxhash64", "base64", undefined, "hnM+wSW5OQQ="],
["abc\\0♥", "xxhash64", "base64", undefined, "S5o0KX3APSA="],
["abc\\0💩", "xxhash64", "base52", undefined, "cfByjQcJZIU"],
["abc\\0♥", "xxhash64", "base52", undefined, "qdLyAQjLlod"],

["test string", "md4", "hex", 4, "2e06"],
["test string", "md4", "base64", undefined, "Lgbt1PFiMmjFpRcw2KCyrw=="],
["test string", "md4", "base52", undefined, "egWqIKxsDHdZTteemJqXfuo"],
["abc\\0♥", "md4", "hex", undefined, "46b9627fecf49b80eaf01c01d86ae9fd"],
["abc\\0💩", "md4", "hex", undefined, "45aa5b332f8e562aaf0106ad6fc1d78f"],
["abc\\0💩", "md4", "base64", undefined, "RapbMy+OViqvAQatb8HXjw=="],
["abc\\0♥", "md4", "base64", undefined, "Rrlif+z0m4Dq8BwB2Grp/Q=="],
["abc\\0💩", "md4", "base52", undefined, "dtXZENFEkYHXGxOkJbevPoD"],
["abc\\0♥", "md4", "base52", undefined, "fYFFcfXRGsVweukHKlPayHs"],

["test string", "md5", "hex", 4, "6f8d"],
[
"test string",
"md5",
"hex",
undefined,
"6f8db599de986fab7a21625b7916589c",
],
["test string", "md5", "base64", undefined, "2sm1pVmS8xuGJLCdWpJoRL"],
// ["test string", "md5", "base64url", undefined, "b421md6Yb6t6IWJbeRZYnA"],
["test string", "xxhash64", "hex", undefined, "e9e2c351e3c6b198"],
["test string", "xxhash64", "base64", undefined, "9yNNKdhM-bF"],
["test string", "xxhash64", "base52", undefined, "byfYGDmnmyUr"],
// ["test string", "xxhash64", "base64url", undefined, "6eLDUePGsZg"],
["test string", "md4", "hex", 4, "2e06"],
["test string", "md5", "hex", 4, "6f8d"],
["test string", "md5", "base52", undefined, "dJnldHSAutqUacjgfBQGLQx"],
["test string", "md5", "base64", undefined, "b421md6Yb6t6IWJbeRZYnA=="],
["test string", "md5", "base26", 6, "bhtsgu"],
["abc\\0♥", "md5", "hex", undefined, "2e897b64f8050e66aff98d38f7a012c5"],
["abc\\0💩", "md5", "hex", undefined, "63ad5b3d675c5890e0c01ed339ba0187"],
["abc\\0💩", "md5", "base64", undefined, "Y61bPWdcWJDgwB7TOboBhw=="],
["abc\\0♥", "md5", "base64", undefined, "Lol7ZPgFDmav+Y0496ASxQ=="],
["abc\\0💩", "md5", "base52", undefined, "djhVWGHaUKUxqxEhcTnOfBx"],
["abc\\0♥", "md5", "base52", undefined, "eHeasSeRyOnorzxUJpayzJc"],

[
"test string",
"sha512",
"base64",
undefined,
"2IS-kbfIPnVflXb9CzgoNESGCkvkb0urMmucPD9z8q6HuYz8RShY1-tzSUpm5-Ivx_u4H1MEzPgAhyhaZ7RKog",
"EObWR69EYkRC84jCwUp4f/ixfmFluD12fsBHdo2MvLcaGjIm58x4Frx5wEJ9lKnaaIxBo5kse/Xk18w+C+XbrA==",
],
[
"test string",
"md5",
"sha512",
"hex",
undefined,
"6f8db599de986fab7a21625b7916589c",
"10e6d647af44624442f388c2c14a787ff8b17e6165b83d767ec047768d8cbcb71a1a3226e7cc7816bc79c0427d94a9da688c41a3992c7bf5e4d7cc3e0be5dbac",
],
].forEach((test) => {
it(
Expand Down
12 changes: 6 additions & 6 deletions test/interpolateName.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -50,13 +50,13 @@ describe("interpolateName()", () => {
"/app/img/image.png",
"[sha512:hash:base64:7].[ext]",
"test content",
"2BKDTjl.png",
"DL9MrvO.png",
],
[
"/app/img/image.png",
"[sha512:contenthash:base64:7].[ext]",
"test content",
"2BKDTjl.png",
"DL9MrvO.png",
],
[
"/app/dir/file.png",
Expand Down Expand Up @@ -104,19 +104,19 @@ describe("interpolateName()", () => {
"/lib/components/modal/modal.css",
"[name].[md4:hash:base64:20].[ext]",
"test content",
"modal.1kNSGJ6n9ibMUEckC1Cp.css",
"modal.ppiZgUkxKA4vUnIZrWrH.css",
],
[
"/lib/components/modal/modal.css",
"[name].[md5:hash:base64:20].[ext]",
"test content",
"modal.1n8osQznuT8jOAwdzg_n.css",
"modal.lHP90NiApDwht3eNNIch.css",
],
[
"/lib/components/modal/modal.css",
"[name].[md5:contenthash:base64:20].[ext]",
"test content",
"modal.1n8osQznuT8jOAwdzg_n.css",
"modal.lHP90NiApDwht3eNNIch.css",
],
// Should not interpret without `hash` or `contenthash`
[
Expand Down Expand Up @@ -259,7 +259,7 @@ describe("interpolateName()", () => {
],
[
[{}, "[hash:base64]", { content: "test string" }],
"9yNNKdhM-bF",
"6eLDUePGsZg=",
"should interpolate [hash] token with options",
],
[
Expand Down

0 comments on commit 02b1f3f

Please sign in to comment.