[FIX] Ensure proper handling of multi-byte characters in streams

lib/processors/stringReplacer.js

@@ -14,8 +14,9 @@ const replaceStream = require("replacestream");
  */
 module.exports = function({resources, options}) {
 	return Promise.all(resources.map((resource) => {
-		const stream = resource.getStream()
-			.pipe(replaceStream(options.pattern, options.replacement));
+		let stream = resource.getStream();
+		stream.setEncoding("utf8");
+		stream = stream.pipe(replaceStream(options.pattern, options.replacement));
 
 		resource.setStream(stream);
 		return resource;

test/lib/processors/stringReplacer.js

@@ -0,0 +1,89 @@
+const test = require("ava");
+const {Readable} = require("stream");
+const stringReplacer = require("../../../lib/processors/stringReplacer");
+
+const getStringFromStream = (stream) => {
+	return new Promise((resolve, reject) => {
+		const buffers = [];
+		stream.on("data", (data) => {
+			buffers.push(data);
+		});
+		stream.on("error", (err) => {
+			reject(err);
+		});
+		stream.on("end", () => {
+			const buffer = Buffer.concat(buffers);
+			resolve(buffer.toString());
+		});
+	});
+};
+
+test.serial("Replaces string pattern from resource stream", async (t) => {
+	const input = `foo bar foo`;
+	const expected = `foo foo foo`;
+
+	let output;
+
+	const resource = {
+		getStream: () => {
+			const stream = new Readable();
+			stream.push(Buffer.from(input));
+			stream.push(null);
+			return stream;
+		},
+		setStream: (outputStream) => {
+			output = getStringFromStream(outputStream);
+		}
+	};
+
+	const processedResources = await stringReplacer({
+		resources: [resource],
+		options: {
+			pattern: "bar",
+			replacement: "foo"
+		}
+	});
+
+	t.deepEqual(processedResources, [resource], "Input resource is returned");
+	t.deepEqual(await output, expected, "Correct file content should be set");
+});
+
+// Skip test in Node v8 as unicode handling of streams seems to be broken
+test[
+	process.version.startsWith("v8.") ? "skip" : "serial"
+]("Correctly processes utf8 characters within separate chunks", async (t) => {
+	const utf8string = "Κυ";
+	const expected = utf8string;
+
+	let output;
+
+	const resource = {
+		getStream: () => {
+			const stream = new Readable();
+			const utf8stringAsBuffer = Buffer.from(utf8string, "utf8");
+			// Pushing each byte separately makes content unreadable
+			// if stream encoding is not set to utf8
+			// This might happen when reading large files with utf8 characters
+			stream.push(Buffer.from([utf8stringAsBuffer[0]]));
+			stream.push(Buffer.from([utf8stringAsBuffer[1]]));
+			stream.push(Buffer.from([utf8stringAsBuffer[2]]));
+			stream.push(Buffer.from([utf8stringAsBuffer[3]]));
+			stream.push(null);
+			return stream;
+		},
+		setStream: (outputStream) => {
+			output = getStringFromStream(outputStream);
+		}
+	};
+
+	const processedResources = await stringReplacer({
+		resources: [resource],
+		options: {
+			pattern: "n/a",
+			replacement: "n/a"
+		}
+	});
+
+	t.deepEqual(processedResources, [resource], "Input resource is returned");
+	t.deepEqual(await output, expected, "Correct file content should be set");
+});