Skip to content
This repository has been archived by the owner on Apr 22, 2023. It is now read-only.

Commit

Permalink
zlib: support concatenated gzip files
Browse files Browse the repository at this point in the history
Reviewed-By: Fedor Indutny <fedor@indutny.com>
PR-URL: #6442
  • Loading branch information
eendeego authored and chrisdickinson committed Dec 16, 2014
1 parent 5b9e5bd commit 6f6a979
Show file tree
Hide file tree
Showing 5 changed files with 282 additions and 8 deletions.
9 changes: 8 additions & 1 deletion lib/zlib.js
Original file line number Diff line number Diff line change
Expand Up @@ -580,14 +580,21 @@ Zlib.prototype._processChunk = function(chunk, flushFlag, cb) {
self._buffer = new Buffer(self._chunkSize);
}

if (availOutAfter === 0) {
if (availOutAfter === 0 || availInAfter > 0) {
// Not actually done. Need to reprocess.
// Also, update the availInBefore to the availInAfter value,
// so that if we have to hit it a third (fourth, etc.) time,
// it'll have the correct byte counts.
inOff += (availInBefore - availInAfter);
availInBefore = availInAfter;

if (availOutAfter !== 0) {
// There is still some data available for reading.
// This is usually a concatenated stream, so, reset and restart.
self.reset();
self._offset = 0;
}

if (!async)
return true;

Expand Down
31 changes: 24 additions & 7 deletions src/node_zlib.cc
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,11 @@ enum node_zlib_mode {
UNZIP
};

enum node_zlib_error {
NO_ERROR,
FAILED,
WRITE_PENDING
};

void InitZlib(v8::Handle<v8::Object> target);

Expand Down Expand Up @@ -207,7 +212,7 @@ class ZCtx : public AsyncWrap {
if (!async) {
// sync version
Process(work_req);
if (CheckError(ctx))
if (CheckError(ctx) == NO_ERROR)
AfterSync(ctx, args);
return;
}
Expand Down Expand Up @@ -292,7 +297,7 @@ class ZCtx : public AsyncWrap {
}


static bool CheckError(ZCtx* ctx) {
static node_zlib_error CheckError(ZCtx* ctx) {
// Acceptable error states depend on the type of zlib stream.
switch (ctx->err_) {
case Z_OK:
Expand All @@ -305,14 +310,18 @@ class ZCtx : public AsyncWrap {
ZCtx::Error(ctx, "Missing dictionary");
else
ZCtx::Error(ctx, "Bad dictionary");
return false;
return FAILED;
default:
// something else.
ZCtx::Error(ctx, "Zlib error");
return false;
if (ctx->strm_.total_out == 0) {
ZCtx::Error(ctx, "Zlib error");
return FAILED;
} else {
return WRITE_PENDING;
}
}

return true;
return NO_ERROR;
}


Expand All @@ -326,7 +335,8 @@ class ZCtx : public AsyncWrap {
HandleScope handle_scope(env->isolate());
Context::Scope context_scope(env->context());

if (!CheckError(ctx))
node_zlib_error error = CheckError(ctx);
if (error == FAILED)
return;

Local<Integer> avail_out = Integer::New(env->isolate(),
Expand All @@ -340,6 +350,11 @@ class ZCtx : public AsyncWrap {
Local<Value> args[2] = { avail_in, avail_out };
ctx->MakeCallback(env->callback_string(), ARRAY_SIZE(args), args);

if (error == WRITE_PENDING) {
ZCtx::Error(ctx, "Zlib error");
return;
}

ctx->Unref();
if (ctx->pending_close_)
ctx->Close();
Expand Down Expand Up @@ -557,10 +572,12 @@ class ZCtx : public AsyncWrap {
switch (ctx->mode_) {
case DEFLATE:
case DEFLATERAW:
case GZIP:
ctx->err_ = deflateReset(&ctx->strm_);
break;
case INFLATE:
case INFLATERAW:
case GUNZIP:
ctx->err_ = inflateReset(&ctx->strm_);
break;
default:
Expand Down
83 changes: 83 additions & 0 deletions test/simple/test-zlib-from-multiple-gzip-with-garbage.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
// Copyright Joyent, Inc. and other Node contributors.
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the
// "Software"), to deal in the Software without restriction, including
// without limitation the rights to use, copy, modify, merge, publish,
// distribute, sublicense, and/or sell copies of the Software, and to permit
// persons to whom the Software is furnished to do so, subject to the
// following conditions:
//
// The above copyright notice and this permission notice shall be included
// in all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
// USE OR OTHER DEALINGS IN THE SOFTWARE.

// test unzipping a file that was created by concatenating multiple gzip
// streams.

var common = require('../common');
var assert = require('assert');
var zlib = require('zlib');

var util = require('util');

var gzipBuffer = new Buffer(128);
var gzipOffset = 0;

var stream1 = '123\n';
var stream2 = '456\n';
var stream3 = '789\n';

function gzipAppend(data) {
data.copy(gzipBuffer, gzipOffset);
gzipOffset += data.length;
}

function writeGzipStream(text, cb) {
var gzip = zlib.createGzip();
gzip.on('data', gzipAppend);
gzip.write(text, function() {
gzip.flush(function() {
gzip.end(function() {
cb();
});
});
});
}

function writeGarbageStream(text, cb) {
gzipAppend(new Buffer(text));
cb();
}

writeGzipStream(stream1, function() {
writeGzipStream(stream2, function() {
writeGarbageStream(stream3, function() {
var gunzip = zlib.createGunzip();
var gunzippedData = new Buffer(2 * 1024);
var gunzippedOffset = 0;
gunzip.on('data', function (data) {
data.copy(gunzippedData, gunzippedOffset);
gunzippedOffset += data.length;
});
gunzip.on('error', function() {
assert.equal(gunzippedData.toString('utf8', 0, gunzippedOffset),
stream1 + stream2);
});
gunzip.on('end', function() {
assert.fail('end event not expected');
});

gunzip.write(gzipBuffer.slice(0, gzipOffset), 'binary', function() {
gunzip.end();
});
});
});
});
74 changes: 74 additions & 0 deletions test/simple/test-zlib-from-multiple-gzip.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
// Copyright Joyent, Inc. and other Node contributors.
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the
// "Software"), to deal in the Software without restriction, including
// without limitation the rights to use, copy, modify, merge, publish,
// distribute, sublicense, and/or sell copies of the Software, and to permit
// persons to whom the Software is furnished to do so, subject to the
// following conditions:
//
// The above copyright notice and this permission notice shall be included
// in all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
// USE OR OTHER DEALINGS IN THE SOFTWARE.

// test unzipping a file that was created by concatenating multiple gzip
// streams.

var common = require('../common');
var assert = require('assert');
var zlib = require('zlib');

var util = require('util');

var gzipBuffer = new Buffer(128);
var gzipOffset = 0;

var stream1 = '123\n';
var stream2 = '456\n';
var stream3 = '789\n';

function gzipAppend(data) {
data.copy(gzipBuffer, gzipOffset);
gzipOffset += data.length;
}

function writeGzipStream(text, cb) {
var gzip = zlib.createGzip();
gzip.on('data', gzipAppend);
gzip.write(text, function() {
gzip.flush(function() {
gzip.end(function() {
cb();
});
});
});
}

writeGzipStream(stream1, function() {
writeGzipStream(stream2, function() {
writeGzipStream(stream3, function() {
var gunzip = zlib.createGunzip();
var gunzippedData = new Buffer(2 * 1024);
var gunzippedOffset = 0;
gunzip.on('data', function (data) {
data.copy(gunzippedData, gunzippedOffset);
gunzippedOffset += data.length;
});
gunzip.on('end', function() {
assert.equal(gunzippedData.toString('utf8', 0, gunzippedOffset), stream1 + stream2 + stream3);
});

gunzip.write(gzipBuffer.slice(0, gzipOffset), 'binary', function() {
gunzip.end();
});
});
});
});
93 changes: 93 additions & 0 deletions test/simple/test-zlib-from-multiple-huge-gzip.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
// Copyright Joyent, Inc. and other Node contributors.
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the
// "Software"), to deal in the Software without restriction, including
// without limitation the rights to use, copy, modify, merge, publish,
// distribute, sublicense, and/or sell copies of the Software, and to permit
// persons to whom the Software is furnished to do so, subject to the
// following conditions:
//
// The above copyright notice and this permission notice shall be included
// in all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
// USE OR OTHER DEALINGS IN THE SOFTWARE.

// test unzipping a file that was created by concatenating multiple gzip
// streams.

var common = require('../common');
var assert = require('assert');
var zlib = require('zlib');

var util = require('util');

var HUGE = 64 * 1024;

var originalBuffer = new Buffer(3 * HUGE);
var originalOffset = 0;

var gzipBuffer = new Buffer(3 * HUGE);
var gzipOffset = 0;

function getRandomLetter() {
return (Math.random() * (122 - 97)) + 97;
}

function generateHugeStream() {
var buffer = new Buffer(HUGE);
for (var i = 0; i < HUGE; i++)
buffer.writeUInt8(getRandomLetter(), i);

buffer.copy(originalBuffer, originalOffset);
originalOffset += HUGE;

return buffer;
}

function gzipAppend(data) {
data.copy(gzipBuffer, gzipOffset);
gzipOffset += data.length;
}

function writeGzipStream(text, cb) {
var gzip = zlib.createGzip();
gzip.on('data', gzipAppend);
gzip.write(text, function() {
gzip.flush(function() {
gzip.end(function() {
cb();
});
});
});
}

writeGzipStream(generateHugeStream(), function() {
writeGzipStream(generateHugeStream(), function() {
writeGzipStream(generateHugeStream(), function() {
var gunzip = zlib.createGunzip();
var gunzippedData = new Buffer(3 * HUGE);
var gunzippedOffset = 0;
gunzip.on('data', function (data) {
data.copy(gunzippedData, gunzippedOffset);
gunzippedOffset += data.length;
});
gunzip.on('end', function() {
var gunzippedStr = gunzippedData.toString('utf8', 0, gunzippedOffset);
var originalStr = originalBuffer.toString('utf8', 0, 3 * HUGE);

assert.equal(gunzippedStr, originalStr);
});

gunzip.write(gzipBuffer.slice(0, gzipOffset), 'binary', function() {
gunzip.end();
});
});
});
});

0 comments on commit 6f6a979

Please sign in to comment.