diff --git a/deps/zlib/BUILD.gn b/deps/zlib/BUILD.gn index 7fff5762e81b6a..f97ab45de2741f 100644 --- a/deps/zlib/BUILD.gn +++ b/deps/zlib/BUILD.gn @@ -441,6 +441,36 @@ executable("zlib_bench") { configs += [ "//build/config/compiler:no_chromium_code" ] } +executable("minigzip") { + include_dirs = [ "." ] + + sources = [ "test/minigzip.c" ] + if (!is_debug) { + configs -= [ "//build/config/compiler:default_optimization" ] + configs += [ "//build/config/compiler:optimize_speed" ] + } + + deps = [ ":zlib" ] + + configs -= [ "//build/config/compiler:chromium_code" ] + configs += [ "//build/config/compiler:no_chromium_code" ] +} + +executable("zpipe") { + include_dirs = [ "." ] + + sources = [ "examples/zpipe.c" ] + if (!is_debug) { + configs -= [ "//build/config/compiler:default_optimization" ] + configs += [ "//build/config/compiler:optimize_speed" ] + } + + deps = [ ":zlib" ] + + configs -= [ "//build/config/compiler:chromium_code" ] + configs += [ "//build/config/compiler:no_chromium_code" ] +} + if (!is_win || target_os != "winuwp") { executable("minizip_bin") { include_dirs = [ "." ] diff --git a/deps/zlib/CMakeLists.txt b/deps/zlib/CMakeLists.txt index c3f424770d92ce..66f7d04966afa5 100644 --- a/deps/zlib/CMakeLists.txt +++ b/deps/zlib/CMakeLists.txt @@ -26,6 +26,8 @@ option(ENABLE_SIMD_AVX512 "Enable SIMD AXV512 optimizations" OFF) option(USE_ZLIB_RABIN_KARP_HASH "Enable bitstream compatibility with canonical zlib" OFF) option(BUILD_UNITTESTS "Enable standalone unit tests build" OFF) option(BUILD_MINIZIP_BIN "Enable building minzip_bin tool" OFF) +option(BUILD_ZPIPE "Enable building zpipe tool" OFF) +option(BUILD_MINIGZIP "Enable building minigzip tool" OFF) if (USE_ZLIB_RABIN_KARP_HASH) add_definitions(-DUSE_ZLIB_RABIN_KARP_ROLLING_HASH) @@ -79,9 +81,16 @@ if (ENABLE_SIMD_OPTIMIZATIONS) add_definitions(-DRISCV_RVV) add_definitions(-DDEFLATE_SLIDE_HASH_RVV) add_definitions(-DADLER32_SIMD_RVV) - #TODO(cavalcantii): add remaining flags as we port optimizations to RVV. - # Required by CPU features detection code. - SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} --target=riscv64-unknown-linux-gnu -march=rv64gcv") + + # TODO(cavalcantii): add remaining flags as we port optimizations to RVV. + # chunk_copy is required for READ64 and unconditional decode of literals. + add_definitions(-DINFLATE_CHUNK_GENERIC) + add_definitions(-DINFLATE_CHUNK_READ_64LE) + + # Tested with clang-17, unaligned loads are required by read64 & chunk_copy. + # TODO(cavalcantii): replace internal clang flags for -munaligned-access + # when we have a newer compiler available. + SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} --target=riscv64-unknown-linux-gnu -march=rv64gcv -Xclang -target-feature -Xclang +unaligned-scalar-mem") endif() endif() @@ -192,9 +201,14 @@ set(ZLIB_SRCS if (ENABLE_SIMD_OPTIMIZATIONS) if (CMAKE_SYSTEM_PROCESSOR STREQUAL "riscv64") message("RISCVV: Add optimizations.") + list(REMOVE_ITEM ZLIB_SRCS inflate.c) list(APPEND ZLIB_PRIVATE_HDRS ${CMAKE_CURRENT_SOURCE_DIR}/adler32_simd.h) + list(APPEND ZLIB_PRIVATE_HDRS ${CMAKE_CURRENT_SOURCE_DIR}/contrib/optimizations/chunkcopy.h) list(APPEND ZLIB_PRIVATE_HDRS ${CMAKE_CURRENT_SOURCE_DIR}/cpu_features.h) + list(APPEND ZLIB_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/adler32_simd.c) + list(APPEND ZLIB_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/contrib/optimizations/inffast_chunk.c) + list(APPEND ZLIB_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/contrib/optimizations/inflate.c) list(APPEND ZLIB_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/cpu_features.c) else() list(REMOVE_ITEM ZLIB_SRCS inflate.c) @@ -339,7 +353,7 @@ if (BUILD_UNITTESTS) endif() #============================================================================ -# Minigzip tool +# Minizip tool #============================================================================ # TODO(cavalcantii): get it working on Windows. if (BUILD_MINIZIP_BIN) @@ -349,3 +363,18 @@ if (BUILD_MINIZIP_BIN) ) target_link_libraries(minizip_bin zlib) endif() + +#============================================================================ +# zpipe tool +#============================================================================ +if (BUILD_ZPIPE) + add_executable(zpipe examples/zpipe.c) + target_link_libraries(zpipe zlib) +endif() +#============================================================================ +# MiniGzip tool +#============================================================================ +if (BUILD_MINIGZIP) + add_executable(minigzip_bin test/minigzip.c) + target_link_libraries(minigzip_bin zlib) +endif() diff --git a/deps/zlib/adler32_simd.c b/deps/zlib/adler32_simd.c index 9970ea9ca71857..b3e1f0a3dfda01 100644 --- a/deps/zlib/adler32_simd.c +++ b/deps/zlib/adler32_simd.c @@ -41,9 +41,6 @@ * [2] zlib adler32_z() uses this fact to implement NMAX-block-based updates * of the adler s1 s2 of uint32_t type (see adler32.c). */ -/* Copyright (C) 2023 SiFive, Inc. All rights reserved. - * For conditions of distribution and use, see copyright notice in zlib.h - */ #include "adler32_simd.h" @@ -368,11 +365,10 @@ uint32_t ZLIB_INTERNAL adler32_simd_( /* NEON */ #elif defined(ADLER32_SIMD_RVV) #include -/* adler32_rvv.c - RVV version of Adler-32 - * RVV 1.0 code contributed by Alex Chiang - * on https://github.com/zlib-ng/zlib-ng/pull/1532 - * Port from Simon Hosie's fork: - * https://github.com/cloudflare/zlib/commit/40688b53c61cb9bfc36471acd2dc0800b7ebcab1 + +/* + * Patch by Simon Hosie, from: + * https://github.com/cloudflare/zlib/pull/55 */ uint32_t ZLIB_INTERNAL adler32_simd_( /* RVV */ @@ -380,91 +376,81 @@ uint32_t ZLIB_INTERNAL adler32_simd_( /* RVV */ const unsigned char *buf, unsigned long len) { - /* split Adler-32 into component sums */ - uint32_t sum2 = (adler >> 16) & 0xffff; - adler &= 0xffff; - - size_t left = len; - size_t vl = __riscv_vsetvlmax_e8m1(); - vl = vl > 256 ? 256 : vl; - vuint32m4_t v_buf32_accu = __riscv_vmv_v_x_u32m4(0, vl); - vuint32m4_t v_adler32_prev_accu = __riscv_vmv_v_x_u32m4(0, vl); - vuint16m2_t v_buf16_accu; - - /* - * We accumulate 8-bit data, and to prevent overflow, we have to use a 32-bit accumulator. - * However, adding 8-bit data into a 32-bit accumulator isn't efficient. We use 16-bit & 32-bit - * accumulators to boost performance. - * - * The block_size is the largest multiple of vl that <= 256, because overflow would occur when - * vl > 256 (255 * 256 <= UINT16_MAX). - * - * We accumulate 8-bit data into a 16-bit accumulator and then - * move the data into the 32-bit accumulator at the last iteration. + size_t vl = __riscv_vsetvlmax_e8m2(); + const vuint16m4_t zero16 = __riscv_vmv_v_x_u16m4(0, vl); + vuint16m4_t a_sum = zero16; + vuint32m8_t b_sum = __riscv_vmv_v_x_u32m8(0, vl); + + /* Deal with the part which is not a multiple of vl first; because it's + * easier to zero-stuff the beginning of the checksum than it is to tweak the + * multipliers and sums for odd lengths afterwards. + */ + size_t head = len & (vl - 1); + if (head > 0) { + vuint8m2_t zero8 = __riscv_vmv_v_x_u8m2(0, vl); + vuint8m2_t in = __riscv_vle8_v_u8m2(buf, vl); + in = __riscv_vslideup(zero8, in, vl - head, vl); + vuint16m4_t in16 = __riscv_vwcvtu_x(in, vl); + a_sum = in16; + buf += head; + } + + /* We have a 32-bit accumulator, and in each iteration we add 22-times a + * 16-bit value, plus another 16-bit value. We periodically subtract up to + * 65535 times BASE to avoid overflow. b_overflow estimates how often we + * need to do this subtraction. + */ + const int b_overflow = BASE / 23; + int fixup = b_overflow; + ssize_t iters = (len - head) / vl; + while (iters > 0) { + const vuint16m4_t a_overflow = __riscv_vrsub(a_sum, BASE, vl); + int batch = iters < 22 ? iters : 22; + iters -= batch; + b_sum = __riscv_vwmaccu(b_sum, batch, a_sum, vl); + vuint16m4_t a_batch = zero16, b_batch = zero16; + + /* Do a short batch, where neither a_sum nor b_sum can overflow a 16-bit + * register. Then add them back into the main accumulators. */ - size_t block_size = (256 / vl) * vl; - size_t nmax_limit = (NMAX / block_size); - size_t cnt = 0; - while (left >= block_size) { - v_buf16_accu = __riscv_vmv_v_x_u16m2(0, vl); - size_t subprob = block_size; - while (subprob > 0) { - vuint8m1_t v_buf8 = __riscv_vle8_v_u8m1(buf, vl); - v_adler32_prev_accu = __riscv_vwaddu_wv_u32m4(v_adler32_prev_accu, v_buf16_accu, vl); - v_buf16_accu = __riscv_vwaddu_wv_u16m2(v_buf16_accu, v_buf8, vl); - buf += vl; - subprob -= vl; - } - v_adler32_prev_accu = __riscv_vmacc_vx_u32m4(v_adler32_prev_accu, block_size / vl, v_buf32_accu, vl); - v_buf32_accu = __riscv_vwaddu_wv_u32m4(v_buf32_accu, v_buf16_accu, vl); - left -= block_size; - /* do modulo once each block of NMAX size */ - if (++cnt >= nmax_limit) { - v_adler32_prev_accu = __riscv_vremu_vx_u32m4(v_adler32_prev_accu, BASE, vl); - cnt = 0; - } + while (batch-- > 0) { + vuint8m2_t in8 = __riscv_vle8_v_u8m2(buf, vl); + buf += vl; + b_batch = __riscv_vadd(b_batch, a_batch, vl); + a_batch = __riscv_vwaddu_wv(a_batch, in8, vl); } - /* the left len <= 256 now, we can use 16-bit accum safely */ - v_buf16_accu = __riscv_vmv_v_x_u16m2(0, vl); - size_t res = left; - while (left >= vl) { - vuint8m1_t v_buf8 = __riscv_vle8_v_u8m1(buf, vl); - v_adler32_prev_accu = __riscv_vwaddu_wv_u32m4(v_adler32_prev_accu, v_buf16_accu, vl); - v_buf16_accu = __riscv_vwaddu_wv_u16m2(v_buf16_accu, v_buf8, vl); - buf += vl; - left -= vl; + vbool4_t ov = __riscv_vmsgeu(a_batch, a_overflow, vl); + a_sum = __riscv_vadd(a_sum, a_batch, vl); + a_sum = __riscv_vadd_mu(ov, a_sum, a_sum, 65536 - BASE, vl); + b_sum = __riscv_vwaddu_wv(b_sum, b_batch, vl); + if (--fixup <= 0) { + b_sum = __riscv_vnmsac(b_sum, BASE, __riscv_vsrl(b_sum, 16, vl), vl); + fixup = b_overflow; } - v_adler32_prev_accu = __riscv_vmacc_vx_u32m4(v_adler32_prev_accu, res / vl, v_buf32_accu, vl); - v_adler32_prev_accu = __riscv_vremu_vx_u32m4(v_adler32_prev_accu, BASE, vl); - v_buf32_accu = __riscv_vwaddu_wv_u32m4(v_buf32_accu, v_buf16_accu, vl); - - vuint32m4_t v_seq = __riscv_vid_v_u32m4(vl); - vuint32m4_t v_rev_seq = __riscv_vrsub_vx_u32m4(v_seq, vl, vl); - vuint32m4_t v_sum32_accu = __riscv_vmul_vv_u32m4(v_buf32_accu, v_rev_seq, vl); - - v_sum32_accu = __riscv_vadd_vv_u32m4(v_sum32_accu, __riscv_vmul_vx_u32m4(v_adler32_prev_accu, vl, vl), vl); - - vuint32m1_t v_sum2_sum = __riscv_vmv_s_x_u32m1(0, vl); - v_sum2_sum = __riscv_vredsum_vs_u32m4_u32m1(v_sum32_accu, v_sum2_sum, vl); - uint32_t sum2_sum = __riscv_vmv_x_s_u32m1_u32(v_sum2_sum); - - sum2 += (sum2_sum + adler * (len - left)); - - vuint32m1_t v_adler_sum = __riscv_vmv_s_x_u32m1(0, vl); - v_adler_sum = __riscv_vredsum_vs_u32m4_u32m1(v_buf32_accu, v_adler_sum, vl); - uint32_t adler_sum = __riscv_vmv_x_s_u32m1_u32(v_adler_sum); - - adler += adler_sum; - - while (left--) { - adler += *buf++; - sum2 += adler; - } - - sum2 %= BASE; - adler %= BASE; - - return adler | (sum2 << 16); + } + /* Adjust per-lane sums to have appropriate offsets from the end of the + * buffer. + */ + const vuint16m4_t off = __riscv_vrsub(__riscv_vid_v_u16m4(vl), vl, vl); + vuint16m4_t bsum16 = __riscv_vncvt_x(__riscv_vremu(b_sum, BASE, vl), vl); + b_sum = __riscv_vadd(__riscv_vwmulu(a_sum, off, vl), + __riscv_vwmulu(bsum16, vl, vl), vl); + bsum16 = __riscv_vncvt_x(__riscv_vremu(b_sum, BASE, vl), vl); + + /* And finally, do a horizontal sum across the registers for the final + * result. + */ + uint32_t a = adler & 0xffff; + uint32_t b = ((adler >> 16) + a * (len % BASE)) % BASE; + vuint32m1_t sca = __riscv_vmv_v_x_u32m1(a, 1); + vuint32m1_t scb = __riscv_vmv_v_x_u32m1(b, 1); + sca = __riscv_vwredsumu(a_sum, sca, vl); + scb = __riscv_vwredsumu(bsum16, scb, vl); + a = __riscv_vmv_x(sca); + b = __riscv_vmv_x(scb); + a %= BASE; + b %= BASE; + return (b << 16) | a; } #endif /* ADLER32_SIMD_SSSE3 */ diff --git a/deps/zlib/contrib/optimizations/chunkcopy.h b/deps/zlib/contrib/optimizations/chunkcopy.h index f40546d54dbe77..97efff3a42496d 100644 --- a/deps/zlib/contrib/optimizations/chunkcopy.h +++ b/deps/zlib/contrib/optimizations/chunkcopy.h @@ -21,8 +21,10 @@ #if defined(__clang__) || defined(__GNUC__) || defined(__llvm__) #define Z_BUILTIN_MEMCPY __builtin_memcpy +#define Z_BUILTIN_MEMSET __builtin_memset #else #define Z_BUILTIN_MEMCPY zmemcpy +#define Z_BUILTIN_MEMSET zmemset #endif #if defined(INFLATE_CHUNK_SIMD_NEON) @@ -31,6 +33,8 @@ typedef uint8x16_t z_vec128i_t; #elif defined(INFLATE_CHUNK_SIMD_SSE2) #include typedef __m128i z_vec128i_t; +#elif defined(INFLATE_CHUNK_GENERIC) +typedef struct { uint8_t x[16]; } z_vec128i_t; #else #error chunkcopy.h inflate chunk SIMD is not defined for your build target #endif @@ -265,6 +269,77 @@ static inline z_vec128i_t v_load8_dup(const void* src) { static inline void v_store_128(void* out, const z_vec128i_t vec) { _mm_storeu_si128((__m128i*)out, vec); } +#elif defined(INFLATE_CHUNK_GENERIC) +/* + * Default implementations for chunk-copy functions rely on memcpy() being + * inlined by the compiler for best performance. This is most likely to work + * as expected when the length argument is constant (as is the case here) and + * the target supports unaligned loads and stores. Since that's not always a + * safe assumption, this may need extra compiler arguments such as + * `-mno-strict-align` or `-munaligned-access`, or the availability of + * extensions like SIMD. + */ + +/* + * v_load64_dup(): load *src as an unaligned 64-bit int and duplicate it in + * every 64-bit component of the 128-bit result (64-bit int splat). + */ +static inline z_vec128i_t v_load64_dup(const void* src) { + int64_t in; + Z_BUILTIN_MEMCPY(&in, src, sizeof(in)); + z_vec128i_t out; + for (int i = 0; i < sizeof(out); i += sizeof(in)) { + Z_BUILTIN_MEMCPY((uint8_t*)&out + i, &in, sizeof(in)); + } + return out; +} + +/* + * v_load32_dup(): load *src as an unaligned 32-bit int and duplicate it in + * every 32-bit component of the 128-bit result (32-bit int splat). + */ +static inline z_vec128i_t v_load32_dup(const void* src) { + int32_t in; + Z_BUILTIN_MEMCPY(&in, src, sizeof(in)); + z_vec128i_t out; + for (int i = 0; i < sizeof(out); i += sizeof(in)) { + Z_BUILTIN_MEMCPY((uint8_t*)&out + i, &in, sizeof(in)); + } + return out; +} + +/* + * v_load16_dup(): load *src as an unaligned 16-bit int and duplicate it in + * every 16-bit component of the 128-bit result (16-bit int splat). + */ +static inline z_vec128i_t v_load16_dup(const void* src) { + int16_t in; + Z_BUILTIN_MEMCPY(&in, src, sizeof(in)); + z_vec128i_t out; + for (int i = 0; i < sizeof(out); i += sizeof(in)) { + Z_BUILTIN_MEMCPY((uint8_t*)&out + i, &in, sizeof(in)); + } + return out; +} + +/* + * v_load8_dup(): load the 8-bit int *src and duplicate it in every 8-bit + * component of the 128-bit result (8-bit int splat). + */ +static inline z_vec128i_t v_load8_dup(const void* src) { + int8_t in = *(const uint8_t*)src; + z_vec128i_t out; + Z_BUILTIN_MEMSET(&out, in, sizeof(out)); + return out; +} + +/* + * v_store_128(): store the 128-bit vec in a memory destination (that might + * not be 16-byte aligned) void* out. + */ +static inline void v_store_128(void* out, const z_vec128i_t vec) { + Z_BUILTIN_MEMCPY(out, &vec, sizeof(vec)); +} #endif /* diff --git a/deps/zlib/contrib/tests/utils_unittest.cc b/deps/zlib/contrib/tests/utils_unittest.cc index 0cc10813775f3e..f487a06996c98b 100644 --- a/deps/zlib/contrib/tests/utils_unittest.cc +++ b/deps/zlib/contrib/tests/utils_unittest.cc @@ -20,7 +20,8 @@ #include "zlib.h" -void TestPayloads(size_t input_size, zlib_internal::WrapperType type) { +void TestPayloads(size_t input_size, zlib_internal::WrapperType type, + const int compression_level = Z_DEFAULT_COMPRESSION) { std::vector input; input.reserve(input_size); for (size_t i = 1; i <= input_size; ++i) @@ -36,7 +37,7 @@ void TestPayloads(size_t input_size, zlib_internal::WrapperType type) { unsigned long compressed_size = static_cast(compressed.size()); int result = zlib_internal::CompressHelper( type, compressed.data(), &compressed_size, input.data(), input.size(), - Z_DEFAULT_COMPRESSION, nullptr, nullptr); + compression_level, nullptr, nullptr); ASSERT_EQ(result, Z_OK); unsigned long decompressed_size = @@ -67,6 +68,25 @@ TEST(ZlibTest, RawWrapper) { TestPayloads(i, zlib_internal::WrapperType::ZRAW); } +TEST(ZlibTest, LargePayloads) { + static const size_t lengths[] = { 6000, 8000, 10'000, 15'000, 20'000, 30'000, + 50'000, 100'000, 150'000, 2'500'000, + 5'000'000, 10'000'000, 20'000'000 }; + + for (size_t length: lengths) { + TestPayloads(length, zlib_internal::WrapperType::ZLIB); + TestPayloads(length, zlib_internal::WrapperType::GZIP); + } +} + +TEST(ZlibTest, CompressionLevels) { + static const int levels[] = { 1, 2, 3, 4, 5, 6, 7, 8, 9 }; + for (int level: levels) { + TestPayloads(5'000'000, zlib_internal::WrapperType::ZLIB, level); + TestPayloads(5'000'000, zlib_internal::WrapperType::GZIP, level); + } +} + TEST(ZlibTest, InflateCover) { cover_support(); cover_wrap(); diff --git a/deps/zlib/examples/zpipe.c b/deps/zlib/examples/zpipe.c new file mode 100644 index 00000000000000..51dec4745772e5 --- /dev/null +++ b/deps/zlib/examples/zpipe.c @@ -0,0 +1,209 @@ +/* zpipe.c: example of proper use of zlib's inflate() and deflate() + Not copyrighted -- provided to the public domain + Version 1.4 11 December 2005 Mark Adler */ + +/* Version history: + 1.0 30 Oct 2004 First version + 1.1 8 Nov 2004 Add void casting for unused return values + Use switch statement for inflate() return values + 1.2 9 Nov 2004 Add assertions to document zlib guarantees + 1.3 6 Apr 2005 Remove incorrect assertion in inf() + 1.4 11 Dec 2005 Add hack to avoid MSDOS end-of-line conversions + Avoid some compiler warnings for input and output buffers + */ + +#if defined(_WIN32) && !defined(_CRT_NONSTDC_NO_DEPRECATE) +# define _CRT_NONSTDC_NO_DEPRECATE +#endif + +#include +#include +#include +#include "zlib.h" + +#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(__CYGWIN__) +# include +# include +# define SET_BINARY_MODE(file) setmode(fileno(file), O_BINARY) +#else +# define SET_BINARY_MODE(file) +#endif + +#define CHUNK 16384 + +/* Compress from file source to file dest until EOF on source. + def() returns Z_OK on success, Z_MEM_ERROR if memory could not be + allocated for processing, Z_STREAM_ERROR if an invalid compression + level is supplied, Z_VERSION_ERROR if the version of zlib.h and the + version of the library linked do not match, or Z_ERRNO if there is + an error reading or writing the files. */ +int def(FILE *source, FILE *dest, int level) +{ + int ret, flush; + unsigned have; + z_stream strm; + unsigned char in[CHUNK]; + unsigned char out[CHUNK]; + + /* allocate deflate state */ + strm.zalloc = Z_NULL; + strm.zfree = Z_NULL; + strm.opaque = Z_NULL; + ret = deflateInit(&strm, level); + if (ret != Z_OK) + return ret; + + /* compress until end of file */ + do { + strm.avail_in = fread(in, 1, CHUNK, source); + if (ferror(source)) { + (void)deflateEnd(&strm); + return Z_ERRNO; + } + flush = feof(source) ? Z_FINISH : Z_NO_FLUSH; + strm.next_in = in; + + /* run deflate() on input until output buffer not full, finish + compression if all of source has been read in */ + do { + strm.avail_out = CHUNK; + strm.next_out = out; + ret = deflate(&strm, flush); /* no bad return value */ + assert(ret != Z_STREAM_ERROR); /* state not clobbered */ + have = CHUNK - strm.avail_out; + if (fwrite(out, 1, have, dest) != have || ferror(dest)) { + (void)deflateEnd(&strm); + return Z_ERRNO; + } + } while (strm.avail_out == 0); + assert(strm.avail_in == 0); /* all input will be used */ + + /* done when last data in file processed */ + } while (flush != Z_FINISH); + assert(ret == Z_STREAM_END); /* stream will be complete */ + + /* clean up and return */ + (void)deflateEnd(&strm); + return Z_OK; +} + +/* Decompress from file source to file dest until stream ends or EOF. + inf() returns Z_OK on success, Z_MEM_ERROR if memory could not be + allocated for processing, Z_DATA_ERROR if the deflate data is + invalid or incomplete, Z_VERSION_ERROR if the version of zlib.h and + the version of the library linked do not match, or Z_ERRNO if there + is an error reading or writing the files. */ +int inf(FILE *source, FILE *dest) +{ + int ret; + unsigned have; + z_stream strm; + unsigned char in[CHUNK]; + unsigned char out[CHUNK]; + + /* allocate inflate state */ + strm.zalloc = Z_NULL; + strm.zfree = Z_NULL; + strm.opaque = Z_NULL; + strm.avail_in = 0; + strm.next_in = Z_NULL; + ret = inflateInit(&strm); + if (ret != Z_OK) + return ret; + + /* decompress until deflate stream ends or end of file */ + do { + strm.avail_in = fread(in, 1, CHUNK, source); + if (ferror(source)) { + (void)inflateEnd(&strm); + return Z_ERRNO; + } + if (strm.avail_in == 0) + break; + strm.next_in = in; + + /* run inflate() on input until output buffer not full */ + do { + strm.avail_out = CHUNK; + strm.next_out = out; + ret = inflate(&strm, Z_NO_FLUSH); + assert(ret != Z_STREAM_ERROR); /* state not clobbered */ + switch (ret) { + case Z_NEED_DICT: + ret = Z_DATA_ERROR; /* and fall through */ + case Z_DATA_ERROR: + case Z_MEM_ERROR: + (void)inflateEnd(&strm); + return ret; + } + have = CHUNK - strm.avail_out; + if (fwrite(out, 1, have, dest) != have || ferror(dest)) { + (void)inflateEnd(&strm); + return Z_ERRNO; + } + } while (strm.avail_out == 0); + + /* done when inflate() says it's done */ + } while (ret != Z_STREAM_END); + + /* clean up and return */ + (void)inflateEnd(&strm); + return ret == Z_STREAM_END ? Z_OK : Z_DATA_ERROR; +} + +/* report a zlib or i/o error */ +void zerr(int ret) +{ + fputs("zpipe: ", stderr); + switch (ret) { + case Z_ERRNO: + if (ferror(stdin)) + fputs("error reading stdin\n", stderr); + if (ferror(stdout)) + fputs("error writing stdout\n", stderr); + break; + case Z_STREAM_ERROR: + fputs("invalid compression level\n", stderr); + break; + case Z_DATA_ERROR: + fputs("invalid or incomplete deflate data\n", stderr); + break; + case Z_MEM_ERROR: + fputs("out of memory\n", stderr); + break; + case Z_VERSION_ERROR: + fputs("zlib version mismatch!\n", stderr); + } +} + +/* compress or decompress from stdin to stdout */ +int main(int argc, char **argv) +{ + int ret; + + /* avoid end-of-line conversions */ + SET_BINARY_MODE(stdin); + SET_BINARY_MODE(stdout); + + /* do compression if no arguments */ + if (argc == 1) { + ret = def(stdin, stdout, Z_DEFAULT_COMPRESSION); + if (ret != Z_OK) + zerr(ret); + return ret; + } + + /* do decompression if -d specified */ + else if (argc == 2 && strcmp(argv[1], "-d") == 0) { + ret = inf(stdin, stdout); + if (ret != Z_OK) + zerr(ret); + return ret; + } + + /* otherwise, report usage */ + else { + fputs("zpipe usage: zpipe [-d] < source > dest\n", stderr); + return 1; + } +} diff --git a/deps/zlib/google/compression_utils.cc b/deps/zlib/google/compression_utils.cc index c2b17e4ced6f22..0ba31101489fde 100644 --- a/deps/zlib/google/compression_utils.cc +++ b/deps/zlib/google/compression_utils.cc @@ -6,7 +6,6 @@ #include "base/check_op.h" #include "base/process/memory.h" -#include "base/sys_byteorder.h" #include "third_party/zlib/google/compression_utils_portable.h" diff --git a/deps/zlib/google/zip_reader_unittest.cc b/deps/zlib/google/zip_reader_unittest.cc index e6f89d7e4faaaa..9eb7d7d2b10e05 100644 --- a/deps/zlib/google/zip_reader_unittest.cc +++ b/deps/zlib/google/zip_reader_unittest.cc @@ -72,7 +72,7 @@ class FileWrapper { // A mock that provides methods that can be used as callbacks in asynchronous // unzip functions. Tracks the number of calls and number of bytes reported. // Assumes that progress callbacks will be executed in-order. -class MockUnzipListener : public base::SupportsWeakPtr { +class MockUnzipListener final { public: MockUnzipListener() : success_calls_(0), @@ -98,12 +98,18 @@ class MockUnzipListener : public base::SupportsWeakPtr { int progress_calls() { return progress_calls_; } int current_progress() { return current_progress_; } + base::WeakPtr AsWeakPtr() { + return weak_ptr_factory_.GetWeakPtr(); + } + private: int success_calls_; int failure_calls_; int progress_calls_; int64_t current_progress_; + + base::WeakPtrFactory weak_ptr_factory_{this}; }; class MockWriterDelegate : public zip::WriterDelegate { diff --git a/deps/zlib/test/minigzip.c b/deps/zlib/test/minigzip.c new file mode 100644 index 00000000000000..c72356dbccf255 --- /dev/null +++ b/deps/zlib/test/minigzip.c @@ -0,0 +1,579 @@ +/* minigzip.c -- simulate gzip using the zlib compression library + * Copyright (C) 1995-2006, 2010, 2011, 2016 Jean-loup Gailly + * For conditions of distribution and use, see copyright notice in zlib.h + */ + +/* + * minigzip is a minimal implementation of the gzip utility. This is + * only an example of using zlib and isn't meant to replace the + * full-featured gzip. No attempt is made to deal with file systems + * limiting names to 14 or 8+3 characters, etc... Error checking is + * very limited. So use minigzip only for testing; use gzip for the + * real thing. On MSDOS, use only on file names without extension + * or in pipe mode. + */ + +/* @(#) $Id$ */ + +#include "zlib.h" +#include + +#ifdef STDC +# include +# include +#endif + +#ifdef USE_MMAP +# include +# include +# include +#endif + +#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(__CYGWIN__) +# include +# include +# ifdef UNDER_CE +# include +# endif +# define SET_BINARY_MODE(file) setmode(fileno(file), O_BINARY) +#else +# define SET_BINARY_MODE(file) +#endif + +#if defined(_MSC_VER) && _MSC_VER < 1900 +# define snprintf _snprintf +#endif + +#ifdef VMS +# define unlink delete +# define GZ_SUFFIX "-gz" +#endif +#ifdef RISCOS +# define unlink remove +# define GZ_SUFFIX "-gz" +# define fileno(file) file->__file +#endif +#if defined(__MWERKS__) && __dest_os != __be_os && __dest_os != __win32_os +# include /* for fileno */ +#endif + +#if !defined(Z_HAVE_UNISTD_H) && !defined(_LARGEFILE64_SOURCE) +#ifndef WIN32 /* unlink already in stdio.h for WIN32 */ + extern int unlink(const char *); +#endif +#endif + +#if defined(UNDER_CE) +# include +# define perror(s) pwinerror(s) + +/* Map the Windows error number in ERROR to a locale-dependent error + message string and return a pointer to it. Typically, the values + for ERROR come from GetLastError. + + The string pointed to shall not be modified by the application, + but may be overwritten by a subsequent call to strwinerror + + The strwinerror function does not change the current setting + of GetLastError. */ + +static char *strwinerror (error) + DWORD error; +{ + static char buf[1024]; + + wchar_t *msgbuf; + DWORD lasterr = GetLastError(); + DWORD chars = FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM + | FORMAT_MESSAGE_ALLOCATE_BUFFER, + NULL, + error, + 0, /* Default language */ + (LPVOID)&msgbuf, + 0, + NULL); + if (chars != 0) { + /* If there is an \r\n appended, zap it. */ + if (chars >= 2 + && msgbuf[chars - 2] == '\r' && msgbuf[chars - 1] == '\n') { + chars -= 2; + msgbuf[chars] = 0; + } + + if (chars > sizeof (buf) - 1) { + chars = sizeof (buf) - 1; + msgbuf[chars] = 0; + } + + wcstombs(buf, msgbuf, chars + 1); + LocalFree(msgbuf); + } + else { + sprintf(buf, "unknown win32 error (%ld)", error); + } + + SetLastError(lasterr); + return buf; +} + +static void pwinerror (s) + const char *s; +{ + if (s && *s) + fprintf(stderr, "%s: %s\n", s, strwinerror(GetLastError ())); + else + fprintf(stderr, "%s\n", strwinerror(GetLastError ())); +} + +#endif /* UNDER_CE */ + +#ifndef GZ_SUFFIX +# define GZ_SUFFIX ".gz" +#endif +#define SUFFIX_LEN (sizeof(GZ_SUFFIX)-1) + +#define BUFLEN 16384 +#define MAX_NAME_LEN 1024 + +#ifdef MAXSEG_64K +# define local static + /* Needed for systems with limitation on stack size. */ +#else +# define local +#endif + +#ifdef Z_SOLO +/* for Z_SOLO, create simplified gz* functions using deflate and inflate */ + +#if defined(Z_HAVE_UNISTD_H) || defined(Z_LARGE) +# include /* for unlink() */ +#endif + +static void *myalloc(void *q, unsigned n, unsigned m) { + (void)q; + return calloc(n, m); +} + +static void myfree(void *q, void *p) { + (void)q; + free(p); +} + +typedef struct gzFile_s { + FILE *file; + int write; + int err; + char *msg; + z_stream strm; +} *gzFile; + +static gzFile gz_open(const char *path, int fd, const char *mode) { + gzFile gz; + int ret; + + gz = malloc(sizeof(struct gzFile_s)); + if (gz == NULL) + return NULL; + gz->write = strchr(mode, 'w') != NULL; + gz->strm.zalloc = myalloc; + gz->strm.zfree = myfree; + gz->strm.opaque = Z_NULL; + if (gz->write) + ret = deflateInit2(&(gz->strm), -1, 8, 15 + 16, 8, 0); + else { + gz->strm.next_in = 0; + gz->strm.avail_in = Z_NULL; + ret = inflateInit2(&(gz->strm), 15 + 16); + } + if (ret != Z_OK) { + free(gz); + return NULL; + } + gz->file = path == NULL ? fdopen(fd, gz->write ? "wb" : "rb") : + fopen(path, gz->write ? "wb" : "rb"); + if (gz->file == NULL) { + gz->write ? deflateEnd(&(gz->strm)) : inflateEnd(&(gz->strm)); + free(gz); + return NULL; + } + gz->err = 0; + gz->msg = ""; + return gz; +} + +static gzFile gzopen(const char *path, const char *mode) { + return gz_open(path, -1, mode); +} + +static gzFile gzdopen(int fd, const char *mode) { + return gz_open(NULL, fd, mode); +} + +static int gzwrite(gzFile gz, const void *buf, unsigned len) { + z_stream *strm; + unsigned char out[BUFLEN]; + + if (gz == NULL || !gz->write) + return 0; + strm = &(gz->strm); + strm->next_in = (void *)buf; + strm->avail_in = len; + do { + strm->next_out = out; + strm->avail_out = BUFLEN; + (void)deflate(strm, Z_NO_FLUSH); + fwrite(out, 1, BUFLEN - strm->avail_out, gz->file); + } while (strm->avail_out == 0); + return len; +} + +static int gzread(gzFile gz, void *buf, unsigned len) { + int ret; + unsigned got; + unsigned char in[1]; + z_stream *strm; + + if (gz == NULL || gz->write) + return 0; + if (gz->err) + return 0; + strm = &(gz->strm); + strm->next_out = (void *)buf; + strm->avail_out = len; + do { + got = fread(in, 1, 1, gz->file); + if (got == 0) + break; + strm->next_in = in; + strm->avail_in = 1; + ret = inflate(strm, Z_NO_FLUSH); + if (ret == Z_DATA_ERROR) { + gz->err = Z_DATA_ERROR; + gz->msg = strm->msg; + return 0; + } + if (ret == Z_STREAM_END) + inflateReset(strm); + } while (strm->avail_out); + return len - strm->avail_out; +} + +static int gzclose(gzFile gz) { + z_stream *strm; + unsigned char out[BUFLEN]; + + if (gz == NULL) + return Z_STREAM_ERROR; + strm = &(gz->strm); + if (gz->write) { + strm->next_in = Z_NULL; + strm->avail_in = 0; + do { + strm->next_out = out; + strm->avail_out = BUFLEN; + (void)deflate(strm, Z_FINISH); + fwrite(out, 1, BUFLEN - strm->avail_out, gz->file); + } while (strm->avail_out == 0); + deflateEnd(strm); + } + else + inflateEnd(strm); + fclose(gz->file); + free(gz); + return Z_OK; +} + +static const char *gzerror(gzFile gz, int *err) { + *err = gz->err; + return gz->msg; +} + +#endif + +static char *prog; + +/* =========================================================================== + * Display error message and exit + */ +static void error(const char *msg) { + fprintf(stderr, "%s: %s\n", prog, msg); + exit(1); +} + +#ifdef USE_MMAP /* MMAP version, Miguel Albrecht */ + +/* Try compressing the input file at once using mmap. Return Z_OK if + * if success, Z_ERRNO otherwise. + */ +static int gz_compress_mmap(FILE *in, gzFile out) { + int len; + int err; + int ifd = fileno(in); + caddr_t buf; /* mmap'ed buffer for the entire input file */ + off_t buf_len; /* length of the input file */ + struct stat sb; + + /* Determine the size of the file, needed for mmap: */ + if (fstat(ifd, &sb) < 0) return Z_ERRNO; + buf_len = sb.st_size; + if (buf_len <= 0) return Z_ERRNO; + + /* Now do the actual mmap: */ + buf = mmap((caddr_t) 0, buf_len, PROT_READ, MAP_SHARED, ifd, (off_t)0); + if (buf == (caddr_t)(-1)) return Z_ERRNO; + + /* Compress the whole file at once: */ + len = gzwrite(out, (char *)buf, (unsigned)buf_len); + + if (len != (int)buf_len) error(gzerror(out, &err)); + + munmap(buf, buf_len); + fclose(in); + if (gzclose(out) != Z_OK) error("failed gzclose"); + return Z_OK; +} +#endif /* USE_MMAP */ + +/* =========================================================================== + * Compress input to output then close both files. + */ + +static void gz_compress(FILE *in, gzFile out) { + local char buf[BUFLEN]; + int len; + int err; + +#ifdef USE_MMAP + /* Try first compressing with mmap. If mmap fails (minigzip used in a + * pipe), use the normal fread loop. + */ + if (gz_compress_mmap(in, out) == Z_OK) return; +#endif + for (;;) { + len = (int)fread(buf, 1, sizeof(buf), in); + if (ferror(in)) { + perror("fread"); + exit(1); + } + if (len == 0) break; + + if (gzwrite(out, buf, (unsigned)len) != len) error(gzerror(out, &err)); + } + fclose(in); + if (gzclose(out) != Z_OK) error("failed gzclose"); +} + +/* =========================================================================== + * Uncompress input to output then close both files. + */ +static void gz_uncompress(gzFile in, FILE *out) { + local char buf[BUFLEN]; + int len; + int err; + + for (;;) { + len = gzread(in, buf, sizeof(buf)); + if (len < 0) error (gzerror(in, &err)); + if (len == 0) break; + + if ((int)fwrite(buf, 1, (unsigned)len, out) != len) { + error("failed fwrite"); + } + } + if (fclose(out)) error("failed fclose"); + + if (gzclose(in) != Z_OK) error("failed gzclose"); +} + + +/* =========================================================================== + * Compress the given file: create a corresponding .gz file and remove the + * original. + */ +static void file_compress(char *file, char *mode) { + local char outfile[MAX_NAME_LEN]; + FILE *in; + gzFile out; + + if (strlen(file) + strlen(GZ_SUFFIX) >= sizeof(outfile)) { + fprintf(stderr, "%s: filename too long\n", prog); + exit(1); + } + +#if !defined(NO_snprintf) && !defined(NO_vsnprintf) + snprintf(outfile, sizeof(outfile), "%s%s", file, GZ_SUFFIX); +#else + strcpy(outfile, file); + strcat(outfile, GZ_SUFFIX); +#endif + + in = fopen(file, "rb"); + if (in == NULL) { + perror(file); + exit(1); + } + out = gzopen(outfile, mode); + if (out == NULL) { + fprintf(stderr, "%s: can't gzopen %s\n", prog, outfile); + exit(1); + } + gz_compress(in, out); + + unlink(file); +} + + +/* =========================================================================== + * Uncompress the given file and remove the original. + */ +static void file_uncompress(char *file) { + local char buf[MAX_NAME_LEN]; + char *infile, *outfile; + FILE *out; + gzFile in; + z_size_t len = strlen(file); + + if (len + strlen(GZ_SUFFIX) >= sizeof(buf)) { + fprintf(stderr, "%s: filename too long\n", prog); + exit(1); + } + +#if !defined(NO_snprintf) && !defined(NO_vsnprintf) + snprintf(buf, sizeof(buf), "%s", file); +#else + strcpy(buf, file); +#endif + + if (len > SUFFIX_LEN && strcmp(file+len-SUFFIX_LEN, GZ_SUFFIX) == 0) { + infile = file; + outfile = buf; + outfile[len-3] = '\0'; + } else { + outfile = file; + infile = buf; +#if !defined(NO_snprintf) && !defined(NO_vsnprintf) + snprintf(buf + len, sizeof(buf) - len, "%s", GZ_SUFFIX); +#else + strcat(infile, GZ_SUFFIX); +#endif + } + in = gzopen(infile, "rb"); + if (in == NULL) { + fprintf(stderr, "%s: can't gzopen %s\n", prog, infile); + exit(1); + } + out = fopen(outfile, "wb"); + if (out == NULL) { + perror(file); + exit(1); + } + + gz_uncompress(in, out); + + unlink(infile); +} + + +/* =========================================================================== + * Usage: minigzip [-c] [-d] [-f] [-h] [-r] [-1 to -9] [files...] + * -c : write to standard output + * -d : decompress + * -f : compress with Z_FILTERED + * -h : compress with Z_HUFFMAN_ONLY + * -r : compress with Z_RLE + * -1 to -9 : compression level + */ + +int main(int argc, char *argv[]) { + int copyout = 0; + int uncompr = 0; + gzFile file; + char *bname, outmode[20]; + +#if !defined(NO_snprintf) && !defined(NO_vsnprintf) + snprintf(outmode, sizeof(outmode), "%s", "wb6 "); +#else + strcpy(outmode, "wb6 "); +#endif + + prog = argv[0]; + bname = strrchr(argv[0], '/'); + if (bname) + bname++; + else + bname = argv[0]; + argc--, argv++; + + if (!strcmp(bname, "gunzip")) + uncompr = 1; + else if (!strcmp(bname, "zcat")) + copyout = uncompr = 1; + + while (argc > 0) { + if (strcmp(*argv, "-c") == 0) + copyout = 1; + else if (strcmp(*argv, "-d") == 0) + uncompr = 1; + else if (strcmp(*argv, "-f") == 0) + outmode[3] = 'f'; + else if (strcmp(*argv, "-h") == 0) + outmode[3] = 'h'; + else if (strcmp(*argv, "-r") == 0) + outmode[3] = 'R'; + else if ((*argv)[0] == '-' && (*argv)[1] >= '1' && (*argv)[1] <= '9' && + (*argv)[2] == 0) + outmode[2] = (*argv)[1]; + else + break; + argc--, argv++; + } + if (outmode[3] == ' ') + outmode[3] = 0; + if (argc == 0) { + SET_BINARY_MODE(stdin); + SET_BINARY_MODE(stdout); + if (uncompr) { + file = gzdopen(fileno(stdin), "rb"); + if (file == NULL) error("can't gzdopen stdin"); + gz_uncompress(file, stdout); + } else { + file = gzdopen(fileno(stdout), outmode); + if (file == NULL) error("can't gzdopen stdout"); + gz_compress(stdin, file); + } + } else { + if (copyout) { + SET_BINARY_MODE(stdout); + } + do { + if (uncompr) { + if (copyout) { + file = gzopen(*argv, "rb"); + if (file == NULL) + fprintf(stderr, "%s: can't gzopen %s\n", prog, *argv); + else + gz_uncompress(file, stdout); + } else { + file_uncompress(*argv); + } + } else { + if (copyout) { + FILE * in = fopen(*argv, "rb"); + + if (in == NULL) { + perror(*argv); + } else { + file = gzdopen(fileno(stdout), outmode); + if (file == NULL) error("can't gzdopen stdout"); + + gz_compress(in, file); + } + + } else { + file_compress(*argv, outmode); + } + } + } while (argv++, --argc); + } + return 0; +}