From 0e4bd964cc60a199b22ae0621c81c804c74b7964 Mon Sep 17 00:00:00 2001 From: bruvzg <7645683+bruvzg@users.noreply.github.com> Date: Wed, 29 Mar 2023 09:31:25 +0300 Subject: [PATCH] Expose brotli decompression to the scripting API. --- SConstruct | 3 + core/SCsub | 25 ++++ core/io/compression.cpp | 218 ++++++++++++++++++++++---------- core/io/compression.h | 3 +- core/io/file_access.cpp | 1 + core/io/file_access.h | 3 +- doc/classes/FileAccess.xml | 3 + doc/classes/PackedByteArray.xml | 2 +- modules/freetype/SCsub | 18 --- modules/freetype/config.py | 8 -- 10 files changed, 186 insertions(+), 98 deletions(-) diff --git a/SConstruct b/SConstruct index 8b39f120b4be..c1e67d8e08f4 100644 --- a/SConstruct +++ b/SConstruct @@ -181,6 +181,7 @@ opts.Add(BoolVariable("production", "Set defaults to build Godot for use in prod opts.Add(BoolVariable("deprecated", "Enable compatibility code for deprecated and removed features", True)) opts.Add(EnumVariable("precision", "Set the floating-point precision level", "single", ("single", "double"))) opts.Add(BoolVariable("minizip", "Enable ZIP archive support using minizip", True)) +opts.Add(BoolVariable("brotli", "Enable Brotli for decompresson and WOFF2 fonts support", True)) opts.Add(BoolVariable("xaudio2", "Enable the XAudio2 audio driver", False)) opts.Add(BoolVariable("vulkan", "Enable the vulkan rendering driver", True)) opts.Add(BoolVariable("opengl3", "Enable the OpenGL/GLES3 rendering driver", True)) @@ -855,6 +856,8 @@ if selected_platform in platform_list: env.Append(CPPDEFINES=["ADVANCED_GUI_DISABLED"]) if env["minizip"]: env.Append(CPPDEFINES=["MINIZIP_ENABLED"]) + if env["brotli"]: + env.Append(CPPDEFINES=["BROTLI_ENABLED"]) if not env["verbose"]: methods.no_verbose(sys, env) diff --git a/core/SCsub b/core/SCsub index 43deff3ad531..a0176f6c3345 100644 --- a/core/SCsub +++ b/core/SCsub @@ -64,6 +64,31 @@ thirdparty_misc_sources = [ thirdparty_misc_sources = [thirdparty_misc_dir + file for file in thirdparty_misc_sources] env_thirdparty.add_source_files(thirdparty_obj, thirdparty_misc_sources) +# Brotli +if env["brotli"]: + thirdparty_brotli_dir = "#thirdparty/brotli/" + thirdparty_brotli_sources = [ + "common/constants.c", + "common/context.c", + "common/dictionary.c", + "common/platform.c", + "common/shared_dictionary.c", + "common/transform.c", + "dec/bit_reader.c", + "dec/decode.c", + "dec/huffman.c", + "dec/state.c", + ] + thirdparty_brotli_sources = [thirdparty_brotli_dir + file for file in thirdparty_brotli_sources] + + env_thirdparty.Prepend(CPPPATH=[thirdparty_brotli_dir + "include"]) + env.Prepend(CPPPATH=[thirdparty_brotli_dir + "include"]) + + if env.get("use_ubsan") or env.get("use_asan") or env.get("use_tsan") or env.get("use_lsan") or env.get("use_msan"): + env_thirdparty.Append(CPPDEFINES=["BROTLI_BUILD_PORTABLE"]) + + env_thirdparty.add_source_files(thirdparty_obj, thirdparty_brotli_sources) + # Zlib library, can be unbundled if env["builtin_zlib"]: thirdparty_zlib_dir = "#thirdparty/zlib/" diff --git a/core/io/compression.cpp b/core/io/compression.cpp index a6114e4f6322..ac4a63759738 100644 --- a/core/io/compression.cpp +++ b/core/io/compression.cpp @@ -35,11 +35,18 @@ #include "thirdparty/misc/fastlz.h" +#ifdef BROTLI_ENABLED +#include "thirdparty/brotli/include/brotli/decode.h" +#endif + #include #include int Compression::compress(uint8_t *p_dst, const uint8_t *p_src, int p_src_size, Mode p_mode) { switch (p_mode) { + case MODE_BROTLI: { + ERR_FAIL_V_MSG(-1, "Only brotli decompression is supported."); + } break; case MODE_FASTLZ: { if (p_src_size < 16) { uint8_t src[16]; @@ -95,6 +102,9 @@ int Compression::compress(uint8_t *p_dst, const uint8_t *p_src, int p_src_size, int Compression::get_max_compressed_buffer_size(int p_src_size, Mode p_mode) { switch (p_mode) { + case MODE_BROTLI: { + ERR_FAIL_V_MSG(-1, "Only brotli decompression is supported."); + } break; case MODE_FASTLZ: { int ss = p_src_size + p_src_size * 6 / 100; if (ss < 66) { @@ -129,6 +139,16 @@ int Compression::get_max_compressed_buffer_size(int p_src_size, Mode p_mode) { int Compression::decompress(uint8_t *p_dst, int p_dst_max_size, const uint8_t *p_src, int p_src_size, Mode p_mode) { switch (p_mode) { + case MODE_BROTLI: { +#ifdef BROTLI_ENABLED + size_t ret_size = p_dst_max_size; + BrotliDecoderResult res = BrotliDecoderDecompress(p_src_size, p_src, &ret_size, p_dst); + ERR_FAIL_COND_V(res != BROTLI_DECODER_RESULT_SUCCESS, -1); + return ret_size; +#else + ERR_FAIL_V_MSG(-1, "Godot was compiled without brotli support."); +#endif + } break; case MODE_FASTLZ: { int ret_size = 0; @@ -186,87 +206,147 @@ int Compression::decompress(uint8_t *p_dst, int p_dst_max_size, const uint8_t *p This is much slower however than using Compression::decompress because it may result in multiple full copies of the output buffer. */ int Compression::decompress_dynamic(Vector *p_dst_vect, int p_max_dst_size, const uint8_t *p_src, int p_src_size, Mode p_mode) { - int ret; uint8_t *dst = nullptr; int out_mark = 0; - z_stream strm; ERR_FAIL_COND_V(p_src_size <= 0, Z_DATA_ERROR); - // This function only supports GZip and Deflate - int window_bits = p_mode == MODE_DEFLATE ? 15 : 15 + 16; - ERR_FAIL_COND_V(p_mode != MODE_DEFLATE && p_mode != MODE_GZIP, Z_ERRNO); - - // Initialize the stream - strm.zalloc = Z_NULL; - strm.zfree = Z_NULL; - strm.opaque = Z_NULL; - strm.avail_in = 0; - strm.next_in = Z_NULL; - - int err = inflateInit2(&strm, window_bits); - ERR_FAIL_COND_V(err != Z_OK, -1); - - // Setup the stream inputs - strm.next_in = (Bytef *)p_src; - strm.avail_in = p_src_size; - - // Ensure the destination buffer is empty - p_dst_vect->clear(); - - // decompress until deflate stream ends or end of file - do { - // Add another chunk size to the output buffer - // This forces a copy of the whole buffer - p_dst_vect->resize(p_dst_vect->size() + gzip_chunk); - // Get pointer to the actual output buffer - dst = p_dst_vect->ptrw(); - - // Set the stream to the new output stream - // Since it was copied, we need to reset the stream to the new buffer - strm.next_out = &(dst[out_mark]); - strm.avail_out = gzip_chunk; - - // run inflate() on input until output buffer is full and needs to be resized - // or input runs out + if (p_mode == MODE_BROTLI) { +#ifdef BROTLI_ENABLED + BrotliDecoderResult ret; + BrotliDecoderState *state = BrotliDecoderCreateInstance(nullptr, nullptr, nullptr); + ERR_FAIL_COND_V(state == nullptr, Z_DATA_ERROR); + + // Setup the stream inputs. + const uint8_t *next_in = p_src; + size_t avail_in = p_src_size; + uint8_t *next_out = nullptr; + size_t avail_out = 0; + size_t total_out = 0; + + // Ensure the destination buffer is empty. + p_dst_vect->clear(); + + // Decompress until stream ends or end of file. do { - ret = inflate(&strm, Z_SYNC_FLUSH); - - switch (ret) { - case Z_NEED_DICT: - ret = Z_DATA_ERROR; - [[fallthrough]]; - case Z_DATA_ERROR: - case Z_MEM_ERROR: - case Z_STREAM_ERROR: - case Z_BUF_ERROR: - if (strm.msg) { - WARN_PRINT(strm.msg); - } - (void)inflateEnd(&strm); - p_dst_vect->clear(); - return ret; + // Add another chunk size to the output buffer. + // This forces a copy of the whole buffer. + p_dst_vect->resize(p_dst_vect->size() + gzip_chunk); + // Get pointer to the actual output buffer. + dst = p_dst_vect->ptrw(); + + // Set the stream to the new output stream. + // Since it was copied, we need to reset the stream to the new buffer. + next_out = &(dst[out_mark]); + avail_out += gzip_chunk; + + ret = BrotliDecoderDecompressStream(state, &avail_in, &next_in, &avail_out, &next_out, &total_out); + if (ret == BROTLI_DECODER_RESULT_ERROR) { + WARN_PRINT(BrotliDecoderErrorString(BrotliDecoderGetErrorCode(state))); + BrotliDecoderDestroyInstance(state); + p_dst_vect->clear(); + return Z_DATA_ERROR; } - } while (strm.avail_out > 0 && strm.avail_in > 0); - out_mark += gzip_chunk; + out_mark += gzip_chunk - avail_out; - // Enforce max output size - if (p_max_dst_size > -1 && strm.total_out > (uint64_t)p_max_dst_size) { - (void)inflateEnd(&strm); - p_dst_vect->clear(); - return Z_BUF_ERROR; + // Enforce max output size. + if (p_max_dst_size > -1 && total_out > (uint64_t)p_max_dst_size) { + BrotliDecoderDestroyInstance(state); + p_dst_vect->clear(); + return Z_BUF_ERROR; + } + } while (ret != BROTLI_DECODER_RESULT_SUCCESS); + + // If all done successfully, resize the output if it's larger than the actual output. + if ((unsigned long)p_dst_vect->size() > total_out) { + p_dst_vect->resize(total_out); } - } while (ret != Z_STREAM_END); - // If all done successfully, resize the output if it's larger than the actual output - if ((unsigned long)p_dst_vect->size() > strm.total_out) { - p_dst_vect->resize(strm.total_out); - } + // Clean up and return. + BrotliDecoderDestroyInstance(state); + return Z_OK; +#else + ERR_FAIL_V_MSG(Z_ERRNO, "Godot was compiled without brotli support."); +#endif + } else { + // This function only supports GZip and Deflate. + ERR_FAIL_COND_V(p_mode != MODE_DEFLATE && p_mode != MODE_GZIP, Z_ERRNO); + + int ret; + z_stream strm; + int window_bits = p_mode == MODE_DEFLATE ? 15 : 15 + 16; + + // Initialize the stream. + strm.zalloc = Z_NULL; + strm.zfree = Z_NULL; + strm.opaque = Z_NULL; + strm.avail_in = 0; + strm.next_in = Z_NULL; + + int err = inflateInit2(&strm, window_bits); + ERR_FAIL_COND_V(err != Z_OK, -1); + + // Setup the stream inputs. + strm.next_in = (Bytef *)p_src; + strm.avail_in = p_src_size; + + // Ensure the destination buffer is empty. + p_dst_vect->clear(); + + // Decompress until deflate stream ends or end of file. + do { + // Add another chunk size to the output buffer. + // This forces a copy of the whole buffer. + p_dst_vect->resize(p_dst_vect->size() + gzip_chunk); + // Get pointer to the actual output buffer. + dst = p_dst_vect->ptrw(); + + // Set the stream to the new output stream. + // Since it was copied, we need to reset the stream to the new buffer. + strm.next_out = &(dst[out_mark]); + strm.avail_out = gzip_chunk; + + // Run inflate() on input until output buffer is full and needs to be resized or input runs out. + do { + ret = inflate(&strm, Z_SYNC_FLUSH); + + switch (ret) { + case Z_NEED_DICT: + ret = Z_DATA_ERROR; + [[fallthrough]]; + case Z_DATA_ERROR: + case Z_MEM_ERROR: + case Z_STREAM_ERROR: + case Z_BUF_ERROR: + if (strm.msg) { + WARN_PRINT(strm.msg); + } + (void)inflateEnd(&strm); + p_dst_vect->clear(); + return ret; + } + } while (strm.avail_out > 0 && strm.avail_in > 0); + + out_mark += gzip_chunk; + + // Enforce max output size. + if (p_max_dst_size > -1 && strm.total_out > (uint64_t)p_max_dst_size) { + (void)inflateEnd(&strm); + p_dst_vect->clear(); + return Z_BUF_ERROR; + } + } while (ret != Z_STREAM_END); + + // If all done successfully, resize the output if it's larger than the actual output. + if ((unsigned long)p_dst_vect->size() > strm.total_out) { + p_dst_vect->resize(strm.total_out); + } - // clean up and return - (void)inflateEnd(&strm); - return Z_OK; + // Clean up and return. + (void)inflateEnd(&strm); + return Z_OK; + } } int Compression::zlib_level = Z_DEFAULT_COMPRESSION; diff --git a/core/io/compression.h b/core/io/compression.h index 063da6dc7dbf..a5a2d657dad1 100644 --- a/core/io/compression.h +++ b/core/io/compression.h @@ -47,7 +47,8 @@ class Compression { MODE_FASTLZ, MODE_DEFLATE, MODE_ZSTD, - MODE_GZIP + MODE_GZIP, + MODE_BROTLI }; static int compress(uint8_t *p_dst, const uint8_t *p_src, int p_src_size, Mode p_mode = MODE_ZSTD); diff --git a/core/io/file_access.cpp b/core/io/file_access.cpp index 3d10151327a9..a6a1a224b30e 100644 --- a/core/io/file_access.cpp +++ b/core/io/file_access.cpp @@ -871,4 +871,5 @@ void FileAccess::_bind_methods() { BIND_ENUM_CONSTANT(COMPRESSION_DEFLATE); BIND_ENUM_CONSTANT(COMPRESSION_ZSTD); BIND_ENUM_CONSTANT(COMPRESSION_GZIP); + BIND_ENUM_CONSTANT(COMPRESSION_BROTLI); } diff --git a/core/io/file_access.h b/core/io/file_access.h index 47770cad8702..34c80b3dd9b4 100644 --- a/core/io/file_access.h +++ b/core/io/file_access.h @@ -64,7 +64,8 @@ class FileAccess : public RefCounted { COMPRESSION_FASTLZ = Compression::MODE_FASTLZ, COMPRESSION_DEFLATE = Compression::MODE_DEFLATE, COMPRESSION_ZSTD = Compression::MODE_ZSTD, - COMPRESSION_GZIP = Compression::MODE_GZIP + COMPRESSION_GZIP = Compression::MODE_GZIP, + COMPRESSION_BROTLI = Compression::MODE_BROTLI, }; typedef void (*FileCloseFailNotify)(const String &); diff --git a/doc/classes/FileAccess.xml b/doc/classes/FileAccess.xml index fb84943eb43c..8dd73c79e526 100644 --- a/doc/classes/FileAccess.xml +++ b/doc/classes/FileAccess.xml @@ -492,5 +492,8 @@ Uses the [url=https://www.gzip.org/]gzip[/url] compression method. + + Uses the [url=https://github.com/google/brotli]brotli[/url] compression method (only decompression is supported). + diff --git a/doc/classes/PackedByteArray.xml b/doc/classes/PackedByteArray.xml index a3f23fa7ae4d..431ccf634ca7 100644 --- a/doc/classes/PackedByteArray.xml +++ b/doc/classes/PackedByteArray.xml @@ -181,7 +181,7 @@ - Returns a new [PackedByteArray] with the data decompressed. Set the compression mode using one of [enum FileAccess.CompressionMode]'s constants. [b]This method only accepts gzip and deflate compression modes.[/b] + Returns a new [PackedByteArray] with the data decompressed. Set the compression mode using one of [enum FileAccess.CompressionMode]'s constants. [b]This method only accepts brotli, gzip, and deflate compression modes.[/b] This method is potentially slower than [code]decompress[/code], as it may have to re-allocate its output buffer multiple times while decompressing, whereas [code]decompress[/code] knows it's output buffer size from the beginning. GZIP has a maximal compression ratio of 1032:1, meaning it's very possible for a small compressed payload to decompress to a potentially very large output. To guard against this, you may provide a maximum size this function is allowed to allocate in bytes via [param max_output_size]. Passing -1 will allow for unbounded output. If any positive value is passed, and the decompression exceeds that amount in bytes, then an error will be returned. diff --git a/modules/freetype/SCsub b/modules/freetype/SCsub index 0b86bc569f1e..421f200f1a73 100644 --- a/modules/freetype/SCsub +++ b/modules/freetype/SCsub @@ -59,25 +59,7 @@ if env["builtin_freetype"]: thirdparty_sources = [thirdparty_dir + file for file in thirdparty_sources] if env["brotli"]: - thirdparty_brotli_dir = "#thirdparty/brotli/" - thirdparty_brotli_sources = [ - "common/constants.c", - "common/context.c", - "common/dictionary.c", - "common/platform.c", - "common/shared_dictionary.c", - "common/transform.c", - "dec/bit_reader.c", - "dec/decode.c", - "dec/huffman.c", - "dec/state.c", - ] - thirdparty_sources += [thirdparty_brotli_dir + file for file in thirdparty_brotli_sources] env_freetype.Append(CPPDEFINES=["FT_CONFIG_OPTION_USE_BROTLI"]) - env_freetype.Prepend(CPPPATH=[thirdparty_brotli_dir + "include"]) - - if env.get("use_ubsan") or env.get("use_asan") or env.get("use_tsan") or env.get("use_lsan") or env.get("use_msan"): - env_freetype.Append(CPPDEFINES=["BROTLI_BUILD_PORTABLE"]) if env["platform"] == "uwp": # Include header for UWP to fix build issues diff --git a/modules/freetype/config.py b/modules/freetype/config.py index c0586d5536a6..d22f9454ed25 100644 --- a/modules/freetype/config.py +++ b/modules/freetype/config.py @@ -2,13 +2,5 @@ def can_build(env, platform): return True -def get_opts(platform): - from SCons.Variables import BoolVariable - - return [ - BoolVariable("brotli", "Enable Brotli decompressor for WOFF2 fonts support", True), - ] - - def configure(env): pass