Backport HTTPRequest gzip decompression

.github/workflows/javascript_builds.yml

@@ -6,7 +6,7 @@ env:
   GODOT_BASE_BRANCH: 3.x
   SCONSFLAGS: platform=javascript verbose=yes warnings=all werror=yes debug_symbols=no --jobs=2
   SCONS_CACHE_LIMIT: 4096
-  EM_VERSION: 1.39.20
+  EM_VERSION: 2.0.15
   EM_CACHE_FOLDER: 'emsdk-cache'
 
 jobs:

.github/workflows/linux_builds.yml

@@ -208,3 +208,11 @@ jobs:
           echo "----- Run and test project -----"
           DRI_PRIME=0 xvfb-run bin/godot.x11.tools.64s 30 --video-driver GLES3 --audio-driver Dummy --path test_project 2>&1 | tee sanitizers_log.txt || true
           misc/scripts/check_ci_log.py sanitizers_log.txt
+
+      # Check class reference
+      - name: Check for class reference updates
+        run: |
+          echo "Running --doctool to see if this changes the public API without updating the documentation."
+          echo -e "If a diff is shown, it means that your code/doc changes are incomplete and you should update the class reference with --doctool.\n\n"
+          DRI_PRIME=0 xvfb-run bin/godot.x11.tools.64s --doctool . 2>&1 > /dev/null || true
+          git diff --color --exit-code

CHANGELOG.md

@@ -148,6 +148,8 @@ See the [release announcement](https://godotengine.org/article/godot-3-3-has-arr
 - [SVG images can now be used as a project icon](https://github.com/godotengine/godot/pull/43369).
 - [Tweaked log file names for consistency between Mono and non-Mono builds](https://github.com/godotengine/godot/pull/44148).
 - [Tweaked command line `--print-fps` display to display milliseconds per frame timings in addition to FPS](https://github.com/godotengine/godot/pull/47735).
+- [OpenSimplexNoise is now guaranteed to give consistent results across platforms](https://github.com/godotengine/godot/issues/47211).
+  - This change breaks compatibility: you get different results even for the same seed.
 
 #### Editor
 

COPYRIGHT.txt

@@ -133,6 +133,11 @@ Copyright: 2018, Eric Lasota
  2018, Microsoft Corp.
 License: Expat
 
+Files: ./thirdparty/embree/
+Comment: Embree
+Copyright: 2009-2021 Intel Corporation
+License: Apache-2.0
+
 Files: ./thirdparty/enet/
 Comment: ENet
 Copyright: 2002-2020, Lee Salzman

SConstruct

@@ -151,6 +151,7 @@ opts.Add(BoolVariable("use_precise_math_checks", "Math checks use very precise e
 # Thirdparty libraries
 opts.Add(BoolVariable("builtin_bullet", "Use the built-in Bullet library", True))
 opts.Add(BoolVariable("builtin_certs", "Use the built-in SSL certificates bundles", True))
+opts.Add(BoolVariable("builtin_embree", "Use the built-in Embree library", True))
 opts.Add(BoolVariable("builtin_enet", "Use the built-in ENet library", True))
 opts.Add(BoolVariable("builtin_freetype", "Use the built-in FreeType library", True))
 opts.Add(BoolVariable("builtin_libogg", "Use the built-in libogg library", True))
@@ -212,7 +213,7 @@ else:
 if selected_platform in ["linux", "bsd", "linuxbsd"]:
     if selected_platform == "linuxbsd":
         # Alias for forward compatibility.
-        print('Platform "linuxbsd" is still called "x11" in Godot 3.2.x. Building for platform "x11".')
+        print('Platform "linuxbsd" is still called "x11" in Godot 3.x. Building for platform "x11".')
     # Alias for convenience.
     selected_platform = "x11"
 

core/SCsub

@@ -40,6 +40,9 @@ with open("script_encryption_key.gen.cpp", "w") as f:
 
 
 # Add required thirdparty code.
+
+thirdparty_obj = []
+
 env_thirdparty = env.Clone()
 env_thirdparty.disable_warnings()
 
@@ -57,7 +60,7 @@ thirdparty_misc_sources = [
     "clipper.cpp",
 ]
 thirdparty_misc_sources = [thirdparty_misc_dir + file for file in thirdparty_misc_sources]
-env_thirdparty.add_source_files(env.core_sources, thirdparty_misc_sources)
+env_thirdparty.add_source_files(thirdparty_obj, thirdparty_misc_sources)
 
 # Zlib library, can be unbundled
 if env["builtin_zlib"]:
@@ -83,7 +86,7 @@ if env["builtin_zlib"]:
     if env["target"] == "debug":
         env_thirdparty.Append(CPPDEFINES=["ZLIB_DEBUG"])
 
-    env_thirdparty.add_source_files(env.core_sources, thirdparty_zlib_sources)
+    env_thirdparty.add_source_files(thirdparty_obj, thirdparty_zlib_sources)
 
 # Minizip library, could be unbundled in theory
 # However, our version has some custom modifications, so it won't compile with the system one
@@ -94,7 +97,7 @@ thirdparty_minizip_sources = [
     "zip.c",
 ]
 thirdparty_minizip_sources = [thirdparty_minizip_dir + file for file in thirdparty_minizip_sources]
-env_thirdparty.add_source_files(env.core_sources, thirdparty_minizip_sources)
+env_thirdparty.add_source_files(thirdparty_obj, thirdparty_minizip_sources)
 
 # Zstd library, can be unbundled in theory
 # though we currently use some private symbols
@@ -136,10 +139,14 @@ if env["builtin_zstd"]:
     # Also needed in main env includes will trigger warnings
     env.Append(CPPDEFINES=["ZSTD_STATIC_LINKING_ONLY"])
 
-    env_thirdparty.add_source_files(env.core_sources, thirdparty_zstd_sources)
+    env_thirdparty.add_source_files(thirdparty_obj, thirdparty_zstd_sources)
+
 
+env.core_sources += thirdparty_obj
+
+
+# Godot source files
 
-# Godot's own sources
 env.add_source_files(env.core_sources, "*.cpp")
 
 # Certificates
@@ -185,3 +192,6 @@ SConscript("bind/SCsub")
 # Build it all as a library
 lib = env.add_library("core", env.core_sources)
 env.Prepend(LIBS=[lib])
+
+# Needed to force rebuilding the core files when the thirdparty code is updated.
+env.Depends(lib, thirdparty_obj)

core/color.h

@@ -93,14 +93,14 @@ struct Color {
 	Color inverted() const;
 	Color contrasted() const;
 
-	_FORCE_INLINE_ Color linear_interpolate(const Color &p_b, float p_t) const {
+	_FORCE_INLINE_ Color linear_interpolate(const Color &p_to, float p_weight) const {
 
 		Color res = *this;
 
-		res.r += (p_t * (p_b.r - r));
-		res.g += (p_t * (p_b.g - g));
-		res.b += (p_t * (p_b.b - b));
-		res.a += (p_t * (p_b.a - a));
+		res.r += (p_weight * (p_to.r - r));
+		res.g += (p_weight * (p_to.g - g));
+		res.b += (p_weight * (p_to.b - b));
+		res.a += (p_weight * (p_to.a - a));
 
 		return res;
 	}

core/compressed_translation.cpp

@@ -45,6 +45,7 @@ struct _PHashTranslationCmp {
 
 void PHashTranslation::generate(const Ref<Translation> &p_from) {
 #ifdef TOOLS_ENABLED
+	ERR_FAIL_COND(p_from.is_null());
 	List<StringName> keys;
 	p_from->get_message_list(&keys);
 

core/crypto/SCsub

@@ -6,6 +6,7 @@ env_crypto = env.Clone()
 
 is_builtin = env["builtin_mbedtls"]
 has_module = env["module_mbedtls_enabled"]
+thirdparty_obj = []
 
 if is_builtin or not has_module:
     # Use our headers for builtin or if the module is not going to be compiled.
@@ -35,6 +36,16 @@ if not has_module:
         "godot_core_mbedtls_platform.c",
     ]
     thirdparty_mbedtls_sources = [thirdparty_mbedtls_dir + file for file in thirdparty_mbedtls_sources]
-    env_thirdparty.add_source_files(env.core_sources, thirdparty_mbedtls_sources)
+    env_thirdparty.add_source_files(thirdparty_obj, thirdparty_mbedtls_sources)
+    env.core_sources += thirdparty_obj
 
-env_crypto.add_source_files(env.core_sources, "*.cpp")
+
+# Godot source files
+
+core_obj = []
+
+env_crypto.add_source_files(core_obj, "*.cpp")
+env.core_sources += core_obj
+
+# Needed to force rebuilding the core files when the thirdparty library is updated.
+env.Depends(core_obj, thirdparty_obj)

core/io/compression.cpp

@@ -187,8 +187,94 @@ int Compression::decompress(uint8_t *p_dst, int p_dst_max_size, const uint8_t *p
 	ERR_FAIL_V(-1);
 }
 
+/**
+	This will handle both Gzip and Deflat streams. It will automatically allocate the output buffer into the provided p_dst_vect Vector.
+	This is required for compressed data who's final uncompressed size is unknown, as is the case for HTTP response bodies.
+	This is much slower however than using Compression::decompress because it may result in multiple full copies of the output buffer.
+*/
+int Compression::decompress_dynamic(PoolVector<uint8_t> *p_dst, int p_max_dst_size, const uint8_t *p_src, int p_src_size, Mode p_mode) {
+	int ret;
+	uint8_t *dst = nullptr;
+	int out_mark = 0;
+	z_stream strm;
+
+	ERR_FAIL_COND_V(p_src_size <= 0, Z_DATA_ERROR);
+
+	// This function only supports GZip and Deflate
+	int window_bits = p_mode == MODE_DEFLATE ? 15 : 15 + 16;
+	ERR_FAIL_COND_V(p_mode != MODE_DEFLATE && p_mode != MODE_GZIP, Z_ERRNO);
+
+	// Initialize the stream
+	strm.zalloc = Z_NULL;
+	strm.zfree = Z_NULL;
+	strm.opaque = Z_NULL;
+	strm.avail_in = 0;
+	strm.next_in = Z_NULL;
+
+	int err = inflateInit2(&strm, window_bits);
+	ERR_FAIL_COND_V(err != Z_OK, -1);
+
+	// Setup the stream inputs
+	strm.next_in = (Bytef *)p_src;
+	strm.avail_in = p_src_size;
+
+	// Ensure the destination buffer is empty
+	p_dst->resize(0);
+
+	// decompress until deflate stream ends or end of file
+	do {
+		// Add another chunk size to the output buffer
+		// This forces a copy of the whole buffer
+		p_dst->resize(p_dst->size() + gzip_chunk);
+		// Get pointer to the actual output buffer
+		dst = p_dst->write().ptr();
+
+		// Set the stream to the new output stream
+		// Since it was copied, we need to reset the stream to the new buffer
+		strm.next_out = &(dst[out_mark]);
+		strm.avail_out = gzip_chunk;
+
+		// run inflate() on input until output buffer is full and needs to be resized
+		// or input runs out
+		do {
+			ret = inflate(&strm, Z_SYNC_FLUSH);
+
+			switch (ret) {
+				case Z_NEED_DICT:
+					ret = Z_DATA_ERROR;
+				case Z_DATA_ERROR:
+				case Z_MEM_ERROR:
+				case Z_STREAM_ERROR:
+					WARN_PRINT(strm.msg);
+					(void)inflateEnd(&strm);
+					p_dst->resize(0);
+					return ret;
+			}
+		} while (strm.avail_out > 0 && strm.avail_in > 0);
+
+		out_mark += gzip_chunk;
+
+		// Encorce max output size
+		if (p_max_dst_size > -1 && strm.total_out > (uint64_t)p_max_dst_size) {
+			(void)inflateEnd(&strm);
+			p_dst->resize(0);
+			return Z_BUF_ERROR;
+		}
+	} while (ret != Z_STREAM_END);
+
+	// If all done successfully, resize the output if it's larger than the actual output
+	if (ret == Z_STREAM_END && (unsigned long)p_dst->size() > strm.total_out) {
+		p_dst->resize(strm.total_out);
+	}
+
+	// clean up and return
+	(void)inflateEnd(&strm);
+	return ret == Z_STREAM_END ? Z_OK : Z_DATA_ERROR;
+}
+
 int Compression::zlib_level = Z_DEFAULT_COMPRESSION;
 int Compression::gzip_level = Z_DEFAULT_COMPRESSION;
 int Compression::zstd_level = 3;
 bool Compression::zstd_long_distance_matching = false;
 int Compression::zstd_window_log_size = 27; // ZSTD_WINDOWLOG_LIMIT_DEFAULT
+int Compression::gzip_chunk = 16384;

core/io/compression.h

@@ -31,6 +31,7 @@
 #ifndef COMPRESSION_H
 #define COMPRESSION_H
 
+#include "core/pool_vector.h"
 #include "core/typedefs.h"
 
 class Compression {
@@ -41,6 +42,7 @@ class Compression {
 	static int zstd_level;
 	static bool zstd_long_distance_matching;
 	static int zstd_window_log_size;
+	static int gzip_chunk;
 
 	enum Mode {
 		MODE_FASTLZ,
@@ -52,6 +54,7 @@ class Compression {
 	static int compress(uint8_t *p_dst, const uint8_t *p_src, int p_src_size, Mode p_mode = MODE_ZSTD);
 	static int get_max_compressed_buffer_size(int p_src_size, Mode p_mode = MODE_ZSTD);
 	static int decompress(uint8_t *p_dst, int p_dst_max_size, const uint8_t *p_src, int p_src_size, Mode p_mode = MODE_ZSTD);
+	static int decompress_dynamic(PoolVector<uint8_t> *p_dst, int p_max_dst_size, const uint8_t *p_src, int p_src_size, Mode p_mode);
 
 	Compression();
 };

core/io/resource_loader.cpp

@@ -138,18 +138,6 @@ class ResourceInteractiveLoaderDefault : public ResourceInteractiveLoader {
 	ResourceInteractiveLoaderDefault() {}
 };
 
-Ref<ResourceInteractiveLoader> ResourceFormatLoader::load_interactive(const String &p_path, const String &p_original_path, Error *r_error) {
-
-	//either this
-	Ref<Resource> res = load(p_path, p_original_path, r_error);
-	if (res.is_null())
-		return Ref<ResourceInteractiveLoader>();
-
-	Ref<ResourceInteractiveLoaderDefault> ril = Ref<ResourceInteractiveLoaderDefault>(memnew(ResourceInteractiveLoaderDefault));
-	ril->resource = res;
-	return ril;
-}
-
 bool ResourceFormatLoader::exists(const String &p_path) const {
 	return FileAccess::exists(p_path); //by default just check file
 }
@@ -168,25 +156,41 @@ void ResourceFormatLoader::get_recognized_extensions(List<String> *p_extensions)
 	}
 }
 
-RES ResourceFormatLoader::load(const String &p_path, const String &p_original_path, Error *r_error) {
+// Warning: Derived classes must override either `load` or `load_interactive`. The base code
+// here can trigger an infinite recursion otherwise, since `load` calls `load_interactive`
+// vice versa.
+
+Ref<ResourceInteractiveLoader> ResourceFormatLoader::load_interactive(const String &p_path, const String &p_original_path, Error *r_error) {
+	// Warning: See previous note about the risk of infinite recursion.
+	Ref<Resource> res = load(p_path, p_original_path, r_error);
+	if (res.is_null()) {
+		return Ref<ResourceInteractiveLoader>();
+	}
+
+	Ref<ResourceInteractiveLoaderDefault> ril = Ref<ResourceInteractiveLoaderDefault>(memnew(ResourceInteractiveLoaderDefault));
+	ril->resource = res;
+	return ril;
+}
 
+RES ResourceFormatLoader::load(const String &p_path, const String &p_original_path, Error *r_error) {
+	// Check user-defined loader if there's any. Hard fail if it returns an error.
 	if (get_script_instance() && get_script_instance()->has_method("load")) {
 		Variant res = get_script_instance()->call("load", p_path, p_original_path);
 
-		if (res.get_type() == Variant::INT) {
-
-			if (r_error)
+		if (res.get_type() == Variant::INT) { // Error code, abort.
+			if (r_error) {
 				*r_error = (Error)res.operator int64_t();
-
-		} else {
-
-			if (r_error)
+			}
+			return RES();
+		} else { // Success, pass on result.
+			if (r_error) {
 				*r_error = OK;
+			}
 			return res;
 		}
 	}
 
-	//or this must be implemented
+	// Warning: See previous note about the risk of infinite recursion.
 	Ref<ResourceInteractiveLoader> ril = load_interactive(p_path, p_original_path, r_error);
 	if (!ril.is_valid())
 		return RES();

core/math/basis.cpp

@@ -1033,16 +1033,16 @@ void Basis::set_diagonal(const Vector3 &p_diag) {
 	elements[2][2] = p_diag.z;
 }
 
-Basis Basis::slerp(const Basis &target, const real_t &t) const {
+Basis Basis::slerp(const Basis &p_to, const real_t &p_weight) const {
 
 	//consider scale
 	Quat from(*this);
-	Quat to(target);
+	Quat to(p_to);
 
-	Basis b(from.slerp(to, t));
-	b.elements[0] *= Math::lerp(elements[0].length(), target.elements[0].length(), t);
-	b.elements[1] *= Math::lerp(elements[1].length(), target.elements[1].length(), t);
-	b.elements[2] *= Math::lerp(elements[2].length(), target.elements[2].length(), t);
+	Basis b(from.slerp(to, p_weight));
+	b.elements[0] *= Math::lerp(elements[0].length(), p_to.elements[0].length(), p_weight);
+	b.elements[1] *= Math::lerp(elements[1].length(), p_to.elements[1].length(), p_weight);
+	b.elements[2] *= Math::lerp(elements[2].length(), p_to.elements[2].length(), p_weight);
 
 	return b;
 }

core/math/basis.h

@@ -172,7 +172,7 @@ class Basis {
 	bool is_diagonal() const;
 	bool is_rotation() const;
 
-	Basis slerp(const Basis &target, const real_t &t) const;
+	Basis slerp(const Basis &p_to, const real_t &p_weight) const;
 
 	operator String() const;