-Fix freezes caused by etccomp2, closes #9183

-Normalmaps are now detected and imported as RGTC, both in S3TC and ETC2, this improves their quality.

core/global_config.cpp

@@ -970,10 +970,10 @@ GlobalConfig::GlobalConfig() {
 
 	GLOBAL_DEF("debug/profiler/max_functions", 16384);
 
-	GLOBAL_DEF("compression/zstd_compression_level", 3);
-	custom_prop_info["compression/zstd_compression_level"] = PropertyInfo(Variant::INT, "compression/zstd_compression_level", PROPERTY_HINT_RANGE, "1,22,1");
-	GLOBAL_DEF("compression/zlib_compression_level", Z_DEFAULT_COMPRESSION);
-	custom_prop_info["compression/zlib_compression_level"] = PropertyInfo(Variant::INT, "compression/zlib_compression_level", PROPERTY_HINT_RANGE, "-1,9,1");
+	GLOBAL_DEF("compression/zstd/compression_level", 3);
+	custom_prop_info["compression/zstd/compression_level"] = PropertyInfo(Variant::INT, "compression/zstd/compression_level", PROPERTY_HINT_RANGE, "1,22,1");
+	GLOBAL_DEF("compression/zlib/compression_level", Z_DEFAULT_COMPRESSION);
+	custom_prop_info["compression/zlib/compression_level"] = PropertyInfo(Variant::INT, "compression/zlib/compression_level", PROPERTY_HINT_RANGE, "-1,9,1");
 
 	using_datapack = false;
 }

core/image.cpp

@@ -1492,14 +1492,14 @@ Error Image::decompress() {
 	return OK;
 }
 
-Error Image::compress(CompressMode p_mode, bool p_for_srgb, float p_lossy_quality) {
+Error Image::compress(CompressMode p_mode, CompressSource p_source, float p_lossy_quality) {
 
 	switch (p_mode) {
 
 		case COMPRESS_S3TC: {
 
 			ERR_FAIL_COND_V(!_image_compress_bc_func, ERR_UNAVAILABLE);
-			_image_compress_bc_func(this, p_for_srgb);
+			_image_compress_bc_func(this, p_source);
 		} break;
 		case COMPRESS_PVRTC2: {
 
@@ -1519,7 +1519,7 @@ Error Image::compress(CompressMode p_mode, bool p_for_srgb, float p_lossy_qualit
 		case COMPRESS_ETC2: {
 
 			ERR_FAIL_COND_V(!_image_compress_etc2_func, ERR_UNAVAILABLE);
-			_image_compress_etc2_func(this, p_lossy_quality);
+			_image_compress_etc2_func(this, p_lossy_quality, p_source);
 		} break;
 	}
 
@@ -1649,11 +1649,11 @@ void Image::blit_rect(const Ref<Image> &p_src, const Rect2 &p_src_rect, const Po
 Ref<Image> (*Image::_png_mem_loader_func)(const uint8_t *, int) = NULL;
 Ref<Image> (*Image::_jpg_mem_loader_func)(const uint8_t *, int) = NULL;
 
-void (*Image::_image_compress_bc_func)(Image *, bool) = NULL;
+void (*Image::_image_compress_bc_func)(Image *, Image::CompressSource) = NULL;
 void (*Image::_image_compress_pvrtc2_func)(Image *) = NULL;
 void (*Image::_image_compress_pvrtc4_func)(Image *) = NULL;
 void (*Image::_image_compress_etc1_func)(Image *, float) = NULL;
-void (*Image::_image_compress_etc2_func)(Image *, float) = NULL;
+void (*Image::_image_compress_etc2_func)(Image *, float, Image::CompressSource) = NULL;
 void (*Image::_image_decompress_pvrtc)(Image *) = NULL;
 void (*Image::_image_decompress_bc)(Image *) = NULL;
 void (*Image::_image_decompress_etc1)(Image *) = NULL;
@@ -2140,9 +2140,13 @@ void Image::_bind_methods() {
 	BIND_CONSTANT(COMPRESS_PVRTC4);
 	BIND_CONSTANT(COMPRESS_ETC);
 	BIND_CONSTANT(COMPRESS_ETC2);
+
+	BIND_CONSTANT(COMPRESS_SOURCE_GENERIC);
+	BIND_CONSTANT(COMPRESS_SOURCE_SRGB);
+	BIND_CONSTANT(COMPRESS_SOURCE_NORMAL);
 }
 
-void Image::set_compress_bc_func(void (*p_compress_func)(Image *, bool)) {
+void Image::set_compress_bc_func(void (*p_compress_func)(Image *, CompressSource)) {
 
 	_image_compress_bc_func = p_compress_func;
 }

core/image.h

@@ -109,16 +109,22 @@ class Image : public Resource {
 		/* INTERPOLATE GAUSS */
 	};
 
+	enum CompressSource {
+		COMPRESS_SOURCE_GENERIC,
+		COMPRESS_SOURCE_SRGB,
+		COMPRESS_SOURCE_NORMAL
+	};
+
 	//some functions provided by something else
 
 	static Ref<Image> (*_png_mem_loader_func)(const uint8_t *p_png, int p_size);
 	static Ref<Image> (*_jpg_mem_loader_func)(const uint8_t *p_png, int p_size);
 
-	static void (*_image_compress_bc_func)(Image *, bool p_srgb);
+	static void (*_image_compress_bc_func)(Image *, CompressSource p_source);
 	static void (*_image_compress_pvrtc2_func)(Image *);
 	static void (*_image_compress_pvrtc4_func)(Image *);
 	static void (*_image_compress_etc1_func)(Image *, float);
-	static void (*_image_compress_etc2_func)(Image *, float);
+	static void (*_image_compress_etc2_func)(Image *, float, CompressSource p_source);
 
 	static void (*_image_decompress_pvrtc)(Image *);
 	static void (*_image_decompress_bc)(Image *);
@@ -267,7 +273,7 @@ class Image : public Resource {
 		COMPRESS_ETC2,
 	};
 
-	Error compress(CompressMode p_mode = COMPRESS_S3TC, bool p_for_srgb = false, float p_lossy_quality = 0.7);
+	Error compress(CompressMode p_mode = COMPRESS_S3TC, CompressSource p_source = COMPRESS_SOURCE_GENERIC, float p_lossy_quality = 0.7);
 	Error decompress();
 	bool is_compressed() const;
 
@@ -281,7 +287,7 @@ class Image : public Resource {
 	Rect2 get_used_rect() const;
 	Ref<Image> get_rect(const Rect2 &p_area) const;
 
-	static void set_compress_bc_func(void (*p_compress_func)(Image *, bool));
+	static void set_compress_bc_func(void (*p_compress_func)(Image *, CompressSource));
 	static String get_format_name(Format p_format);
 
 	Image(const uint8_t *p_mem_png_jpg, int p_len = -1);
@@ -322,6 +328,7 @@ class Image : public Resource {
 VARIANT_ENUM_CAST(Image::Format)
 VARIANT_ENUM_CAST(Image::Interpolation)
 VARIANT_ENUM_CAST(Image::CompressMode)
+VARIANT_ENUM_CAST(Image::CompressSource)
 VARIANT_ENUM_CAST(Image::AlphaMode)
 
 #endif

core/io/compression.cpp

@@ -58,7 +58,7 @@ int Compression::compress(uint8_t *p_dst, const uint8_t *p_src, int p_src_size,
 			strm.zalloc = zipio_alloc;
 			strm.zfree = zipio_free;
 			strm.opaque = Z_NULL;
-			int level = GLOBAL_GET("compression/zlib_compression_level");
+			int level = GLOBAL_GET("compression/zlib/compression_level");
 			int err = deflateInit(&strm, level);
 			if (err != Z_OK)
 				return -1;
@@ -81,7 +81,7 @@ int Compression::compress(uint8_t *p_dst, const uint8_t *p_src, int p_src_size,
 		case MODE_ZSTD: {
 
 			int max_dst_size = get_max_compressed_buffer_size(p_src_size, MODE_ZSTD);
-			int level = GLOBAL_GET("compression/zstd_compression_level");
+			int level = GLOBAL_GET("compression/zstd/compression_level");
 			return ZSTD_compress(p_dst, max_dst_size, p_src, p_src_size, level);
 		} break;
 	}

drivers/gles3/rasterizer_scene_gles3.cpp

@@ -1207,6 +1207,7 @@ bool RasterizerSceneGLES3::_setup_material(RasterizerStorageGLES3::Material *p_m
 				} break;
 				case ShaderLanguage::ShaderNode::Uniform::HINT_NORMAL: {
 					tex = storage->resources.normal_tex;
+
 				} break;
 				default: {
 					tex = storage->resources.white_tex;
@@ -1220,6 +1221,13 @@ bool RasterizerSceneGLES3::_setup_material(RasterizerStorageGLES3::Material *p_m
 				t->detect_3d(t->detect_3d_ud);
 			}
 #endif
+
+#ifdef TOOLS_ENABLED
+			if (t->detect_normal && texture_hints[i] == ShaderLanguage::ShaderNode::Uniform::HINT_NORMAL) {
+				t->detect_normal(t->detect_normal_ud);
+			}
+#endif
+
 			if (storage->config.srgb_decode_supported) {
 				//if SRGB decode extension is present, simply switch the texture to whathever is needed
 				bool must_srgb = false;

drivers/gles3/rasterizer_storage_gles3.cpp

@@ -1072,6 +1072,14 @@ void RasterizerStorageGLES3::texture_set_detect_srgb_callback(RID p_texture, Vis
 	texture->detect_srgb_ud = p_userdata;
 }
 
+void RasterizerStorageGLES3::texture_set_detect_normal_callback(RID p_texture, VisualServer::TextureDetectCallback p_callback, void *p_userdata) {
+	Texture *texture = texture_owner.get(p_texture);
+	ERR_FAIL_COND(!texture);
+
+	texture->detect_normal = p_callback;
+	texture->detect_normal_ud = p_userdata;
+}
+
 RID RasterizerStorageGLES3::texture_create_radiance_cubemap(RID p_source, int p_resolution) const {
 
 	Texture *texture = texture_owner.get(p_source);

drivers/gles3/rasterizer_storage_gles3.h

@@ -270,6 +270,9 @@ class RasterizerStorageGLES3 : public RasterizerStorage {
 		VisualServer::TextureDetectCallback detect_srgb;
 		void *detect_srgb_ud;
 
+		VisualServer::TextureDetectCallback detect_normal;
+		void *detect_normal_ud;
+
 		Texture() {
 
 			using_srgb = false;
@@ -289,6 +292,8 @@ class RasterizerStorageGLES3 : public RasterizerStorage {
 			detect_3d_ud = NULL;
 			detect_srgb = NULL;
 			detect_srgb_ud = NULL;
+			detect_normal = NULL;
+			detect_normal_ud = NULL;
 		}
 
 		~Texture() {
@@ -329,6 +334,7 @@ class RasterizerStorageGLES3 : public RasterizerStorage {
 
 	virtual void texture_set_detect_3d_callback(RID p_texture, VisualServer::TextureDetectCallback p_callback, void *p_userdata);
 	virtual void texture_set_detect_srgb_callback(RID p_texture, VisualServer::TextureDetectCallback p_callback, void *p_userdata);
+	virtual void texture_set_detect_normal_callback(RID p_texture, VisualServer::TextureDetectCallback p_callback, void *p_userdata);
 
 	/* SKY API */
 

editor/import/resource_importer_texture.cpp

@@ -66,6 +66,22 @@ void ResourceImporterTexture::_texture_reimport_3d(const Ref<StreamTexture> &p_t
 	singleton->mutex->unlock();
 }
 
+void ResourceImporterTexture::_texture_reimport_normal(const Ref<StreamTexture> &p_tex) {
+
+	singleton->mutex->lock();
+	StringName path = p_tex->get_path();
+
+	if (!singleton->make_flags.has(path)) {
+		singleton->make_flags[path] = 0;
+	}
+
+	singleton->make_flags[path] |= MAKE_NORMAL_FLAG;
+
+	print_line("requesting normalfor " + String(path));
+
+	singleton->mutex->unlock();
+}
+
 void ResourceImporterTexture::update_imports() {
 
 	if (EditorFileSystem::get_singleton()->is_scanning() || EditorFileSystem::get_singleton()->is_importing()) {
@@ -96,6 +112,11 @@ void ResourceImporterTexture::update_imports() {
 			changed = true;
 		}
 
+		if (E->get() & MAKE_NORMAL_FLAG && int(cf->get_value("params", "compress/normal_map")) == 0) {
+			cf->set_value("params", "compress/normal_map", 1);
+			changed = true;
+		}
+
 		if (E->get() & MAKE_3D_FLAG && bool(cf->get_value("params", "detect_3d"))) {
 			cf->set_value("params", "detect_3d", false);
 			cf->set_value("params", "compress/mode", 2);
@@ -174,6 +195,7 @@ void ResourceImporterTexture::get_import_options(List<ImportOption> *r_options,
 	r_options->push_back(ImportOption(PropertyInfo(Variant::INT, "compress/mode", PROPERTY_HINT_ENUM, "Lossless,Lossy,Video RAM,Uncompressed", PROPERTY_USAGE_DEFAULT | PROPERTY_USAGE_UPDATE_ALL_IF_MODIFIED), p_preset == PRESET_3D ? 2 : 0));
 	r_options->push_back(ImportOption(PropertyInfo(Variant::REAL, "compress/lossy_quality", PROPERTY_HINT_RANGE, "0,1,0.01"), 0.7));
 	r_options->push_back(ImportOption(PropertyInfo(Variant::INT, "compress/hdr_mode", PROPERTY_HINT_ENUM, "Compress,Force RGBE"), 0));
+	r_options->push_back(ImportOption(PropertyInfo(Variant::INT, "compress/normal_map", PROPERTY_HINT_ENUM, "Detect,Enable,Disabled"), 0));
 	r_options->push_back(ImportOption(PropertyInfo(Variant::INT, "flags/repeat", PROPERTY_HINT_ENUM, "Disabled,Enabled,Mirrored"), p_preset == PRESET_3D ? 1 : 0));
 	r_options->push_back(ImportOption(PropertyInfo(Variant::BOOL, "flags/filter"), p_preset == PRESET_2D_PIXEL ? false : true));
 	r_options->push_back(ImportOption(PropertyInfo(Variant::BOOL, "flags/mipmaps"), p_preset == PRESET_3D ? true : false));
@@ -187,8 +209,9 @@ void ResourceImporterTexture::get_import_options(List<ImportOption> *r_options,
 	r_options->push_back(ImportOption(PropertyInfo(Variant::BOOL, "detect_3d"), p_preset == PRESET_DETECT));
 }
 
-void ResourceImporterTexture::_save_stex(const Ref<Image> &p_image, const String &p_to_path, int p_compress_mode, float p_lossy_quality, Image::CompressMode p_vram_compression, bool p_mipmaps, int p_texture_flags, bool p_streamable, bool p_detect_3d, bool p_detect_srgb, bool p_force_rgbe) {
+void ResourceImporterTexture::_save_stex(const Ref<Image> &p_image, const String &p_to_path, int p_compress_mode, float p_lossy_quality, Image::CompressMode p_vram_compression, bool p_mipmaps, int p_texture_flags, bool p_streamable, bool p_detect_3d, bool p_detect_srgb, bool p_force_rgbe, bool p_detect_normal, bool p_force_normal) {
 
+	print_line("saving: " + p_to_path);
 	FileAccess *f = FileAccess::open(p_to_path, FileAccess::WRITE);
 	f->store_8('G');
 	f->store_8('D');
@@ -209,6 +232,8 @@ void ResourceImporterTexture::_save_stex(const Ref<Image> &p_image, const String
 		format |= StreamTexture::FORMAT_BIT_DETECT_3D;
 	if (p_detect_srgb)
 		format |= StreamTexture::FORMAT_BIT_DETECT_SRGB;
+	if (p_detect_normal)
+		format |= StreamTexture::FORMAT_BIT_DETECT_NORMAL;
 
 	if ((p_compress_mode == COMPRESS_LOSSLESS || p_compress_mode == COMPRESS_LOSSY) && p_image->get_format() > Image::FORMAT_RGBA8) {
 		p_compress_mode == COMPRESS_UNCOMPRESSED; //these can't go as lossy
@@ -281,7 +306,14 @@ void ResourceImporterTexture::_save_stex(const Ref<Image> &p_image, const String
 			if (p_force_rgbe && image->get_format() >= Image::FORMAT_R8 && image->get_format() <= Image::FORMAT_RGBE9995) {
 				image->convert(Image::FORMAT_RGBE9995);
 			} else {
-				image->compress(p_vram_compression, p_texture_flags & VS::TEXTURE_FLAG_CONVERT_TO_LINEAR, p_lossy_quality);
+				Image::CompressSource csource = Image::COMPRESS_SOURCE_GENERIC;
+				if (p_force_normal) {
+					csource = Image::COMPRESS_SOURCE_NORMAL;
+				} else if (p_texture_flags & VS::TEXTURE_FLAG_CONVERT_TO_LINEAR) {
+					csource = Image::COMPRESS_SOURCE_SRGB;
+				}
+
+				image->compress(p_vram_compression, csource, p_lossy_quality);
 			}
 
 			format |= image->get_format();
@@ -292,7 +324,6 @@ void ResourceImporterTexture::_save_stex(const Ref<Image> &p_image, const String
 			int dl = data.size();
 			PoolVector<uint8_t>::Read r = data.read();
 			f->store_buffer(r.ptr(), dl);
-
 		} break;
 		case COMPRESS_UNCOMPRESSED: {
 
@@ -333,6 +364,7 @@ Error ResourceImporterTexture::import(const String &p_source_file, const String
 	int size_limit = p_options["size_limit"];
 	bool force_rgbe = int(p_options["compress/hdr_mode"]) == 1;
 	bool hdr_as_srgb = p_options["process/HDR_as_SRGB"];
+	int normal = p_options["compress/normal_map"];
 
 	Ref<Image> image;
 	image.instance();
@@ -380,20 +412,38 @@ Error ResourceImporterTexture::import(const String &p_source_file, const String
 
 	bool detect_3d = p_options["detect_3d"];
 	bool detect_srgb = srgb == 2;
+	bool detect_normal = normal == 0;
+	bool force_normal = normal == 1;
 
 	if (compress_mode == COMPRESS_VIDEO_RAM) {
 		//must import in all formats
 		//Android, GLES 2.x
-		_save_stex(image, p_save_path + ".etc.stex", compress_mode, lossy, Image::COMPRESS_ETC, mipmaps, tex_flags, stream, detect_3d, detect_srgb, force_rgbe);
-		r_platform_variants->push_back("etc");
-		_save_stex(image, p_save_path + ".etc2.stex", compress_mode, lossy, Image::COMPRESS_ETC2, mipmaps, tex_flags, stream, detect_3d, detect_srgb, force_rgbe);
-		r_platform_variants->push_back("etc2");
-		_save_stex(image, p_save_path + ".s3tc.stex", compress_mode, lossy, Image::COMPRESS_S3TC, mipmaps, tex_flags, stream, detect_3d, detect_srgb, force_rgbe);
-		r_platform_variants->push_back("s3tc");
+		if (GlobalConfig::get_singleton()->get("rendering/vram_formats/use_s3tc")) {
+
+			_save_stex(image, p_save_path + ".s3tc.stex", compress_mode, lossy, Image::COMPRESS_S3TC, mipmaps, tex_flags, stream, detect_3d, detect_srgb, force_rgbe, detect_normal, force_normal);
+			r_platform_variants->push_back("s3tc");
+		}
+
+		if (GlobalConfig::get_singleton()->get("rendering/vram_formats/use_etc")) {
+			_save_stex(image, p_save_path + ".etc.stex", compress_mode, lossy, Image::COMPRESS_ETC, mipmaps, tex_flags, stream, detect_3d, detect_srgb, force_rgbe, detect_normal, force_normal);
+			r_platform_variants->push_back("etc");
+		}
+
+		if (GlobalConfig::get_singleton()->get("rendering/vram_formats/use_etc2")) {
+
+			_save_stex(image, p_save_path + ".etc2.stex", compress_mode, lossy, Image::COMPRESS_ETC2, mipmaps, tex_flags, stream, detect_3d, detect_srgb, force_rgbe, detect_normal, force_normal);
+			r_platform_variants->push_back("etc2");
+		}
+
+		if (GlobalConfig::get_singleton()->get("rendering/vram_formats/use_pvrtc")) {
+
+			_save_stex(image, p_save_path + ".pvrtc.stex", compress_mode, lossy, Image::COMPRESS_PVRTC4, mipmaps, tex_flags, stream, detect_3d, detect_srgb, force_rgbe, detect_normal, force_normal);
+			r_platform_variants->push_back("pvrtc");
+		}
 
 	} else {
 		//import normally
-		_save_stex(image, p_save_path + ".stex", compress_mode, lossy, Image::COMPRESS_S3TC /*this is ignored */, mipmaps, tex_flags, stream, detect_3d, detect_srgb, force_rgbe);
+		_save_stex(image, p_save_path + ".stex", compress_mode, lossy, Image::COMPRESS_S3TC /*this is ignored */, mipmaps, tex_flags, stream, detect_3d, detect_srgb, force_rgbe, detect_normal, force_normal);
 	}
 
 	return OK;
@@ -406,6 +456,7 @@ ResourceImporterTexture::ResourceImporterTexture() {
 	singleton = this;
 	StreamTexture::request_3d_callback = _texture_reimport_3d;
 	StreamTexture::request_srgb_callback = _texture_reimport_srgb;
+	StreamTexture::request_normal_callback = _texture_reimport_normal;
 	mutex = Mutex::create();
 }
 

editor/import/resource_importer_texture.h

@@ -41,14 +41,16 @@ class ResourceImporterTexture : public ResourceImporter {
 protected:
 	enum {
 		MAKE_3D_FLAG = 1,
-		MAKE_SRGB_FLAG = 2
+		MAKE_SRGB_FLAG = 2,
+		MAKE_NORMAL_FLAG = 4
 	};
 
 	Mutex *mutex;
 	Map<StringName, int> make_flags;
 
 	static void _texture_reimport_srgb(const Ref<StreamTexture> &p_tex);
 	static void _texture_reimport_3d(const Ref<StreamTexture> &p_tex);
+	static void _texture_reimport_normal(const Ref<StreamTexture> &p_tex);
 
 	static ResourceImporterTexture *singleton;
 
@@ -80,7 +82,7 @@ class ResourceImporterTexture : public ResourceImporter {
 	virtual void get_import_options(List<ImportOption> *r_options, int p_preset = 0) const;
 	virtual bool get_option_visibility(const String &p_option, const Map<StringName, Variant> &p_options) const;
 
-	void _save_stex(const Ref<Image> &p_image, const String &p_to_path, int p_compress_mode, float p_lossy_quality, Image::CompressMode p_vram_compression, bool p_mipmaps, int p_texture_flags, bool p_streamable, bool p_detect_3d, bool p_detect_srgb, bool p_force_rgbe);
+	void _save_stex(const Ref<Image> &p_image, const String &p_to_path, int p_compress_mode, float p_lossy_quality, Image::CompressMode p_vram_compression, bool p_mipmaps, int p_texture_flags, bool p_streamable, bool p_detect_3d, bool p_detect_srgb, bool p_force_rgbe, bool p_detect_normal, bool p_force_normal);
 
 	virtual Error import(const String &p_source_file, const String &p_save_path, const Map<StringName, Variant> &p_options, List<String> *r_platform_variants, List<String> *r_gen_files = NULL);