[Mono] Initial metadata update support

First part of support for metadata updates.

Contributes to dotnet/runtime#44806

The feature is off by default.  To enable, build with `/p:MonoMetadataUpdate=true` (e.g. `./build.sh --os browser /p:MonoMetadataUpdate=true`).  There are samples in `src/mono/netcore/sample/mbr` (see the README - the samples aren't completely standalone and need some external tooling to build) for console (only tested on Mac and Linux) and wasm.

There's a demo at https://lambdageek.dev/dl0/

mono/cil/tables.def

@@ -44,3 +44,16 @@ TABLEDEF(MONO_TABLE_GENERICPARAM, "GenericParam") /* 0x2a */
 TABLEDEF(MONO_TABLE_METHODSPEC, "MethodSpec")
 TABLEDEF(MONO_TABLE_GENERICPARAMCONSTRAINT, "GenericParamConstraint")
 
+TABLEDEF(MONO_TABLE_UNUSED8, "Unused8")
+TABLEDEF(MONO_TABLE_UNUSED9, "Unused9")
+TABLEDEF(MONO_TABLE_UNUSED10, "Unused10")
+
+/* Portable PDB tables */
+TABLEDEF(MONO_TABLE_DOCUMENT, "Document")
+TABLEDEF(MONO_TABLE_METHODBODY, "Methodbody")
+TABLEDEF(MONO_TABLE_LOCALSCOPE, "LocalScope")
+TABLEDEF(MONO_TABLE_LOCALVARIABLE, "LocalVariable")
+TABLEDEF(MONO_TABLE_LOCALCONSTANT, "LocalConstant")
+TABLEDEF(MONO_TABLE_IMPORTSCOPE, "ImportScope")
+TABLEDEF(MONO_TABLE_STATEMACHINEMETHOD, "StateMachineMethod")
+TABLEDEF(MONO_TABLE_CUSTOMDEBUGINFORMATION, "CustomDebugInformation")

mono/metadata/Makefile.am

@@ -311,6 +311,8 @@ common_sources = \
 	metadata.c		\
 	metadata-verify.c	\
 	metadata-internals.h	\
+	metadata-update.h	\
+	metadata-update.c	\
 	method-builder.h 	\
 	method-builder-internals.h 	\
 	method-builder.c 	\

mono/metadata/class.c

@@ -197,7 +197,7 @@ mono_class_from_typeref_checked (MonoImage *image, guint32 type_token, MonoError
 		break;
 	}
 
-	if (idx > image->tables [MONO_TABLE_ASSEMBLYREF].rows) {
+	if (mono_metadata_table_bounds_check (image, MONO_TABLE_ASSEMBLYREF, idx)) {
 		mono_error_set_bad_image (error, image, "Image with invalid assemblyref token %08x.", idx);
 		return NULL;
 	}

mono/metadata/domain.c

@@ -924,7 +924,12 @@ mono_cleanup (void)
 void
 mono_close_exe_image (void)
 {
-	if (exe_image)
+	gboolean do_close = exe_image != NULL;
+#ifdef ENABLE_METADATA_UPDATE
+	/* FIXME: shutdown hack. We mess something up and try to double-close/free it. */
+	do_close = do_close && !exe_image->delta_image;
+#endif
+	if (do_close)
 		mono_image_close (exe_image);
 }
 

mono/metadata/icall-decl.h

@@ -308,4 +308,8 @@ ICALL_EXPORT void     ves_icall_System_Threading_LowLevelLifoSemaphore_ReleaseIn
 ICALL_EXPORT void ves_icall_System_Runtime_Intrinsics_X86_X86Base___cpuidex (int abcd[4], int function_id, int subfunction_id);
 #endif
 
+#if defined(ENABLE_NETCORE) && defined(ENABLE_METADATA_UPDATE)
+ICALL_EXPORT void ves_icall_Mono_Runtime_LoadMetadataUpdate (MonoAssembly *assm, gconstpointer dmeta_bytes, int32_t dmeta_len, gconstpointer dil_bytes, int32_t dil_len);
+#endif
+
 #endif // __MONO_METADATA_ICALL_DECL_H__

mono/metadata/icall-def-netcore.h

@@ -326,6 +326,11 @@ HANDLES_REUSE_WRAPPER(MPROP_3, "get_metadata_token", ves_icall_reflection_get_to
 HANDLES(MPROP_4, "get_property_info", ves_icall_RuntimePropertyInfo_get_property_info, void, 3, (MonoReflectionProperty, MonoPropertyInfo_ref, PInfo))
 HANDLES(MPROP_5, "internal_from_handle_type", ves_icall_System_Reflection_RuntimePropertyInfo_internal_from_handle_type, MonoReflectionProperty, 2, (MonoProperty_ptr, MonoType_ptr))
 
+#ifdef ENABLE_METADATA_UPDATE
+ICALL_TYPE(RUNF, "System.Runtime.CompilerServices.RuntimeFeature", RUNF_1)
+NOHANDLES(ICALL(RUNF_1, "LoadMetadataUpdate_internal", ves_icall_Mono_Runtime_LoadMetadataUpdate))
+#endif
+
 ICALL_TYPE(RUNH, "System.Runtime.CompilerServices.RuntimeHelpers", RUNH_1)
 HANDLES(RUNH_1, "GetObjectValue", ves_icall_System_Runtime_CompilerServices_RuntimeHelpers_GetObjectValue, MonoObject, 1, (MonoObject))
 HANDLES(RUNH_2, "GetUninitializedObjectInternal", ves_icall_System_Runtime_CompilerServices_RuntimeHelpers_GetUninitializedObjectInternal, MonoObject, 1, (MonoType_ptr))

mono/metadata/icall.c

@@ -6777,6 +6777,24 @@ ves_icall_Mono_Runtime_DumpStateTotal (guint64 *portable_hash, guint64 *unportab
 	return result;
 }
 
+#if defined (ENABLE_NETCORE) && defined (ENABLE_METADATA_UPDATE)
+void
+ves_icall_Mono_Runtime_LoadMetadataUpdate (MonoAssembly *assm,
+					   gconstpointer dmeta_bytes, int32_t dmeta_len,
+					   gconstpointer dil_bytes, int32_t dil_len)
+{
+	ERROR_DECL (error);
+	g_assert (assm);
+	g_assert (dmeta_len >= 0);
+	MonoImage *image_base = assm->image;
+	g_assert (image_base);
+
+	MonoDomain *domain = mono_domain_get ();
+	mono_image_load_enc_delta (domain, image_base, dmeta_bytes, dmeta_len, dil_bytes, dil_len, error);
+	mono_error_set_pending_exception (error);
+}
+#endif
+
 MonoBoolean
 ves_icall_System_Reflection_AssemblyName_ParseAssemblyName (const char *name, MonoAssemblyName *aname, MonoBoolean *is_version_defined_arg, MonoBoolean *is_token_defined_arg)
 {

mono/metadata/image.c

@@ -24,6 +24,7 @@
 #include "tabledefs.h"
 #include "tokentype.h"
 #include "metadata-internals.h"
+#include "metadata-update.h"
 #include "profiler-private.h"
 #include "loader.h"
 #include "marshal.h"
@@ -45,6 +46,7 @@
 #include <mono/metadata/verify.h>
 #include <mono/metadata/image-internals.h>
 #include <mono/metadata/loaded-images-internals.h>
+#include <mono/metadata/metadata-update.h>
 #include <mono/metadata/w32process-internals.h>
 #include <mono/metadata/debug-internals.h>
 #include <mono/metadata/mono-private-unstable.h>
@@ -598,7 +600,7 @@ load_metadata_ptrs (MonoImage *image, MonoCLIImageInfo *iinfo)
 }
 
 /*
- * Load representation of logical metadata tables, from the "#~" stream
+ * Load representation of logical metadata tables, from the "#~" or "#-" stream
  */
 static gboolean
 load_tables (MonoImage *image)
@@ -613,6 +615,15 @@ load_tables (MonoImage *image)
 	image->idx_string_wide = ((heap_sizes & 0x01) == 1);
 	image->idx_guid_wide   = ((heap_sizes & 0x02) == 2);
 	image->idx_blob_wide   = ((heap_sizes & 0x04) == 4);
+
+#ifdef ENABLE_METADATA_UPDATE
+	if (G_UNLIKELY (image->minimal_delta)) {
+		/* sanity check */
+		g_assert (image->idx_string_wide);
+		g_assert (image->idx_guid_wide);
+		g_assert (image->idx_blob_wide);
+	}
+#endif
 
 	valid_mask = read64 (heap_tables + 8);
 	rows = (const guint32 *) (heap_tables + 24);
@@ -1462,6 +1473,26 @@ mono_is_problematic_image (MonoImage *image)
 	return FALSE;
 }
 
+#ifdef ENABLE_METADATA_UPDATE
+static void
+dump_encmap (MonoImage *image)
+{
+	MonoTableInfo *encmap = &image->tables [MONO_TABLE_ENCMAP];
+	if (!encmap || !encmap->rows)
+		return;
+
+	if (mono_trace_is_traced (G_LOG_LEVEL_DEBUG, MONO_TRACE_METADATA_UPDATE)) {
+		mono_trace (G_LOG_LEVEL_DEBUG, MONO_TRACE_METADATA_UPDATE, "ENCMAP for %s", image->filename);
+		for (int i = 0; i < encmap->rows; ++i) {
+			guint32 cols [MONO_ENCMAP_SIZE];
+			mono_metadata_decode_row (encmap, i, cols, MONO_ENCMAP_SIZE);
+			int token = cols [MONO_ENCMAP_TOKEN];
+			mono_trace (G_LOG_LEVEL_DEBUG, MONO_TRACE_METADATA_UPDATE, "\t0x%08x: 0x%08x table: %s", i+1, token, mono_meta_table_name (mono_metadata_token_table (token)));
+		}
+	}
+}
+#endif
+
 static MonoImage *
 do_mono_image_load (MonoImage *image, MonoImageOpenStatus *status,
 		    gboolean care_about_cli, gboolean care_about_pecoff)
@@ -1526,6 +1557,10 @@ do_mono_image_load (MonoImage *image, MonoImageOpenStatus *status,
 	if (image->loader == &pe_loader && !image->metadata_only && !mono_verifier_verify_table_data (image, error))
 		goto invalid_image;
 
+#ifdef ENABLE_METADATA_UPDATE
+	dump_encmap (image);
+#endif
+
 	mono_image_load_names (image);
 
 	mono_image_load_time_date_stamp (image);
@@ -2513,6 +2548,20 @@ mono_image_close_except_pools_all (MonoImage**images, int image_count)
 	}
 }
 
+#ifdef ENABLE_METADATA_UPDATE
+static void
+mono_image_close_except_pools_all_list (GList *images)
+{
+	for (GList *ptr = images; ptr; ptr = ptr->next) {
+		MonoImage *image = (MonoImage *)ptr->data;
+		if (image) {
+			if (!mono_image_close_except_pools (image))
+			    ptr->data = NULL;
+		}
+	}
+}
+#endif
+
 /*
  * Returns whether mono_image_close_finish() must be called as well.
  * We must unload images in two steps because clearing the domain in
@@ -2552,6 +2601,10 @@ mono_image_close_except_pools (MonoImage *image)
 
 	mono_metadata_clean_for_image (image);
 
+#ifdef ENABLE_METADATA_UPDATE
+	mono_metadata_update_cleanup_on_close (image);
+#endif
+
 	/*
 	 * The caches inside a MonoImage might refer to metadata which is stored in referenced 
 	 * assemblies, so we can't release these references in mono_assembly_close () since the
@@ -2679,6 +2732,11 @@ mono_image_close_except_pools (MonoImage *image)
 	mono_image_close_except_pools_all (image->modules, image->module_count);
 	g_free (image->modules_loaded);
 
+#ifdef ENABLE_METADATA_UPDATE
+	if (image->delta_image)
+		mono_image_close_except_pools_all_list (image->delta_image);
+#endif
+
 	mono_os_mutex_destroy (&image->szarray_cache_lock);
 	mono_os_mutex_destroy (&image->lock);
 
@@ -2705,6 +2763,20 @@ mono_image_close_all (MonoImage**images, int image_count)
 		g_free (images);
 }
 
+#ifdef ENABLE_METADATA_UPDATE
+static void
+mono_image_close_all_list (GList *images)
+{
+	for (GList *ptr = images; ptr; ptr = ptr->next) {
+		MonoImage *image = (MonoImage *)ptr->data;
+		if (image)
+			mono_image_close_finish (image);
+	}
+
+	g_list_free (images);
+}
+#endif
+
 void
 mono_image_close_finish (MonoImage *image)
 {
@@ -2723,6 +2795,10 @@ mono_image_close_finish (MonoImage *image)
 	mono_image_close_all (image->files, image->file_count);
 	mono_image_close_all (image->modules, image->module_count);
 
+#ifdef ENABLE_METADATA_UPDATE
+	mono_image_close_all_list (image->delta_image);
+#endif
+
 #ifndef DISABLE_PERFCOUNTERS
 	/* FIXME: use an explicit subtraction method as soon as it's available */
 	mono_atomic_fetch_add_i32 (&mono_perfcounters->loader_bytes, -1 * mono_mempool_get_allocated (image->mempool));
@@ -3425,5 +3501,3 @@ mono_image_append_class_to_reflection_info_set (MonoClass *klass)
 	mono_image_unlock (image);
 }
 
-
-

mono/metadata/loader.c

@@ -31,6 +31,7 @@
 #include <mono/metadata/tokentype.h>
 #include <mono/metadata/tabledefs.h>
 #include <mono/metadata/metadata-internals.h>
+#include <mono/metadata/metadata-update.h>
 #include <mono/metadata/loader.h>
 #include <mono/metadata/loader-internals.h>
 #include <mono/metadata/class-init.h>
@@ -864,7 +865,7 @@ method_from_memberref (MonoImage *image, guint32 idx, MonoGenericContext *typesp
 
 	error_init (error);
 
-	mono_metadata_decode_row (&tables [MONO_TABLE_MEMBERREF], idx-1, cols, 3);
+	mono_metadata_decode_row (&tables [MONO_TABLE_MEMBERREF], idx-1, cols, MONO_MEMBERREF_SIZE);
 	nindex = cols [MONO_MEMBERREF_CLASS] >> MONO_MEMBERREF_PARENT_BITS;
 	class_index = cols [MONO_MEMBERREF_CLASS] & MONO_MEMBERREF_PARENT_MASK;
 	/*g_print ("methodref: 0x%x 0x%x %s\n", class, nindex,
@@ -1073,7 +1074,7 @@ mono_get_method_from_token (MonoImage *image, guint32 token, MonoClass *klass,
 
 	if (used_context) *used_context = FALSE;
 
-	if (idx > image->tables [MONO_TABLE_METHOD].rows) {
+	if (mono_metadata_table_bounds_check (image, MONO_TABLE_METHOD, idx)) {
 		mono_error_set_bad_image (error, image, "Bad method token 0x%08x (out of bounds).", token);
 		return NULL;
 	}
@@ -2017,6 +2018,26 @@ mono_method_has_no_body (MonoMethod *method)
 		(method->flags & METHOD_ATTRIBUTE_PINVOKE_IMPL));
 }
 
+#ifdef ENABLE_METADATA_UPDATE
+static gpointer
+get_method_update_rva (MonoImage *image_base, uint32_t idx)
+{
+	gpointer loc = NULL;
+	uint32_t cur = mono_metadata_update_get_thread_generation ();
+	GList *ptr = image_base->delta_image;
+	/* Go through all the updates that the current thread can see and see
+	 * if they updated the method.  Keep the latest visible update */
+	for (; ptr != NULL; ptr = ptr->next) {
+		MonoImage *image_delta = (MonoImage*) ptr->data;
+		if (image_delta->generation > cur)
+			break;
+		if (image_delta->method_table_update)
+			loc = g_hash_table_lookup (image_delta->method_table_update, GUINT_TO_POINTER (idx));
+	}
+	return loc;
+}
+#endif
+
 // FIXME Replace all internal callers of mono_method_get_header_checked with
 // mono_method_get_header_internal; the difference is in error initialization.
 MonoMethodHeader*
@@ -2025,7 +2046,7 @@ mono_method_get_header_internal (MonoMethod *method, MonoError *error)
 	int idx;
 	guint32 rva;
 	MonoImage* img;
-	gpointer loc;
+	gpointer loc = NULL;
 	MonoGenericContainer *container;
 
 	error_init (error);
@@ -2070,12 +2091,27 @@ mono_method_get_header_internal (MonoMethod *method, MonoError *error)
 	 */
 	g_assert (mono_metadata_token_table (method->token) == MONO_TABLE_METHOD);
 	idx = mono_metadata_token_index (method->token);
-	rva = mono_metadata_decode_row_col (&img->tables [MONO_TABLE_METHOD], idx - 1, MONO_METHOD_RVA);
 
-	if (!mono_verifier_verify_method_header (img, rva, error))
-		return NULL;
+#ifdef ENABLE_METADATA_UPDATE
+	/* EnC case */
+	if (G_UNLIKELY (img->method_table_update)) {
+		/* pre-computed rva pointer into delta IL image */
+		uint32_t gen = GPOINTER_TO_UINT (g_hash_table_lookup (img->method_table_update, GUINT_TO_POINTER (idx)));
+		if (G_UNLIKELY (gen > 0)) {
+			loc = get_method_update_rva (img, idx);
+		}
+	}
+#endif
+
+	if (!loc) {
+		rva = mono_metadata_decode_row_col (&img->tables [MONO_TABLE_METHOD], idx - 1, MONO_METHOD_RVA);
+
+		if (!mono_verifier_verify_method_header (img, rva, error))
+			return NULL;
+
+		loc = mono_image_rva_map (img, rva);
+	}
 
-	loc = mono_image_rva_map (img, rva);
 	if (!loc) {
 		mono_error_set_bad_image (error, img, "Method has zero rva");
 		return NULL;