[release/8.0] Fix a possible infinite wait for GC completion at proce…

…ss shutdown. (#107844)

* Add a test

* Minimal part of "Use RtlDllShutdownInProgress to detect process shutdown on Windows (#103877)"

Fixes:107800

* Use RtlDllShutdownInProgress to detect process shutdown on Windows

Switching to cooperative mode is not safe during process shutdown on
Windows. Process shutdown can terminate a thread in the middle of the
GC. The shutdown thread deadlocks if it tries to switch to cooperative
mode and wait for the GC to finish in this situation.

Use RtlDllShutdownInProgress Windows API to detect process
shutdown to avoid waiting for GC completion when that may lead to deadlocks.

* Update src/coreclr/vm/vars.hpp

Co-authored-by: Manish Godse <61718172+mangod9@users.noreply.github.com>

---------

Co-authored-by: Jan Kotas <jkotas@microsoft.com>
Co-authored-by: Manish Godse <61718172+mangod9@users.noreply.github.com>

src/coreclr/vm/ceemain.cpp

@@ -392,6 +392,12 @@ BOOL g_singleVersionHosting;
 typedef BOOL(WINAPI* PINITIALIZECONTEXT2)(PVOID Buffer, DWORD ContextFlags, PCONTEXT* Context, PDWORD ContextLength, ULONG64 XStateCompactionMask);
 PINITIALIZECONTEXT2 g_pfnInitializeContext2 = NULL;
 
+static BOOLEAN WINAPI RtlDllShutdownInProgressFallback()
+{
+    return g_fProcessDetach;
+}
+PRTLDLLSHUTDOWNINPROGRESS g_pfnRtlDllShutdownInProgress = &RtlDllShutdownInProgressFallback;
+
 #ifdef TARGET_X86
 typedef VOID(__cdecl* PRTLRESTORECONTEXT)(PCONTEXT ContextRecord, struct _EXCEPTION_RECORD* ExceptionRecord);
 PRTLRESTORECONTEXT g_pfnRtlRestoreContext = NULL;
@@ -402,8 +408,12 @@ void InitializeOptionalWindowsAPIPointers()
     HMODULE hm = GetModuleHandleW(_T("kernel32.dll"));
     g_pfnInitializeContext2 = (PINITIALIZECONTEXT2)GetProcAddress(hm, "InitializeContext2");
 
-#ifdef TARGET_X86
     hm = GetModuleHandleW(_T("ntdll.dll"));
+    PRTLDLLSHUTDOWNINPROGRESS pfn = (PRTLDLLSHUTDOWNINPROGRESS)GetProcAddress(hm, "RtlDllShutdownInProgress");
+    if (pfn != NULL)
+        g_pfnRtlDllShutdownInProgress = pfn;
+
+#ifdef TARGET_X86
     g_pfnRtlRestoreContext = (PRTLRESTORECONTEXT)GetProcAddress(hm, "RtlRestoreContext");
 #endif //TARGET_X86
 }

src/coreclr/vm/vars.hpp

@@ -478,12 +478,24 @@ GVAL_DECL(bool, g_metadataUpdatesApplied);
 #endif
 EXTERN bool g_fManagedAttach;
 
+#ifdef HOST_WINDOWS
+typedef BOOLEAN (WINAPI* PRTLDLLSHUTDOWNINPROGRESS)();
+EXTERN PRTLDLLSHUTDOWNINPROGRESS g_pfnRtlDllShutdownInProgress;
+#endif
+
 // Indicates whether we're executing shut down as a result of DllMain
 // (DLL_PROCESS_DETACH). See comments at code:EEShutDown for details.
-inline BOOL    IsAtProcessExit()
+inline bool IsAtProcessExit()
 {
     SUPPORTS_DAC;
+#if defined(DACCESS_COMPILE) || !defined(HOST_WINDOWS)
     return g_fProcessDetach;
+#else
+    // RtlDllShutdownInProgress provides more accurate information about whether the process is shutting down.
+    // Use it if it is available to avoid shutdown deadlocks.
+    // https://learn.microsoft.com/windows/win32/devnotes/rtldllshutdowninprogress
+    return g_pfnRtlDllShutdownInProgress();
+#endif
 }
 
 enum FWStatus

src/tests/Regressions/coreclr/GitHub_107800/test107800.cs

@@ -0,0 +1,25 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+
+public class Program
+{
+    public static int Main()
+    {
+        Task.Run(() =>
+        {
+            for (; ; )
+            {
+                GC.Collect();
+            }
+        });
+
+        Thread.Sleep(10);
+        Environment.Exit(100);
+
+        throw new Exception("UNREACHABLE");
+    }
+}

src/tests/Regressions/coreclr/GitHub_107800/test107800.csproj

@@ -0,0 +1,11 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <!-- Calls ExitProcess -->
+    <RequiresProcessIsolation>true</RequiresProcessIsolation>
+    <CLRTestPriority>0</CLRTestPriority>
+    <OutputType>Exe</OutputType>
+  </PropertyGroup>
+  <ItemGroup>
+    <Compile Include="test107800.cs" />
+  </ItemGroup>
+</Project>