Improve coreclr test infra

docs/workflow/testing/coreclr/test-configuration.md

@@ -49,8 +49,20 @@ Therefore the managed portion of each test **must not contain**:
     * e.g. `<DisableProjectBuild>true</DisableProjectBuild>`
 * Exclude test from GCStress runs by adding the following to the csproj:
     * `<GCStressIncompatible>true</GCStressIncompatible>`
+* Exclude test from HeapVerify testing runs runs by adding the following to the csproj:
+    * `<HeapVerifyIncompatible>true</HeapVerifyIncompatible>`
 * Exclude test from JIT stress runs runs by adding the following to the csproj:
     * `<JitOptimizationSensitive>true</JitOptimizationSensitive>`
+* Exclude test from NativeAOT runs runs by adding the following to the csproj:
+    * `<NativeAotIncompatible>true</NativeAotIncompatible>`
+* Exclude the test from ilasm round trip testing by adding the following to the csproj
+    * `<IlasmRoundTripIncompatible>true</IlasmRoundTripIncompatible>`
+* Exclude the test for unloadability (collectible assemblies) testing
+    * `<UnloadabilityIncompatible>true</UnloadabilityIncompatible>`
+* If the test is specific for testing crossgen2, and should be compiled as such in all test modes
+    * `<AlwaysUseCrossGen2>true</AlwaysUseCrossGen2>`
+* When `CrossGenTest` is set to false, this test is not run with standard R2R compilation even if running an R2R test pass.
+    * `<CrossGenTest>false</CrossGenTest>`
 * Add NuGet references by updating the following [test project](https://github.com/dotnet/runtime/blob/main/src/tests/Common/test_dependencies/test_dependencies.csproj).
 * Any System.Private.CoreLib types and methods used by tests must be available for building on all platforms.
 This means there must be enough implementation for the C# compiler to find the referenced types and methods. Unsupported target platforms
@@ -94,3 +106,17 @@ should simply `throw new PlatformNotSupportedException()` in its dummy method im
     * CMake reference: `<CMakeProjectReference Include="../NativeDll/CMakeLists.txt" />`
 1. Build the test.
 1. Follow the steps to re-run a failed test to validate the new test.
+
+### Creating a merged test runner project
+1. Use an existing test such as `<repo_root>\src\tests\JIT\Methodical\Methodical_d1.csproj` as a template.
+1. If your new merged test runner has MANY tests in it, and takes too long to run under GC Stress, set `<NumberOfStripesToUseInStress>` to a number such as 10 to make it possible for the test to complete in a reasonable timeframe.
+
+#### Command line arguments for merged test runner projects
+Unless tests are manually run on the command line to repro a problem, these parameters are handled internally by the test infrastructure, but for running tests locally, there are a set of standard parameters that these merged test runners support.
+
+`[testFilterString] [-stripe <whichStripe> <totalStripes>]`
+
+`testFilterString` is any string other that `-stripe`. The only filters supported today are the simple form supported in 'dotnet test --filter' (substrings of the test's fully qualified name).
+
+Either the -stripe <whichStripe> <totalStripes> parameter can be used or the TEST_HARNESS_STRIPE_TO_EXECUTE environment variable may be used to control striping. The TEST_HARNESS_STRIPE_TO_EXECUTE environment variable must be set to a string of the form `.<whichStripe>.<totalStripes>` if it is used. `<whichStripe>` is a 0 based index into the count of stripes, `<totalStripes>` is the total number of stripes.
+

src/tests/Common/CoreCLRTestLibrary/PlatformDetection.cs

@@ -20,5 +20,11 @@ public static class PlatformDetection
                                             && (AppContext.TryGetSwitch("System.Runtime.InteropServices.BuiltInComInterop.IsSupported", out bool isEnabled)
                                                 ? isEnabled
                                                 : true);
+
+        static string _variant = Environment.GetEnvironmentVariable("DOTNET_RUNTIME_VARIANT");
+
+        public static bool IsMonoLLVMAOT => _variant == "llvmaot";
+        public static bool IsMonoLLVMFULLAOT => _variant == "llvmfullaot";
+        public static bool IsMonoInterpreter => _variant == "monointerpreter";
     }
 }

src/tests/Common/XUnitWrapperGenerator/XUnitWrapperGenerator.cs

@@ -154,22 +154,51 @@ private static string GenerateFullTestRunner(ImmutableArray<ITestInfo> testInfos
         builder.AppendLine(string.Join("\n", aliasMap.Values.Where(alias => alias != "global").Select(alias => $"extern alias {alias};")));
         builder.AppendLine("System.Collections.Generic.HashSet<string> testExclusionList = XUnitWrapperLibrary.TestFilter.LoadTestExclusionList();");
 
-        builder.AppendLine("XUnitWrapperLibrary.TestFilter filter = new (args.Length != 0 ? args[0] : null, testExclusionList);");
+        builder.AppendLine("XUnitWrapperLibrary.TestFilter filter = new (args, testExclusionList);");
         builder.AppendLine("XUnitWrapperLibrary.TestSummary summary = new();");
         builder.AppendLine("System.Diagnostics.Stopwatch stopwatch = System.Diagnostics.Stopwatch.StartNew();");
         builder.AppendLine("XUnitWrapperLibrary.TestOutputRecorder outputRecorder = new(System.Console.Out);");
         builder.AppendLine("System.Console.SetOut(outputRecorder);");
 
         ITestReporterWrapper reporter = new WrapperLibraryTestSummaryReporting("summary", "filter", "outputRecorder");
 
-        foreach (ITestInfo test in testInfos)
+        StringBuilder testExecutorBuilder = new();
+        int testsLeftInCurrentTestExecutor = 0;
+        int currentTestExecutor = 0;
+        int totalTestsEmitted = 0;
+
+        if (testInfos.Length > 0)
         {
-            builder.AppendLine(test.GenerateTestExecution(reporter));
+            // Break tests into groups of 50 so that we don't create an unreasonably large main method
+            // Excessively large methods are known to take a long time to compile, and use excessive stack
+            // leading to test failures.
+            foreach (ITestInfo test in testInfos)
+            {
+                if (testsLeftInCurrentTestExecutor == 0)
+                {
+                    if (currentTestExecutor != 0)
+                        testExecutorBuilder.AppendLine("}");
+                    currentTestExecutor++;
+                    testExecutorBuilder.AppendLine($"void TestExecutor{currentTestExecutor}(){{");
+                    builder.AppendLine($"TestExecutor{currentTestExecutor}();");
+                    testsLeftInCurrentTestExecutor = 50; // Break test executors into groups of 50, which empircally seems to work well
+                }
+                testExecutorBuilder.AppendLine(test.GenerateTestExecution(reporter));
+                totalTestsEmitted++;
+                testsLeftInCurrentTestExecutor--;
+            }
+            testExecutorBuilder.AppendLine("}");
         }
 
-        builder.AppendLine($@"System.IO.File.WriteAllText(""{assemblyName}.testResults.xml"", summary.GetTestResultOutput(""{assemblyName}""));");
+        builder.AppendLine($@"string testResults = summary.GetTestResultOutput(""{assemblyName}"");");
+        builder.AppendLine($@"string workitemUploadRoot = System.Environment.GetEnvironmentVariable(""HELIX_WORKITEM_UPLOAD_ROOT"");");
+        builder.AppendLine($@"if (workitemUploadRoot != null) System.IO.File.WriteAllText(System.IO.Path.Combine(workitemUploadRoot, ""{assemblyName}.testResults.xml.txt""), testResults);");
+        builder.AppendLine($@"System.IO.File.WriteAllText(""{assemblyName}.testResults.xml"", testResults);");
         builder.AppendLine("return 100;");
 
+        builder.Append(testExecutorBuilder);
+
+        builder.AppendLine("public static class TestCount { public const int Count = " + totalTestsEmitted.ToString() + "; }");
         return builder.ToString();
     }
 
@@ -193,14 +222,37 @@ private static string GenerateXHarnessTestRunner(ImmutableArray<ITestInfo> testI
 
         ITestReporterWrapper reporter = new WrapperLibraryTestSummaryReporting("summary", "filter", "outputRecorder");
 
-        foreach (ITestInfo test in testInfos)
+        StringBuilder testExecutorBuilder = new();
+        int testsLeftInCurrentTestExecutor = 0;
+        int currentTestExecutor = 0;
+
+        if (testInfos.Length > 0)
         {
-            builder.AppendLine(test.GenerateTestExecution(reporter));
+            // Break tests into groups of 50 so that we don't create an unreasonably large main method
+            // Excessively large methods are known to take a long time to compile, and use excessive stack
+            // leading to test failures.
+            foreach (ITestInfo test in testInfos)
+            {
+                if (testsLeftInCurrentTestExecutor == 0)
+                {
+                    if (currentTestExecutor != 0)
+                        testExecutorBuilder.AppendLine("}");
+                    currentTestExecutor++;
+                    testExecutorBuilder.AppendLine($"static void TestExecutor{currentTestExecutor}(XUnitWrapperLibrary.TestSummary summary, XUnitWrapperLibrary.TestFilter filter, XUnitWrapperLibrary.TestOutputRecorder outputRecorder, System.Diagnostics.Stopwatch stopwatch){{");
+                    builder.AppendLine($"TestExecutor{currentTestExecutor}(summary, filter, outputRecorder, stopwatch);");
+                    testsLeftInCurrentTestExecutor = 50; // Break test executors into groups of 50, which empirically seems to work well
+                }
+                testExecutorBuilder.AppendLine(test.GenerateTestExecution(reporter));
+                testsLeftInCurrentTestExecutor--;
+            }
+            testExecutorBuilder.AppendLine("}");
         }
 
         builder.AppendLine("return summary;");
         builder.AppendLine("}");
 
+        builder.Append(testExecutorBuilder);
+
         return builder.ToString();
     }
 
@@ -354,7 +406,8 @@ private static IEnumerable<ITestInfo> GetTestMethodInfosForMethod(IMethodSymbol
                             testInfos,
                             conditionType,
                             conditionMembers,
-                            aliasMap[conditionType.ContainingAssembly.MetadataName]);
+                            aliasMap[conditionType.ContainingAssembly.MetadataName],
+                            false /* do not negate the condition, as this attribute indicates that a test will be run */);
                         break;
                     }
                 case "Xunit.OuterloopAttribute":
@@ -373,7 +426,8 @@ private static IEnumerable<ITestInfo> GetTestMethodInfosForMethod(IMethodSymbol
                             testInfos,
                             conditionType,
                             filterAttribute.ConstructorArguments[2].Values,
-                            aliasMap[conditionType.ContainingAssembly.MetadataName]);
+                            aliasMap[conditionType.ContainingAssembly.MetadataName],
+                            true /* negate the condition, as this attribute indicates that a test will NOT be run */);
                         break;
                     }
                     else if (filterAttribute.AttributeConstructor.Parameters.Length == 4)
@@ -667,9 +721,12 @@ private static ImmutableArray<ITestInfo> DecorateWithUserDefinedCondition(
         ImmutableArray<ITestInfo> testInfos,
         ITypeSymbol conditionType,
         ImmutableArray<TypedConstant> values,
-        string externAlias)
+        string externAlias,
+        bool negate)
     {
         string condition = string.Join("&&", values.Select(v => $"{externAlias}::{conditionType.ToDisplayString(FullyQualifiedWithoutGlobalNamespace)}.{v.Value}"));
+        if (negate)
+            condition = $"!({condition})";
         return ImmutableArray.CreateRange<ITestInfo>(testInfos.Select(m => new ConditionalTest(m, condition)));
     }
 

src/tests/Common/XUnitWrapperLibrary/TestFilter.cs

@@ -118,9 +118,45 @@ public override string ToString()
     // issues.targets file as a split model would be very confusing for developers
     // and test monitors.
     private readonly HashSet<string>? _testExclusionList;
+    private readonly int _stripe = 0;
+    private readonly int _stripeCount = 1;
+    private int _shouldRunQuery = -1;
 
-    public TestFilter(string? filterString, HashSet<string>? testExclusionList)
+    public TestFilter(string? filterString, HashSet<string>? testExclusionList) : 
+        this(filterString == null ? Array.Empty<string>() : new string[]{filterString}, testExclusionList)
     {
+    }
+
+    public TestFilter(string[] filterArgs, HashSet<string>? testExclusionList)
+    {
+        string? filterString = null;
+
+        for (int i = 0; i < filterArgs.Length; i++)
+        {
+            if (filterArgs[i].StartsWith("-stripe"))
+            {
+                _stripe = Int32.Parse(filterArgs[++i]);
+                _stripeCount = Int32.Parse(filterArgs[++i]);
+            }
+            else
+            {
+                if (filterString == null)
+                    filterString = filterArgs[0];
+            }
+        }
+
+        var stripeEnvironment = Environment.GetEnvironmentVariable("TEST_HARNESS_STRIPE_TO_EXECUTE");
+        if (!String.IsNullOrEmpty(stripeEnvironment) && stripeEnvironment != ".0.1")
+        {
+            var stripes = stripeEnvironment.Split('.');
+            if (stripes.Length == 3)
+            {
+                Console.WriteLine($"Test striping enabled via TEST_HARNESS_STRIPE_TO_EXECUTE environment variable set to '{stripeEnvironment}'");
+                _stripe = Int32.Parse(stripes[1]);
+                _stripeCount = Int32.Parse(stripes[2]);
+            }
+        }
+
         if (filterString is not null)
         {
             if (filterString.IndexOfAny(new[] { '!', '(', ')', '~', '=' }) != -1)
@@ -140,15 +176,26 @@ public TestFilter(ISearchClause? filter, HashSet<string>? testExclusionList)
 
     public bool ShouldRunTest(string fullyQualifiedName, string displayName, string[]? traits = null)
     {
+        bool shouldRun = false;
         if (_testExclusionList is not null && _testExclusionList.Contains(displayName.Replace("\\", "/")))
         {
-            return false;
+            shouldRun = false;
         }
-        if (_filter is null)
+        else if (_filter is null)
+        {
+            shouldRun = true;
+        }
+        else
+        {
+            shouldRun = _filter.IsMatch(fullyQualifiedName, displayName, traits ?? Array.Empty<string>());
+        }
+
+        if (shouldRun)
         {
-            return true;
+            // Test stripe, if true, then report success
+            return ((System.Threading.Interlocked.Increment(ref _shouldRunQuery)) % _stripeCount) == _stripe;
         }
-        return _filter.IsMatch(fullyQualifiedName, displayName, traits ?? Array.Empty<string>());
+        return false;
     }
 
     public static HashSet<string> LoadTestExclusionList()

src/tests/Common/XUnitWrapperLibrary/TestSummary.cs

@@ -73,11 +73,29 @@ public string GetTestResultOutput(string assemblyName)
             string outputElement = !string.IsNullOrWhiteSpace(test.Output) ? $"<output><![CDATA[{test.Output}]]></output>" : string.Empty;
             if (test.Exception is not null)
             {
-                resultsFile.AppendLine($@"result=""Fail""><failure exception-type=""{test.Exception.GetType()}""><message><![CDATA[{test.Exception.Message}]]></message><stack-trace><![CDATA[{test.Exception.StackTrace}]]></stack-trace></failure>{outputElement}</test>");
+                string exceptionMessage = test.Exception.Message;
+                if (test.Exception is System.Reflection.TargetInvocationException tie)
+                {
+                    if (tie.InnerException != null)
+                    {
+                        exceptionMessage = $"{exceptionMessage} \n INNER EXCEPTION--\n {tie.InnerException.GetType()}--\n{tie.InnerException.Message}--\n{tie.InnerException.StackTrace}";
+                    }
+                }
+                if (string.IsNullOrWhiteSpace(exceptionMessage))
+                {
+                    exceptionMessage = "NoExceptionMessage";
+                }
+
+                string? stackTrace = test.Exception.StackTrace;
+                if (string.IsNullOrWhiteSpace(stackTrace))
+                {
+                    stackTrace = "NoStackTrace";
+                }
+                resultsFile.AppendLine($@"result=""Fail""><failure exception-type=""{test.Exception.GetType()}""><message><![CDATA[{exceptionMessage}]]></message><stack-trace><![CDATA[{stackTrace}]]></stack-trace></failure>{outputElement}</test>");
             }
             else if (test.SkipReason is not null)
             {
-                resultsFile.AppendLine($@"result=""Skip""><reason><![CDATA[{test.SkipReason}]]></reason></test>");
+                resultsFile.AppendLine($@"result=""Skip""><reason><![CDATA[{(!string.IsNullOrWhiteSpace(test.SkipReason) ? test.SkipReason : "No Known Skip Reason")}]]></reason></test>");
             }
             else
             {