From cacb3bf4f73e25b20a2032691ff71214f1d16c6b Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Mon, 21 Feb 2022 16:58:09 -0500 Subject: [PATCH] [wasm] Add support for a random test case orderer, for xunit tests (#65628) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [wasm] Add support for a random test case orderer, for xunit tests This is enabled by default for wasm with `$(XUnitUseRandomizedTestOrderer)=true`. When the library tests run, they print two messages like: ``` info: Using random seed for test cases: 700149826 info: Using random seed for collections: 700149826 info: Starting: System.Collections.Immutable.Tests.dll ``` These seeds are picked randomly every time the tests are run. To run the tests with a specific seed, use environment variable `XUNIT_RANDOM_ORDER_SEED`. When running with tests, this can be used as: `WasmXHarnessMonoArgs="--setenv=XUNIT_RANDOM_ORDER_SEED=` * Enable test orderer only for libraries tests * [wasm] Automatically pass XUNIT_RANDOM_ORDER_SEED envvar to wasm apps * Disable random test ordering for `System.Xml.RW.XmlWriterApi.Tests` `System.Xml.Tests.TCCloseOutput.*` tests seem to depend on the order of execution. ``` [06:35:14] fail: [FAIL] System.Xml.Tests.TCCloseOutput.CloseOutput_4(utils: XmlWriterUtils { Async = False, WriterType = UTF8Writer }, outputType: "Stream") [06:35:14] info: System.IO.FileNotFoundException : File Not Found: writer.out [06:35:14] info: at XmlCoreTest.Common.FilePathUtil.getStream(String filename) [06:35:14] info: at System.Xml.Tests.TCCloseOutput.CloseOutput_4(XmlWriterUtils utils, String outputType) [06:35:14] info: at System.Reflection.RuntimeMethodInfo.InvokeWorker(Object obj, BindingFlags invokeAttr, Span`1 parameters) ``` * [wasm] Disable random tests ordering for `System.Runtime.Loader` It seems to cause failures like, reproducible with `XUNIT_RANDOM_ORDER_SEED=2106784294`: ``` [06:25:43] fail: [FAIL] System.Runtime.Loader.Tests.SatelliteAssembliesTests.SatelliteLoadsCorrectly_FromName(alc: "Empty", assemblyName: "System.Runtime.Loader.Tests", culture: "en") [06:25:43] info: Assert.Same() Failure [06:25:43] info: Expected: "Default" System.Runtime.Loader.DefaultAssemblyLoadContext #0 [06:25:43] info: Actual: "Empty" System.Runtime.Loader.AssemblyLoadContext #4 [06:25:43] info: at System.Runtime.Loader.Tests.SatelliteAssembliesTests.SatelliteLoadsCorrectly_FromName(String alc, String assemblyName, String culture) [06:25:43] info: at System.Reflection.RuntimeMethodInfo.InvokeWorker(Object obj, BindingFlags invokeAttr, Span`1 parameters) ``` * [wasm] Disable random test ordering for `System.Runtime.Numerics` Randomized runs seem to fail with: ``` [03:29:54] info: Starting: System.Runtime.Numerics.Tests.dll [03:29:58] fail: [FAIL] System.Numerics.Tests.cast_toTest.RunDoubleExplicitCastToBigIntegerTests [03:29:58] info: Assert.Equal() Failure [03:29:58] info: ↓ (pos 0) [03:29:58] info: Expected: -0 [03:29:58] info: Actual: 0 [03:29:58] info: ↑ (pos 0) [03:29:58] info: at System.Numerics.Tests.cast_toTest.VerifyDoubleExplicitCastToBigInteger(Double value) [03:29:58] info: at System.Numerics.Tests.cast_toTest.RunDoubleExplicitCastToBigIntegerTests() [03:29:58] info: at System.Reflection.RuntimeMethodInfo.InvokeWorker(Object obj, BindingFlags invokeAttr, Span`1 parameters) ``` Reproducible with `XUNIT_RANDOM_SEED_ORDER=1883302047`. * Add issues for the failing tests --- eng/testing/WasmRunnerTemplate.cmd | 4 + eng/testing/WasmRunnerTemplate.sh | 7 +- eng/testing/tests.mobile.targets | 4 + eng/testing/tests.wasm.targets | 1 + .../TestUtilities/RandomTestCaseOrderer.cs | 77 +++++++++++++++++++ .../RandomTestCollectionOrderer.cs | 29 +++++++ .../tests/TestUtilities/TestUtilities.csproj | 3 + .../Tests/RandomizedTestOrderAssemblyInfo.cs | 7 ++ .../System.Xml.RW.XmlWriterApi.Tests.csproj | 1 + .../tests/System.Runtime.Loader.Tests.csproj | 3 + .../System.Runtime.Numerics.Tests.csproj | 5 +- 11 files changed, 138 insertions(+), 3 deletions(-) create mode 100644 src/libraries/Common/tests/TestUtilities/RandomTestCaseOrderer.cs create mode 100644 src/libraries/Common/tests/TestUtilities/RandomTestCollectionOrderer.cs create mode 100644 src/libraries/Common/tests/Tests/RandomizedTestOrderAssemblyInfo.cs diff --git a/eng/testing/WasmRunnerTemplate.cmd b/eng/testing/WasmRunnerTemplate.cmd index a668ce2c6a464..026a06066c318 100644 --- a/eng/testing/WasmRunnerTemplate.cmd +++ b/eng/testing/WasmRunnerTemplate.cmd @@ -57,6 +57,10 @@ if [%XHARNESS_ARGS%] == [] ( set "XHARNESS_ARGS=%JS_ENGINE% %JS_ENGINE_ARGS% %BROWSER_PATH% %MAIN_JS%" ) +if [%XUNIT_RANDOM_ORDER_SEED%] NEQ [] ( + set "WasmXHarnessMonoArgs=%WasmXHarnessMonoArgs% --setenv=XUNIT_RANDOM_ORDER_SEED=%XUNIT_RANDOM_ORDER_SEED%" +) + echo EXECUTION_DIR=%EXECUTION_DIR% echo SCENARIO=%SCENARIO% echo XHARNESS_OUT=%XHARNESS_OUT% diff --git a/eng/testing/WasmRunnerTemplate.sh b/eng/testing/WasmRunnerTemplate.sh index ae78d5304e8ae..6c054ba9d23e1 100644 --- a/eng/testing/WasmRunnerTemplate.sh +++ b/eng/testing/WasmRunnerTemplate.sh @@ -53,6 +53,10 @@ if [[ -z "$XHARNESS_ARGS" ]]; then XHARNESS_ARGS="$JS_ENGINE $JS_ENGINE_ARGS $MAIN_JS" fi +if [[ -n "$XUNIT_RANDOM_ORDER_SEED" ]]; then + WasmXHarnessMonoArgs="${WasmXHarnessMonoArgs} --setenv=XUNIT_RANDOM_ORDER_SEED=${XUNIT_RANDOM_ORDER_SEED}" +fi + echo EXECUTION_DIR=$EXECUTION_DIR echo SCENARIO=$SCENARIO echo XHARNESS_OUT=$XHARNESS_OUT @@ -64,7 +68,6 @@ echo JS_ENGINE=$JS_ENGINE echo JS_ENGINE_ARGS=$JS_ENGINE_ARGS echo XHARNESS_ARGS=$XHARNESS_ARGS - pushd $EXECUTION_DIR # ========================= BEGIN Test Execution ============================= @@ -83,4 +86,4 @@ echo ----- end $(date) ----- exit code $_exitCode ------------------------------ echo "XHarness artifacts: $XHARNESS_OUT" -exit $_exitCode \ No newline at end of file +exit $_exitCode diff --git a/eng/testing/tests.mobile.targets b/eng/testing/tests.mobile.targets index da8da1d2f4bfa..837dc9716536a 100644 --- a/eng/testing/tests.mobile.targets +++ b/eng/testing/tests.mobile.targets @@ -51,6 +51,10 @@ $(AdditionalXHarnessArguments) -- -c=$(XUnitClassName) + + + + diff --git a/eng/testing/tests.wasm.targets b/eng/testing/tests.wasm.targets index 4e3addd55acb2..6b3a1d61d7eae 100644 --- a/eng/testing/tests.wasm.targets +++ b/eng/testing/tests.wasm.targets @@ -15,6 +15,7 @@ <_ShellCommandSeparator Condition="'$(OS)' != 'Windows_NT'">&& false <_WasmMainJSFileName Condition="'$(WasmMainJSPath)' != ''">$([System.IO.Path]::GetFileName('$(WasmMainJSPath)')) + true diff --git a/src/libraries/Common/tests/TestUtilities/RandomTestCaseOrderer.cs b/src/libraries/Common/tests/TestUtilities/RandomTestCaseOrderer.cs new file mode 100644 index 0000000000000..effe3a08e1404 --- /dev/null +++ b/src/libraries/Common/tests/TestUtilities/RandomTestCaseOrderer.cs @@ -0,0 +1,77 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Linq; +using System.Threading; +using Xunit.Abstractions; +using Xunit.Sdk; + +#nullable enable + +namespace TestUtilities; + +// Based on https://github.com/xunit/xunit/blob/v2/src/xunit.execution/Sdk/DefaultTestCaseOrderer.cs + +public class RandomTestCaseOrderer : ITestCaseOrderer +{ + public const string RandomSeedEnvironmentVariableName = "XUNIT_RANDOM_ORDER_SEED"; + + public static readonly Lazy LazySeed = new (GetSeed, LazyThreadSafetyMode.ExecutionAndPublication); + private readonly IMessageSink _diagnosticMessageSink; + + private static int GetSeed() + { + string? seedEnvVar = Environment.GetEnvironmentVariable(RandomSeedEnvironmentVariableName); + if (string.IsNullOrEmpty(seedEnvVar) || !int.TryParse(seedEnvVar, out int seed)) + { + seed = new Random().Next(); + } + + return seed; + } + + public RandomTestCaseOrderer(IMessageSink diagnosticMessageSink) + { + diagnosticMessageSink.OnMessage(new DiagnosticMessage($"Using random seed for test cases: {LazySeed.Value}")); + _diagnosticMessageSink = diagnosticMessageSink; + } + + public IEnumerable OrderTestCases(IEnumerable testCases) where TTestCase : ITestCase + => TryRandomize(testCases.ToList(), _diagnosticMessageSink, out List? randomizedTests) + ? randomizedTests + : testCases; + + public static bool TryRandomize(List tests, IMessageSink messageSink, [NotNullWhen(true)] out List? randomizedTests) + { + randomizedTests = null; + try + { + randomizedTests = Randomize(tests.ToList()); + return true; + } + catch (Exception ex) + { + messageSink.OnMessage(new DiagnosticMessage($"Failed to randomize test cases: {ex}")); + return false; + } + + static List Randomize(List tests) + { + var result = new List(tests.Count); + + var randomizer = new Random(LazySeed.Value); + + while (tests.Count > 0) + { + int next = randomizer.Next(tests.Count); + result.Add(tests[next]); + tests.RemoveAt(next); + } + + return result; + } + } +} diff --git a/src/libraries/Common/tests/TestUtilities/RandomTestCollectionOrderer.cs b/src/libraries/Common/tests/TestUtilities/RandomTestCollectionOrderer.cs new file mode 100644 index 0000000000000..758fef8039755 --- /dev/null +++ b/src/libraries/Common/tests/TestUtilities/RandomTestCollectionOrderer.cs @@ -0,0 +1,29 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Linq; +using Xunit; +using Xunit.Abstractions; +using Xunit.Sdk; + +#nullable enable + +namespace TestUtilities; + +public class RandomTestCollectionOrderer : ITestCollectionOrderer +{ + private readonly IMessageSink _diagnosticMessageSink; + + public RandomTestCollectionOrderer(IMessageSink diagnosticMessageSink) + { + diagnosticMessageSink.OnMessage(new DiagnosticMessage( + $"Using random seed for collections: {RandomTestCaseOrderer.LazySeed.Value}")); + _diagnosticMessageSink = diagnosticMessageSink; + } + + public IEnumerable OrderTestCollections(IEnumerable testCollections) + => RandomTestCaseOrderer.TryRandomize(testCollections.ToList(), _diagnosticMessageSink, out List? randomizedTests) + ? randomizedTests + : testCollections; +} diff --git a/src/libraries/Common/tests/TestUtilities/TestUtilities.csproj b/src/libraries/Common/tests/TestUtilities/TestUtilities.csproj index ecbb20d8ee57c..b24afba9d9e70 100644 --- a/src/libraries/Common/tests/TestUtilities/TestUtilities.csproj +++ b/src/libraries/Common/tests/TestUtilities/TestUtilities.csproj @@ -43,6 +43,9 @@ variant from the Common folder and adding the missing members manually. --> + + + diff --git a/src/libraries/Common/tests/Tests/RandomizedTestOrderAssemblyInfo.cs b/src/libraries/Common/tests/Tests/RandomizedTestOrderAssemblyInfo.cs new file mode 100644 index 0000000000000..3b051bad65bc8 --- /dev/null +++ b/src/libraries/Common/tests/Tests/RandomizedTestOrderAssemblyInfo.cs @@ -0,0 +1,7 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Xunit; + +[assembly: TestCaseOrderer("TestUtilities.RandomTestCaseOrderer", "TestUtilities")] +[assembly: TestCollectionOrderer("TestUtilities.RandomTestCollectionOrderer", "TestUtilities")] diff --git a/src/libraries/System.Private.Xml/tests/Writers/XmlWriterApi/System.Xml.RW.XmlWriterApi.Tests.csproj b/src/libraries/System.Private.Xml/tests/Writers/XmlWriterApi/System.Xml.RW.XmlWriterApi.Tests.csproj index 9569073b4185a..7ac73f30358cf 100644 --- a/src/libraries/System.Private.Xml/tests/Writers/XmlWriterApi/System.Xml.RW.XmlWriterApi.Tests.csproj +++ b/src/libraries/System.Private.Xml/tests/Writers/XmlWriterApi/System.Xml.RW.XmlWriterApi.Tests.csproj @@ -1,6 +1,7 @@ $(NetCoreAppCurrent) + false diff --git a/src/libraries/System.Runtime.Loader/tests/System.Runtime.Loader.Tests.csproj b/src/libraries/System.Runtime.Loader/tests/System.Runtime.Loader.Tests.csproj index e21bc7e673f5f..de19f64769380 100644 --- a/src/libraries/System.Runtime.Loader/tests/System.Runtime.Loader.Tests.csproj +++ b/src/libraries/System.Runtime.Loader/tests/System.Runtime.Loader.Tests.csproj @@ -8,6 +8,9 @@ false --setenv=DOTNET_MODIFIABLE_ASSEMBLIES=debug + + + false diff --git a/src/libraries/System.Runtime.Numerics/tests/System.Runtime.Numerics.Tests.csproj b/src/libraries/System.Runtime.Numerics/tests/System.Runtime.Numerics.Tests.csproj index 5dd112190af3c..976ae7909313e 100644 --- a/src/libraries/System.Runtime.Numerics/tests/System.Runtime.Numerics.Tests.csproj +++ b/src/libraries/System.Runtime.Numerics/tests/System.Runtime.Numerics.Tests.csproj @@ -2,6 +2,9 @@ true $(NetCoreAppCurrent) + + + false @@ -51,4 +54,4 @@ - \ No newline at end of file +