From 6e1585f375b41e1486420f264b0704d5f24d3694 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Wed, 30 Mar 2022 09:59:31 +0200 Subject: [PATCH] Add NativeAOT (and CoreRT) support for micro benchmarks (#2328) * prefer AppContext.BaseDirectory over typeof(X).Assembly.Location * GetReferencedAssemblies() is not supported on AOT * Arrays with non-zero lower bounds are not supported * BinaryFormatter serialization and deserialization are disabled within this application. * Dynamic code generation is not supported on this platform. * disable AddRemoveFromDifferentThreads as it hangs * disable more serialization benchmarks * disable Activator.CreateInstance(_assemblyName, _typeName) benchmark as it's not supported * add NativeAOT and CoreRT support to the python scripts * bump BDN version * address code review feedback, don't use the CoreRT name anymore * add NativeAOT CI leg * increase build timeout --- azure-pipelines.yml | 19 +++++++++++++++++++ eng/performance/benchmark_jobs.yml | 10 ++++++++-- scripts/channel_map.py | 10 ++++++++++ scripts/dotnet.py | 19 ++++++++++++------- scripts/micro_benchmarks.py | 4 +++- src/benchmarks/micro/Program.cs | 2 +- .../micro/Serializers/Binary_FromStream.cs | 2 ++ .../micro/Serializers/Binary_ToStream.cs | 2 ++ .../micro/Serializers/Json_FromStream.cs | 2 ++ .../micro/Serializers/Json_FromString.cs | 2 ++ .../micro/Serializers/Json_ToStream.cs | 2 ++ .../micro/Serializers/Json_ToString.cs | 2 ++ .../micro/Serializers/Xml_FromStream.cs | 2 ++ .../micro/Serializers/Xml_ToStream.cs | 2 ++ .../AddRemoveFromDifferentThreads.cs | 2 ++ .../System.IO.Compression/CompressedFile.cs | 2 +- .../Configuration.Certificates.cs | 2 +- .../Perf.BinaryFormatter.cs | 2 ++ .../Serializer/ColdStartSerialization.cs | 2 ++ .../System.Text.Json/Serializer/ReadJson.cs | 2 ++ .../ReadMissingAndCaseInsensitive.cs | 2 ++ .../Serializer/ReadPreservedReferences.cs | 1 + .../System.Text.Json/Serializer/WriteJson.cs | 2 ++ .../Serializer/WritePreservedReferences.cs | 1 + .../Perf.Regex.Industry.cs | 2 +- .../BenchmarksGame/Inputs/InputFileHelper.cs | 4 ++-- .../Benchstones/MDBenchI/MDGeneralArray.cs | 2 ++ .../micro/runtime/Roslyn/CscBench.cs | 2 ++ .../runtime/System.Reflection/Activator.cs | 2 ++ .../RecommendedConfig.cs | 10 +++++++--- 30 files changed, 101 insertions(+), 19 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 2419110754d..8eeceadcf78 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -2,6 +2,8 @@ resources: containers: - container: ubuntu_x64_build_container image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-20210311173856-047508b + - container: centos_x64_build_container + image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7-20220107135107-9b5bbc2 # CI Trigger on main branch trigger: @@ -233,6 +235,23 @@ jobs: channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks - main + # Ubuntu 1804 x64 NativeAOT micro benchmarks + - template: /eng/performance/benchmark_jobs.yml + parameters: + osName: centos + osVersion: 7 + kind: micro + architecture: x64 + pool: + vmImage: ubuntu-latest + machinePool: Open + queue: centos.7.amd64.open.rt + container: centos_x64_build_container + csproj: src/benchmarks/micro/MicroBenchmarks.csproj + runCategories: 'runtime libraries' + channels: + - nativeaot7.0 + # Ubuntu 1804 x64 ML.NET benchmarks - template: /eng/performance/benchmark_jobs.yml parameters: diff --git a/eng/performance/benchmark_jobs.yml b/eng/performance/benchmark_jobs.yml index 55d57336062..ba04a30034b 100644 --- a/eng/performance/benchmark_jobs.yml +++ b/eng/performance/benchmark_jobs.yml @@ -58,6 +58,12 @@ jobs: - ${{ if or(eq(parameters.osVersion, '1604'), eq(parameters.architecture, 'arm64'))}}: - name: ArtifactsDirectory value: '$HELIX_WORKITEM_PAYLOAD/artifacts/BenchmarkDotNet.Artifacts' + - ${{ if eq(parameters.osName, 'ubuntu')}}: + - name: InstallPrerequisites + value: 'sudo apt-get -y install python3-venv' + - ${{ if eq(parameters.osName, 'centos')}}: + - name: InstallPrerequisites + value: 'sudo yum -y install clang zlib-devel krb5-libs krb5-devel' - ${{ if eq(variables['System.TeamProject'], 'public') }}: # for public runs, we do not want to upload perflab data - name: PerfLabArguments @@ -70,7 +76,7 @@ jobs: - name: HelixApiAccessToken value: '' - name: HelixPreCommand - value: '$(PreservePythonPath);$(AdditionalHelixPreCommands)' + value: '$(PreservePythonPath);$(AdditionalHelixPreCommands);$(InstallPrerequisites);' - name: HelixSourcePrefix value: "pr" - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: @@ -89,7 +95,7 @@ jobs: value: '$(PreservePythonPath);py -3 -m venv %HELIX_WORKITEM_PAYLOAD%\.venv;call %HELIX_WORKITEM_PAYLOAD%\.venv\Scripts\activate.bat;set PYTHONPATH=;py -3 -m pip install azure.storage.blob==12.0.0 --force-reinstall;py -3 -m pip install azure.storage.queue==12.0.0 --force-reinstall;set "PERFLAB_UPLOAD_TOKEN=$(PerfCommandUploadToken)";$(AdditionalHelixPreCommands)' - ${{ if ne(parameters.osName, 'windows') }}: - name: HelixPreCommand - value: '$(PreservePythonPath);$(SetAllowOpenSsl102);sudo apt-get -y install python3-venv;python3 -m venv $HELIX_WORKITEM_PAYLOAD/.venv;. $HELIX_WORKITEM_PAYLOAD/.venv/bin/activate;export PYTHONPATH=;python3 -m pip install -U pip;pip3 install azure.storage.blob==12.0.0 --force-reinstall;pip3 install azure.storage.queue==12.0.0 --force-reinstall;export PERFLAB_UPLOAD_TOKEN="$(PerfCommandUploadTokenLinux)";$(AdditionalHelixPreCommands)' + value: '$(PreservePythonPath);$(SetAllowOpenSsl102);$(InstallPrerequisites);python3 -m venv $HELIX_WORKITEM_PAYLOAD/.venv;. $HELIX_WORKITEM_PAYLOAD/.venv/bin/activate;export PYTHONPATH=;python3 -m pip install -U pip;pip3 install azure.storage.blob==12.0.0 --force-reinstall;pip3 install azure.storage.queue==12.0.0 --force-reinstall;export PERFLAB_UPLOAD_TOKEN="$(PerfCommandUploadTokenLinux)";$(AdditionalHelixPreCommands)' - group: DotNet-HelixApi-Access # perflab upload tokens still exist in this variable group - group: dotnet-benchview diff --git a/scripts/channel_map.py b/scripts/channel_map.py index aeab5521553..02a39daff8a 100644 --- a/scripts/channel_map.py +++ b/scripts/channel_map.py @@ -37,10 +37,20 @@ class ChannelMap(): 'branch': '7.0.1xx', 'quality': 'daily' }, + 'nativeaot7.0': { + 'tfm': 'nativeaot7.0', + 'branch': '7.0.1xx', + 'quality': 'daily' + }, 'master': { 'tfm': 'net6.0', 'branch': 'master' }, + 'nativeaot6.0': { + 'tfm': 'nativeaot6.0', + 'branch': '6.0.1xx', + 'quality': 'daily' + }, '5.0':{ 'tfm': 'net5.0', 'branch': '5.0.4xx', diff --git a/scripts/dotnet.py b/scripts/dotnet.py index 3f21ad0bcbc..2c27250f7a7 100755 --- a/scripts/dotnet.py +++ b/scripts/dotnet.py @@ -78,24 +78,29 @@ def __call__(self, parser, namespace, values, option_string=None): def get_target_framework_moniker(framework: str) -> str: ''' Translates framework name to target framework moniker (TFM) - To run CoreRT benchmarks we need to run the host BDN process as latest - .NET Core the host process will build and run CoreRT benchmarks + To run NativeAOT benchmarks we need to run the host BDN process as latest + .NET the host process will build and run AOT benchmarks ''' - return ChannelMap.get_target_framework_moniker("main") if framework == 'corert' else framework + if framework == 'nativeaot6.0': + return 'net6.0' + if framework == 'nativeaot7.0': + return 'net7.0' + else: + return framework @staticmethod def get_target_framework_monikers(frameworks: list) -> list: ''' Translates framework names to target framework monikers (TFM) - Required to run CoreRT benchmarks where the host process must be .NET - Core, not CoreRT. + Required to run AOT benchmarks where the host process must be .NET + , not NativeAOT. ''' monikers = [ FrameworkAction.get_target_framework_moniker(framework) for framework in frameworks ] - # ['net5.0', 'corert'] should become ['net5.0'] + # ['net6.0', 'nativeaot6.0'] should become ['net6.0'] return list(set(monikers)) class VersionsAction(Action): @@ -136,7 +141,7 @@ class CompilationAction(Action): For PRs it is recommended to kick off a Tiered run, and being able to manually kick-off NoTiering and FullyJittedNoTiering modes when needed. ''' - # TODO: Would 'Default' make sense for .NET Framework / CoreRT / Mono? + # TODO: Would 'Default' make sense for .NET Framework / NativeAOT / Mono? # TODO: Should only be required for benchmark execution under certain tools TIERED = 'Tiered' diff --git a/scripts/micro_benchmarks.py b/scripts/micro_benchmarks.py index b27e57e22ab..4f2ec567331 100755 --- a/scripts/micro_benchmarks.py +++ b/scripts/micro_benchmarks.py @@ -255,8 +255,10 @@ def __get_benchmarkdotnet_arguments(framework: str, args: tuple) -> list: # if we don't it's gonna restore to default global folder run_args += ['--packages', get_packages_directory()] - # Required for CoreRT where: + # Required for WASM and NativeAOT where: # host process framework != benchmark process framework + if framework.startswith("nativeaot"): + run_args += ['--runtimes', framework] if args.wasm: if framework == "net5.0" or framework == "net6.0": run_args += ['--runtimes', 'wasm'] diff --git a/src/benchmarks/micro/Program.cs b/src/benchmarks/micro/Program.cs index 389ffc8e51b..9348f0572e5 100644 --- a/src/benchmarks/micro/Program.cs +++ b/src/benchmarks/micro/Program.cs @@ -45,7 +45,7 @@ static int Main(string[] args) .FromAssembly(typeof(Program).Assembly) .Run(argsList.ToArray(), RecommendedConfig.Create( - artifactsPath: new DirectoryInfo(Path.Combine(Path.GetDirectoryName(typeof(Program).Assembly.Location), "BenchmarkDotNet.Artifacts")), + artifactsPath: new DirectoryInfo(Path.Combine(AppContext.BaseDirectory, "BenchmarkDotNet.Artifacts")), mandatoryCategories: ImmutableHashSet.Create(Categories.Libraries, Categories.Runtime, Categories.ThirdParty), partitionCount: partitionCount, partitionIndex: partitionIndex, diff --git a/src/benchmarks/micro/Serializers/Binary_FromStream.cs b/src/benchmarks/micro/Serializers/Binary_FromStream.cs index 1885daf2a20..e8fed0f8309 100644 --- a/src/benchmarks/micro/Serializers/Binary_FromStream.cs +++ b/src/benchmarks/micro/Serializers/Binary_FromStream.cs @@ -6,6 +6,7 @@ using System.IO; using System.Runtime.Serialization.Formatters.Binary; using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Attributes.Filters; using MicroBenchmarks.Serializers.Helpers; namespace MicroBenchmarks.Serializers @@ -16,6 +17,7 @@ namespace MicroBenchmarks.Serializers [GenericTypeArguments(typeof(MyEventsListerViewModel))] [GenericTypeArguments(typeof(CollectionsOfPrimitives))] [BenchmarkCategory(Categories.NoWASM)] + [AotFilter("Dynamic code generation is not supported.")] public class Binary_FromStream { private T value; diff --git a/src/benchmarks/micro/Serializers/Binary_ToStream.cs b/src/benchmarks/micro/Serializers/Binary_ToStream.cs index 8fe712e748f..522dedaab84 100644 --- a/src/benchmarks/micro/Serializers/Binary_ToStream.cs +++ b/src/benchmarks/micro/Serializers/Binary_ToStream.cs @@ -6,6 +6,7 @@ using System.IO; using System.Runtime.Serialization.Formatters.Binary; using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Attributes.Filters; using MicroBenchmarks.Serializers.Helpers; namespace MicroBenchmarks.Serializers @@ -16,6 +17,7 @@ namespace MicroBenchmarks.Serializers [GenericTypeArguments(typeof(MyEventsListerViewModel))] [GenericTypeArguments(typeof(CollectionsOfPrimitives))] [BenchmarkCategory(Categories.NoWASM)] + [AotFilter("Dynamic code generation is not supported.")] public class Binary_ToStream { private readonly T value; diff --git a/src/benchmarks/micro/Serializers/Json_FromStream.cs b/src/benchmarks/micro/Serializers/Json_FromStream.cs index e8878442787..b9ab21cae8f 100644 --- a/src/benchmarks/micro/Serializers/Json_FromStream.cs +++ b/src/benchmarks/micro/Serializers/Json_FromStream.cs @@ -6,6 +6,7 @@ using System.Runtime.Serialization.Json; using System.Text; using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Attributes.Filters; namespace MicroBenchmarks.Serializers { @@ -15,6 +16,7 @@ namespace MicroBenchmarks.Serializers [GenericTypeArguments(typeof(MyEventsListerViewModel))] [GenericTypeArguments(typeof(CollectionsOfPrimitives))] [BenchmarkCategory(Categories.NoAOT)] + [AotFilter("Dynamic code generation is not supported.")] public class Json_FromStream { private T value; diff --git a/src/benchmarks/micro/Serializers/Json_FromString.cs b/src/benchmarks/micro/Serializers/Json_FromString.cs index ed2b3aa0c6d..7013f264f8a 100644 --- a/src/benchmarks/micro/Serializers/Json_FromString.cs +++ b/src/benchmarks/micro/Serializers/Json_FromString.cs @@ -3,6 +3,7 @@ // See the LICENSE file in the project root for more information. using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Attributes.Filters; namespace MicroBenchmarks.Serializers { @@ -12,6 +13,7 @@ namespace MicroBenchmarks.Serializers [GenericTypeArguments(typeof(MyEventsListerViewModel))] [GenericTypeArguments(typeof(CollectionsOfPrimitives))] [BenchmarkCategory(Categories.NoAOT)] + [AotFilter("Dynamic code generation is not supported.")] public class Json_FromString { private string serialized; diff --git a/src/benchmarks/micro/Serializers/Json_ToStream.cs b/src/benchmarks/micro/Serializers/Json_ToStream.cs index 4f263cf2e2c..aa1de7c43e0 100644 --- a/src/benchmarks/micro/Serializers/Json_ToStream.cs +++ b/src/benchmarks/micro/Serializers/Json_ToStream.cs @@ -6,6 +6,7 @@ using System.Runtime.Serialization.Json; using System.Text; using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Attributes.Filters; namespace MicroBenchmarks.Serializers { @@ -15,6 +16,7 @@ namespace MicroBenchmarks.Serializers [GenericTypeArguments(typeof(MyEventsListerViewModel))] [GenericTypeArguments(typeof(CollectionsOfPrimitives))] [BenchmarkCategory(Categories.NoAOT)] + [AotFilter("Dynamic code generation is not supported.")] public class Json_ToStream { private T value; diff --git a/src/benchmarks/micro/Serializers/Json_ToString.cs b/src/benchmarks/micro/Serializers/Json_ToString.cs index 5bee715dbe9..05d80c1061d 100644 --- a/src/benchmarks/micro/Serializers/Json_ToString.cs +++ b/src/benchmarks/micro/Serializers/Json_ToString.cs @@ -3,6 +3,7 @@ // See the LICENSE file in the project root for more information. using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Attributes.Filters; namespace MicroBenchmarks.Serializers { @@ -12,6 +13,7 @@ namespace MicroBenchmarks.Serializers [GenericTypeArguments(typeof(MyEventsListerViewModel))] [GenericTypeArguments(typeof(CollectionsOfPrimitives))] [BenchmarkCategory(Categories.NoAOT)] + [AotFilter("Dynamic code generation is not supported.")] public class Json_ToString { private T value; diff --git a/src/benchmarks/micro/Serializers/Xml_FromStream.cs b/src/benchmarks/micro/Serializers/Xml_FromStream.cs index b594e3e95e6..9cd0412a976 100644 --- a/src/benchmarks/micro/Serializers/Xml_FromStream.cs +++ b/src/benchmarks/micro/Serializers/Xml_FromStream.cs @@ -7,6 +7,7 @@ using System.Xml; using System.Xml.Serialization; using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Attributes.Filters; namespace MicroBenchmarks.Serializers { @@ -18,6 +19,7 @@ namespace MicroBenchmarks.Serializers [GenericTypeArguments(typeof(XmlElement))] [GenericTypeArguments(typeof(SimpleStructWithProperties))] [GenericTypeArguments(typeof(ClassImplementingIXmlSerialiable))] + [AotFilter("Currently not supported due to missing metadata.")] public class Xml_FromStream { private T value; diff --git a/src/benchmarks/micro/Serializers/Xml_ToStream.cs b/src/benchmarks/micro/Serializers/Xml_ToStream.cs index ee05d664f71..e00a823fcc1 100644 --- a/src/benchmarks/micro/Serializers/Xml_ToStream.cs +++ b/src/benchmarks/micro/Serializers/Xml_ToStream.cs @@ -7,6 +7,7 @@ using System.Xml; using System.Xml.Serialization; using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Attributes.Filters; namespace MicroBenchmarks.Serializers { @@ -18,6 +19,7 @@ namespace MicroBenchmarks.Serializers [GenericTypeArguments(typeof(XmlElement))] [GenericTypeArguments(typeof(SimpleStructWithProperties))] [GenericTypeArguments(typeof(ClassImplementingIXmlSerialiable))] + [AotFilter("Currently not supported due to missing metadata.")] public class Xml_ToStream { private T value; diff --git a/src/benchmarks/micro/libraries/System.Collections/Concurrent/AddRemoveFromDifferentThreads.cs b/src/benchmarks/micro/libraries/System.Collections/Concurrent/AddRemoveFromDifferentThreads.cs index b824a672c3f..69cb9e4ab12 100644 --- a/src/benchmarks/micro/libraries/System.Collections/Concurrent/AddRemoveFromDifferentThreads.cs +++ b/src/benchmarks/micro/libraries/System.Collections/Concurrent/AddRemoveFromDifferentThreads.cs @@ -6,6 +6,7 @@ using System.Threading; using System.Threading.Tasks; using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Attributes.Filters; using MicroBenchmarks; namespace System.Collections.Concurrent @@ -15,6 +16,7 @@ namespace System.Collections.Concurrent [GenericTypeArguments(typeof(string))] // reference type [MinWarmupCount(6, forceAutoWarmup: true)] [MaxWarmupCount(10, forceAutoWarmup: true)] + [AotFilter("It hangs. https://github.com/dotnet/runtime/issues/66987")] public class AddRemoveFromDifferentThreads { const int NumThreads = 2; diff --git a/src/benchmarks/micro/libraries/System.IO.Compression/CompressedFile.cs b/src/benchmarks/micro/libraries/System.IO.Compression/CompressedFile.cs index e9a146395ba..ecf97c5f544 100644 --- a/src/benchmarks/micro/libraries/System.IO.Compression/CompressedFile.cs +++ b/src/benchmarks/micro/libraries/System.IO.Compression/CompressedFile.cs @@ -34,7 +34,7 @@ public CompressedFile(string fileName, CompressionLevel compressionLevel, Func Path.Combine( - Path.GetDirectoryName(typeof(CompressedFile).Assembly.Location), + AppContext.BaseDirectory, "libraries", "System.IO.Compression", "TestData", fileName); } diff --git a/src/benchmarks/micro/libraries/System.Net.Http/Configuration.Certificates.cs b/src/benchmarks/micro/libraries/System.Net.Http/Configuration.Certificates.cs index 9c1ccbb2d4d..ce1f142c678 100644 --- a/src/benchmarks/micro/libraries/System.Net.Http/Configuration.Certificates.cs +++ b/src/benchmarks/micro/libraries/System.Net.Http/Configuration.Certificates.cs @@ -27,7 +27,7 @@ private static X509Certificate2 GetCertificate(string certificateFileName) => new X509Certificate2( File.ReadAllBytes( Path.Combine( - Path.GetDirectoryName(typeof(Certificates).Assembly.Location), + AppContext.BaseDirectory, "libraries", "System.Net.Http", certificateFileName)), diff --git a/src/benchmarks/micro/libraries/System.Runtime.Serialization.Formatters/Perf.BinaryFormatter.cs b/src/benchmarks/micro/libraries/System.Runtime.Serialization.Formatters/Perf.BinaryFormatter.cs index 294bfe8716f..a8beada29fc 100644 --- a/src/benchmarks/micro/libraries/System.Runtime.Serialization.Formatters/Perf.BinaryFormatter.cs +++ b/src/benchmarks/micro/libraries/System.Runtime.Serialization.Formatters/Perf.BinaryFormatter.cs @@ -7,11 +7,13 @@ using System.Linq; using System.Runtime.Serialization.Formatters.Binary; using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Attributes.Filters; using MicroBenchmarks; namespace System.Runtime.Serialization.Formatters.Tests { [BenchmarkCategory(Categories.Libraries, Categories.NoWASM)] + [AotFilter("Disabled by default.")] public class Perf_BinaryFormatter { private readonly BinaryFormatter _formatter = new BinaryFormatter(); diff --git a/src/benchmarks/micro/libraries/System.Text.Json/Serializer/ColdStartSerialization.cs b/src/benchmarks/micro/libraries/System.Text.Json/Serializer/ColdStartSerialization.cs index 837046d5a46..26971355f58 100644 --- a/src/benchmarks/micro/libraries/System.Text.Json/Serializer/ColdStartSerialization.cs +++ b/src/benchmarks/micro/libraries/System.Text.Json/Serializer/ColdStartSerialization.cs @@ -3,6 +3,7 @@ // See the LICENSE file in the project root for more information. using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Attributes.Filters; using MicroBenchmarks; using MicroBenchmarks.Serializers; @@ -10,6 +11,7 @@ namespace System.Text.Json.Serialization.Tests { [BenchmarkCategory(Categories.Libraries, Categories.JSON)] [GenericTypeArguments(typeof(SimpleStructWithProperties))] + [AotFilter("Currently not supported due to missing metadata.")] public class ColdStartSerialization { // Measures cold start performance for the serializer diff --git a/src/benchmarks/micro/libraries/System.Text.Json/Serializer/ReadJson.cs b/src/benchmarks/micro/libraries/System.Text.Json/Serializer/ReadJson.cs index eedc6b4fdb8..77b8d9e391e 100644 --- a/src/benchmarks/micro/libraries/System.Text.Json/Serializer/ReadJson.cs +++ b/src/benchmarks/micro/libraries/System.Text.Json/Serializer/ReadJson.cs @@ -3,6 +3,7 @@ // See the LICENSE file in the project root for more information. using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Attributes.Filters; using MicroBenchmarks; using MicroBenchmarks.Serializers; using System.Collections; @@ -28,6 +29,7 @@ namespace System.Text.Json.Serialization.Tests [GenericTypeArguments(typeof(LargeStructWithProperties))] [GenericTypeArguments(typeof(DateTimeOffset?))] [GenericTypeArguments(typeof(int))] + [AotFilter("Currently not supported due to missing metadata.")] public class ReadJson { private string _serialized; diff --git a/src/benchmarks/micro/libraries/System.Text.Json/Serializer/ReadMissingAndCaseInsensitive.cs b/src/benchmarks/micro/libraries/System.Text.Json/Serializer/ReadMissingAndCaseInsensitive.cs index d77ae85c685..dd3b59e6e8c 100644 --- a/src/benchmarks/micro/libraries/System.Text.Json/Serializer/ReadMissingAndCaseInsensitive.cs +++ b/src/benchmarks/micro/libraries/System.Text.Json/Serializer/ReadMissingAndCaseInsensitive.cs @@ -3,6 +3,7 @@ // See the LICENSE file in the project root for more information. using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Attributes.Filters; using MicroBenchmarks; using MicroBenchmarks.Serializers; @@ -10,6 +11,7 @@ namespace System.Text.Json.Serialization.Tests { [BenchmarkCategory(Categories.Libraries, Categories.JSON)] [GenericTypeArguments(typeof(Location))] + [AotFilter("Currently not supported due to missing metadata.")] public class ReadMissingAndCaseInsensitive { private string _serialized; diff --git a/src/benchmarks/micro/libraries/System.Text.Json/Serializer/ReadPreservedReferences.cs b/src/benchmarks/micro/libraries/System.Text.Json/Serializer/ReadPreservedReferences.cs index 15b3e883ab1..caacd0c6868 100644 --- a/src/benchmarks/micro/libraries/System.Text.Json/Serializer/ReadPreservedReferences.cs +++ b/src/benchmarks/micro/libraries/System.Text.Json/Serializer/ReadPreservedReferences.cs @@ -15,6 +15,7 @@ namespace System.Text.Json.Serialization.Tests [GenericTypeArguments(typeof(MyEventsListerViewModel))] [GenericTypeArguments(typeof(SimpleListOfInt))] [GenericTypeArguments(typeof(SimpleStructWithProperties))] + [AotFilter("Currently not supported due to missing metadata.")] public class ReadPreservedReferences { [Params(false, true)] diff --git a/src/benchmarks/micro/libraries/System.Text.Json/Serializer/WriteJson.cs b/src/benchmarks/micro/libraries/System.Text.Json/Serializer/WriteJson.cs index 51ed6a997cf..11d26d008d7 100644 --- a/src/benchmarks/micro/libraries/System.Text.Json/Serializer/WriteJson.cs +++ b/src/benchmarks/micro/libraries/System.Text.Json/Serializer/WriteJson.cs @@ -3,6 +3,7 @@ // See the LICENSE file in the project root for more information. using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Attributes.Filters; using MicroBenchmarks; using MicroBenchmarks.Serializers; using System.Collections; @@ -29,6 +30,7 @@ namespace System.Text.Json.Serialization.Tests [GenericTypeArguments(typeof(LargeStructWithProperties))] [GenericTypeArguments(typeof(DateTimeOffset?))] [GenericTypeArguments(typeof(int))] + [AotFilter("Currently not supported due to missing metadata.")] public class WriteJson { private T _value; diff --git a/src/benchmarks/micro/libraries/System.Text.Json/Serializer/WritePreservedReferences.cs b/src/benchmarks/micro/libraries/System.Text.Json/Serializer/WritePreservedReferences.cs index 647ba854ed5..ac11a2eb2ab 100644 --- a/src/benchmarks/micro/libraries/System.Text.Json/Serializer/WritePreservedReferences.cs +++ b/src/benchmarks/micro/libraries/System.Text.Json/Serializer/WritePreservedReferences.cs @@ -15,6 +15,7 @@ namespace System.Text.Json.Serialization.Tests [GenericTypeArguments(typeof(MyEventsListerViewModel))] [GenericTypeArguments(typeof(SimpleListOfInt))] [GenericTypeArguments(typeof(SimpleStructWithProperties))] + [AotFilter("Currently not supported due to missing metadata.")] public class WritePreservedReferences { private T _value; diff --git a/src/benchmarks/micro/libraries/System.Text.RegularExpressions/Perf.Regex.Industry.cs b/src/benchmarks/micro/libraries/System.Text.RegularExpressions/Perf.Regex.Industry.cs index 19a642ee830..e317d8b9429 100644 --- a/src/benchmarks/micro/libraries/System.Text.RegularExpressions/Perf.Regex.Industry.cs +++ b/src/benchmarks/micro/libraries/System.Text.RegularExpressions/Perf.Regex.Industry.cs @@ -15,7 +15,7 @@ internal static class Perf_Regex_Industry { public static string ReadInputFile(string name) { - string path = Path.Combine(Path.GetDirectoryName(typeof(Perf_Regex_Industry).Assembly.Location), "libraries", "System.Text.RegularExpressions", "TestData", name); + string path = Path.Combine(AppContext.BaseDirectory, "libraries", "System.Text.RegularExpressions", "TestData", name); using (FileStream fs = File.OpenRead(path)) using (var gz = new GZipStream(fs, CompressionMode.Decompress)) using (var reader = new StreamReader(gz)) diff --git a/src/benchmarks/micro/runtime/BenchmarksGame/Inputs/InputFileHelper.cs b/src/benchmarks/micro/runtime/BenchmarksGame/Inputs/InputFileHelper.cs index 67ff91de9ab..7b23c728ab1 100644 --- a/src/benchmarks/micro/runtime/BenchmarksGame/Inputs/InputFileHelper.cs +++ b/src/benchmarks/micro/runtime/BenchmarksGame/Inputs/InputFileHelper.cs @@ -17,9 +17,9 @@ internal static string FindInputFile(string inputFile) return fullPath; } - internal static int GetFileLength(string filePath) => (int) new FileInfo(filePath).Length; + internal static int GetFileLength(string filePath) => (int)new FileInfo(filePath).Length; private static string GetFullPath(string fileName) - => Path.Combine(Path.GetDirectoryName(typeof(InputFileHelper).Assembly.Location), "runtime", "BenchmarksGame", "Inputs", fileName); + => Path.Combine(AppContext.BaseDirectory, "runtime", "BenchmarksGame", "Inputs", fileName); } } \ No newline at end of file diff --git a/src/benchmarks/micro/runtime/Benchstones/MDBenchI/MDGeneralArray.cs b/src/benchmarks/micro/runtime/Benchstones/MDBenchI/MDGeneralArray.cs index 3a8ea0b63ed..0f4cd91c87c 100644 --- a/src/benchmarks/micro/runtime/Benchstones/MDBenchI/MDGeneralArray.cs +++ b/src/benchmarks/micro/runtime/Benchstones/MDBenchI/MDGeneralArray.cs @@ -5,6 +5,7 @@ using System.Runtime.CompilerServices; using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Attributes.Filters; using MicroBenchmarks; namespace Benchstone.MDBenchI @@ -66,6 +67,7 @@ public bool Test() { } [Benchmark(Description = "MDGeneralArray2")] + [AotFilter("Arrays with non-zero lower bounds are not supported.")] public bool Test2() { int[] lengths = new int[3] { 10, 10, 10 }; int[] lowerBounds = new int[3] { -5, 0, 5 }; diff --git a/src/benchmarks/micro/runtime/Roslyn/CscBench.cs b/src/benchmarks/micro/runtime/Roslyn/CscBench.cs index e9893e0f088..2f948c3873d 100644 --- a/src/benchmarks/micro/runtime/Roslyn/CscBench.cs +++ b/src/benchmarks/micro/runtime/Roslyn/CscBench.cs @@ -13,8 +13,10 @@ using System.Runtime.CompilerServices; using BenchmarkDotNet.Attributes; using MicroBenchmarks; +using BenchmarkDotNet.Attributes.Filters; [BenchmarkCategory(Categories.Runtime, Categories.JIT, Categories.CscBench, Categories.NoWASM)] +[AotFilter("GetReferencedAssemblies() is not supported")] public class CscBench { public const int CompileIterations = 1500; diff --git a/src/benchmarks/micro/runtime/System.Reflection/Activator.cs b/src/benchmarks/micro/runtime/System.Reflection/Activator.cs index 0e510a98dec..9767fd9ee45 100644 --- a/src/benchmarks/micro/runtime/System.Reflection/Activator.cs +++ b/src/benchmarks/micro/runtime/System.Reflection/Activator.cs @@ -3,6 +3,7 @@ // See the LICENSE file in the project root for more information. using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Attributes.Filters; using MicroBenchmarks; namespace System.Reflection @@ -22,6 +23,7 @@ public class Activator public object CreateInstanceType() => System.Activator.CreateInstance(typeof(T)); [Benchmark] + [AotFilter("Not supported.")] public object CreateInstanceNames() => System.Activator.CreateInstance(_assemblyName, _typeName); } diff --git a/src/harness/BenchmarkDotNet.Extensions/RecommendedConfig.cs b/src/harness/BenchmarkDotNet.Extensions/RecommendedConfig.cs index 134f6865a67..7efdcc7dd04 100644 --- a/src/harness/BenchmarkDotNet.Extensions/RecommendedConfig.cs +++ b/src/harness/BenchmarkDotNet.Extensions/RecommendedConfig.cs @@ -38,12 +38,16 @@ public static IConfig Create( .WithMaxIterationCount(20) // we don't want to run more that 20 iterations .DontEnforcePowerPlan(); // make sure BDN does not try to enforce High Performance power plan on Windows - // See https://github.com/dotnet/roslyn/issues/42393 - job = job.WithArguments(new Argument[] { new MsBuildArgument("/p:DebugType=portable"), new MsBuildArgument("-bl:benchmarkdotnet.binlog") }); + job = job.WithArguments(new Argument[] + { + new MsBuildArgument("/p:DebugType=portable"), // See https://github.com/dotnet/roslyn/issues/42393 + new MsBuildArgument("-bl:benchmarkdotnet.binlog"), // for diagnosing CI failures + new MsBuildArgument("/p:TrimmerDefaultAction=link"), // to ensure that trimmer only analyzes the parts of the dependencies that are used + }); } var config = ManualConfig.CreateEmpty() - .WithBuildTimeout(TimeSpan.FromMinutes(10)) // for slow machines + .WithBuildTimeout(TimeSpan.FromMinutes(15)) // for slow machines .AddLogger(ConsoleLogger.Default) // log output to console .AddValidator(DefaultConfig.Instance.GetValidators().ToArray()) // copy default validators .AddAnalyser(DefaultConfig.Instance.GetAnalysers().ToArray()) // copy default analysers