From 25ccce87cd6c3180c69b928319e19fbaac6bd8c0 Mon Sep 17 00:00:00 2001 From: Peter <34331512+pmaytak@users.noreply.github.com> Date: Wed, 9 Aug 2023 23:33:34 -0700 Subject: [PATCH] Add Action to run performance benchmarks (#4285) * Pass console args to BDN. * Comment tests temporarily. * Add perf benchmark. * Update perf project to net6.0 * Trigger action. * Fix path * Trigger. * Fix artifact path. Use .NET Core 3.1. * Comment out build path temporarily. * Install .NET. * Fix * Fix warnings. * Enable graphs. * Fix push gh pages * Fix graphs. * Added tests. * Add all tests to benchmark action. Remove (1, 1000) test case. * Fix test naming. * Update run command in ADO perf yml. * Change threshold. * Update path. * Temporary change. * Update yml, trigger on pull request, but don't publish the charts. * Test trigger. * Test. * Test3. Fix ADO perf yml. * Fix push GH pages. * Revert. * Revert ADO yml. * Update perf alert threshold. * Cleanup. * Try upload to GH pages. * Fix for testing. * Fix for test. * Fix for tests. * Revert. --- .github/workflows/benchmark-action.yml | 96 +++++++++++++++++++ .../AcquireTokenForClientCacheTests.cs | 1 - .../AcquireTokenForOboCacheTests.cs | 1 - ...Microsoft.Identity.Test.Performance.csproj | 1 - .../Program.cs | 4 +- .../TokenCacheTests.cs | 1 - 6 files changed, 98 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/benchmark-action.yml diff --git a/.github/workflows/benchmark-action.yml b/.github/workflows/benchmark-action.yml new file mode 100644 index 0000000000..8558ce2f3a --- /dev/null +++ b/.github/workflows/benchmark-action.yml @@ -0,0 +1,96 @@ +name: Run performance benchmarks +on: + workflow_dispatch: # Allows manual triggering of workflow + push: + branches: + - main + paths: + - src/client/Microsoft.Identity.Client/**/*.cs + +permissions: + # Deployments permission to deploy GitHub pages website + deployments: write + # Contents permission to update benchmark contents in gh-pages branch + contents: write + +jobs: + benchmark: + name: Run performance benchmarks + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-dotnet@v3 + with: + dotnet-version: '3.1.426' + - name: Run benchmark + run: cd tests/Microsoft.Identity.Test.Performance && dotnet run -c release -f netcoreapp3.1 --exporters json + + - name: Download previous benchmark data + uses: actions/cache@v3 + with: + path: ./cache + key: ${{ runner.os }}-benchmark + + - name: Store AcquireTokenNoCacheTests result + uses: benchmark-action/github-action-benchmark@v1 + with: + name: AcquireTokenNoCache + tool: 'benchmarkdotnet' + output-file-path: tests/Microsoft.Identity.Test.Performance/BenchmarkDotNet.Artifacts/results/Microsoft.Identity.Test.Performance.AcquireTokenNoCacheTests-report-full-compressed.json + github-token: ${{ secrets.GITHUB_TOKEN }} + auto-push: true + # Show alert with commit comment on detecting possible performance regression + alert-threshold: '130%' + comment-on-alert: true + fail-on-alert: true + summary-always: true + gh-pages-branch: gh-pages + benchmark-data-dir-path: benchmarks + + - name: Store AcquireTokenForClientCacheTests result + uses: benchmark-action/github-action-benchmark@v1 + with: + name: AcquireTokenForClientWithCache + tool: 'benchmarkdotnet' + output-file-path: tests/Microsoft.Identity.Test.Performance/BenchmarkDotNet.Artifacts/results/Microsoft.Identity.Test.Performance.AcquireTokenForClientCacheTests-report-full-compressed.json + github-token: ${{ secrets.GITHUB_TOKEN }} + auto-push: true + # Show alert with commit comment on detecting possible performance regression + alert-threshold: '130%' + comment-on-alert: true + fail-on-alert: true + summary-always: true + gh-pages-branch: gh-pages + benchmark-data-dir-path: benchmarks + + - name: Store AcquireTokenForOboCacheTests result + uses: benchmark-action/github-action-benchmark@v1 + with: + name: AcquireTokenForOboWithCache + tool: 'benchmarkdotnet' + output-file-path: tests/Microsoft.Identity.Test.Performance/BenchmarkDotNet.Artifacts/results/Microsoft.Identity.Test.Performance.AcquireTokenForOboCacheTests-report-full-compressed.json + github-token: ${{ secrets.GITHUB_TOKEN }} + auto-push: true + # Show alert with commit comment on detecting possible performance regression + alert-threshold: '130%' + comment-on-alert: true + fail-on-alert: true + summary-always: true + gh-pages-branch: gh-pages + benchmark-data-dir-path: benchmarks + + - name: Store TokenCacheTests result + uses: benchmark-action/github-action-benchmark@v1 + with: + name: TokenCacheTestsWithCache + tool: 'benchmarkdotnet' + output-file-path: tests/Microsoft.Identity.Test.Performance/BenchmarkDotNet.Artifacts/results/Microsoft.Identity.Test.Performance.TokenCacheTests-report-full-compressed.json + github-token: ${{ secrets.GITHUB_TOKEN }} + auto-push: true + # Show alert with commit comment on detecting possible performance regression + alert-threshold: '130%' + comment-on-alert: true + fail-on-alert: true + summary-always: true + gh-pages-branch: gh-pages + benchmark-data-dir-path: benchmarks diff --git a/tests/Microsoft.Identity.Test.Performance/AcquireTokenForClientCacheTests.cs b/tests/Microsoft.Identity.Test.Performance/AcquireTokenForClientCacheTests.cs index 03a0b4afee..cd87dc9893 100644 --- a/tests/Microsoft.Identity.Test.Performance/AcquireTokenForClientCacheTests.cs +++ b/tests/Microsoft.Identity.Test.Performance/AcquireTokenForClientCacheTests.cs @@ -41,7 +41,6 @@ public class AcquireTokenForClientCacheTests // This is a workaround to specify the exact param combinations to be used. public IEnumerable<(int, int)> CacheSizeSource => new[] { (1, 10), - (1, 1000), (10000, 10), }; diff --git a/tests/Microsoft.Identity.Test.Performance/AcquireTokenForOboCacheTests.cs b/tests/Microsoft.Identity.Test.Performance/AcquireTokenForOboCacheTests.cs index c0a951ce91..a1afef99b1 100644 --- a/tests/Microsoft.Identity.Test.Performance/AcquireTokenForOboCacheTests.cs +++ b/tests/Microsoft.Identity.Test.Performance/AcquireTokenForOboCacheTests.cs @@ -43,7 +43,6 @@ public class AcquireTokenForOboCacheTests // This is a workaround to specify the exact param combinations to be used. public IEnumerable<(int, int)> CacheSizeSource => new[] { (1, 10), - (1, 1000), (10000, 10), }; diff --git a/tests/Microsoft.Identity.Test.Performance/Microsoft.Identity.Test.Performance.csproj b/tests/Microsoft.Identity.Test.Performance/Microsoft.Identity.Test.Performance.csproj index 75d9acd8b0..b572404e90 100644 --- a/tests/Microsoft.Identity.Test.Performance/Microsoft.Identity.Test.Performance.csproj +++ b/tests/Microsoft.Identity.Test.Performance/Microsoft.Identity.Test.Performance.csproj @@ -20,7 +20,6 @@ - diff --git a/tests/Microsoft.Identity.Test.Performance/Program.cs b/tests/Microsoft.Identity.Test.Performance/Program.cs index 53bab64487..24bb4910c7 100644 --- a/tests/Microsoft.Identity.Test.Performance/Program.cs +++ b/tests/Microsoft.Identity.Test.Performance/Program.cs @@ -33,14 +33,14 @@ public static void Main(string[] args) #endif .WithOptions(ConfigOptions.DisableLogFile) .WithOptions(ConfigOptions.StopOnFirstError) - .WithOptions(ConfigOptions.JoinSummary) + //.WithOptions(ConfigOptions.JoinSummary) .WithOrderer(new DefaultOrderer(SummaryOrderPolicy.Method)) .HideColumns(Column.UnrollFactor, Column.Type, Column.InvocationCount, Column.Error, Column.StdDev, Column.Median, Column.Job) .AddDiagnoser(MemoryDiagnoser.Default) // https://benchmarkdotnet.org/articles/configs/diagnosers.html //.AddDiagnoser(new EtwProfiler()) // https://adamsitnik.com/ETW-Profiler/ .AddJob( Job.Default - .WithId("Job-PerfTests"))); + .WithId("Job-PerfTests")), args); } catch (Exception ex) { diff --git a/tests/Microsoft.Identity.Test.Performance/TokenCacheTests.cs b/tests/Microsoft.Identity.Test.Performance/TokenCacheTests.cs index 56b7b16542..09d2358d29 100644 --- a/tests/Microsoft.Identity.Test.Performance/TokenCacheTests.cs +++ b/tests/Microsoft.Identity.Test.Performance/TokenCacheTests.cs @@ -37,7 +37,6 @@ public class TokenCacheTests // This is a workaround to specify the exact param combinations to be used. public IEnumerable<(int, int)> CacheSizeSource => new[] { (1, 10), - (1, 1000), (10000, 10), };