From 25ccce87cd6c3180c69b928319e19fbaac6bd8c0 Mon Sep 17 00:00:00 2001
From: Peter <34331512+pmaytak@users.noreply.github.com>
Date: Wed, 9 Aug 2023 23:33:34 -0700
Subject: [PATCH] Add Action to run performance benchmarks (#4285)
* Pass console args to BDN.
* Comment tests temporarily.
* Add perf benchmark.
* Update perf project to net6.0
* Trigger action.
* Fix path
* Trigger.
* Fix artifact path. Use .NET Core 3.1.
* Comment out build path temporarily.
* Install .NET.
* Fix
* Fix warnings.
* Enable graphs.
* Fix push gh pages
* Fix graphs.
* Added tests.
* Add all tests to benchmark action. Remove (1, 1000) test case.
* Fix test naming.
* Update run command in ADO perf yml.
* Change threshold.
* Update path.
* Temporary change.
* Update yml, trigger on pull request, but don't publish the charts.
* Test trigger.
* Test.
* Test3. Fix ADO perf yml.
* Fix push GH pages.
* Revert.
* Revert ADO yml.
* Update perf alert threshold.
* Cleanup.
* Try upload to GH pages.
* Fix for testing.
* Fix for test.
* Fix for tests.
* Revert.
---
.github/workflows/benchmark-action.yml | 96 +++++++++++++++++++
.../AcquireTokenForClientCacheTests.cs | 1 -
.../AcquireTokenForOboCacheTests.cs | 1 -
...Microsoft.Identity.Test.Performance.csproj | 1 -
.../Program.cs | 4 +-
.../TokenCacheTests.cs | 1 -
6 files changed, 98 insertions(+), 6 deletions(-)
create mode 100644 .github/workflows/benchmark-action.yml
diff --git a/.github/workflows/benchmark-action.yml b/.github/workflows/benchmark-action.yml
new file mode 100644
index 0000000000..8558ce2f3a
--- /dev/null
+++ b/.github/workflows/benchmark-action.yml
@@ -0,0 +1,96 @@
+name: Run performance benchmarks
+on:
+ workflow_dispatch: # Allows manual triggering of workflow
+ push:
+ branches:
+ - main
+ paths:
+ - src/client/Microsoft.Identity.Client/**/*.cs
+
+permissions:
+ # Deployments permission to deploy GitHub pages website
+ deployments: write
+ # Contents permission to update benchmark contents in gh-pages branch
+ contents: write
+
+jobs:
+ benchmark:
+ name: Run performance benchmarks
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v3
+ - uses: actions/setup-dotnet@v3
+ with:
+ dotnet-version: '3.1.426'
+ - name: Run benchmark
+ run: cd tests/Microsoft.Identity.Test.Performance && dotnet run -c release -f netcoreapp3.1 --exporters json
+
+ - name: Download previous benchmark data
+ uses: actions/cache@v3
+ with:
+ path: ./cache
+ key: ${{ runner.os }}-benchmark
+
+ - name: Store AcquireTokenNoCacheTests result
+ uses: benchmark-action/github-action-benchmark@v1
+ with:
+ name: AcquireTokenNoCache
+ tool: 'benchmarkdotnet'
+ output-file-path: tests/Microsoft.Identity.Test.Performance/BenchmarkDotNet.Artifacts/results/Microsoft.Identity.Test.Performance.AcquireTokenNoCacheTests-report-full-compressed.json
+ github-token: ${{ secrets.GITHUB_TOKEN }}
+ auto-push: true
+ # Show alert with commit comment on detecting possible performance regression
+ alert-threshold: '130%'
+ comment-on-alert: true
+ fail-on-alert: true
+ summary-always: true
+ gh-pages-branch: gh-pages
+ benchmark-data-dir-path: benchmarks
+
+ - name: Store AcquireTokenForClientCacheTests result
+ uses: benchmark-action/github-action-benchmark@v1
+ with:
+ name: AcquireTokenForClientWithCache
+ tool: 'benchmarkdotnet'
+ output-file-path: tests/Microsoft.Identity.Test.Performance/BenchmarkDotNet.Artifacts/results/Microsoft.Identity.Test.Performance.AcquireTokenForClientCacheTests-report-full-compressed.json
+ github-token: ${{ secrets.GITHUB_TOKEN }}
+ auto-push: true
+ # Show alert with commit comment on detecting possible performance regression
+ alert-threshold: '130%'
+ comment-on-alert: true
+ fail-on-alert: true
+ summary-always: true
+ gh-pages-branch: gh-pages
+ benchmark-data-dir-path: benchmarks
+
+ - name: Store AcquireTokenForOboCacheTests result
+ uses: benchmark-action/github-action-benchmark@v1
+ with:
+ name: AcquireTokenForOboWithCache
+ tool: 'benchmarkdotnet'
+ output-file-path: tests/Microsoft.Identity.Test.Performance/BenchmarkDotNet.Artifacts/results/Microsoft.Identity.Test.Performance.AcquireTokenForOboCacheTests-report-full-compressed.json
+ github-token: ${{ secrets.GITHUB_TOKEN }}
+ auto-push: true
+ # Show alert with commit comment on detecting possible performance regression
+ alert-threshold: '130%'
+ comment-on-alert: true
+ fail-on-alert: true
+ summary-always: true
+ gh-pages-branch: gh-pages
+ benchmark-data-dir-path: benchmarks
+
+ - name: Store TokenCacheTests result
+ uses: benchmark-action/github-action-benchmark@v1
+ with:
+ name: TokenCacheTestsWithCache
+ tool: 'benchmarkdotnet'
+ output-file-path: tests/Microsoft.Identity.Test.Performance/BenchmarkDotNet.Artifacts/results/Microsoft.Identity.Test.Performance.TokenCacheTests-report-full-compressed.json
+ github-token: ${{ secrets.GITHUB_TOKEN }}
+ auto-push: true
+ # Show alert with commit comment on detecting possible performance regression
+ alert-threshold: '130%'
+ comment-on-alert: true
+ fail-on-alert: true
+ summary-always: true
+ gh-pages-branch: gh-pages
+ benchmark-data-dir-path: benchmarks
diff --git a/tests/Microsoft.Identity.Test.Performance/AcquireTokenForClientCacheTests.cs b/tests/Microsoft.Identity.Test.Performance/AcquireTokenForClientCacheTests.cs
index 03a0b4afee..cd87dc9893 100644
--- a/tests/Microsoft.Identity.Test.Performance/AcquireTokenForClientCacheTests.cs
+++ b/tests/Microsoft.Identity.Test.Performance/AcquireTokenForClientCacheTests.cs
@@ -41,7 +41,6 @@ public class AcquireTokenForClientCacheTests
// This is a workaround to specify the exact param combinations to be used.
public IEnumerable<(int, int)> CacheSizeSource => new[] {
(1, 10),
- (1, 1000),
(10000, 10),
};
diff --git a/tests/Microsoft.Identity.Test.Performance/AcquireTokenForOboCacheTests.cs b/tests/Microsoft.Identity.Test.Performance/AcquireTokenForOboCacheTests.cs
index c0a951ce91..a1afef99b1 100644
--- a/tests/Microsoft.Identity.Test.Performance/AcquireTokenForOboCacheTests.cs
+++ b/tests/Microsoft.Identity.Test.Performance/AcquireTokenForOboCacheTests.cs
@@ -43,7 +43,6 @@ public class AcquireTokenForOboCacheTests
// This is a workaround to specify the exact param combinations to be used.
public IEnumerable<(int, int)> CacheSizeSource => new[] {
(1, 10),
- (1, 1000),
(10000, 10),
};
diff --git a/tests/Microsoft.Identity.Test.Performance/Microsoft.Identity.Test.Performance.csproj b/tests/Microsoft.Identity.Test.Performance/Microsoft.Identity.Test.Performance.csproj
index 75d9acd8b0..b572404e90 100644
--- a/tests/Microsoft.Identity.Test.Performance/Microsoft.Identity.Test.Performance.csproj
+++ b/tests/Microsoft.Identity.Test.Performance/Microsoft.Identity.Test.Performance.csproj
@@ -20,7 +20,6 @@
-
diff --git a/tests/Microsoft.Identity.Test.Performance/Program.cs b/tests/Microsoft.Identity.Test.Performance/Program.cs
index 53bab64487..24bb4910c7 100644
--- a/tests/Microsoft.Identity.Test.Performance/Program.cs
+++ b/tests/Microsoft.Identity.Test.Performance/Program.cs
@@ -33,14 +33,14 @@ public static void Main(string[] args)
#endif
.WithOptions(ConfigOptions.DisableLogFile)
.WithOptions(ConfigOptions.StopOnFirstError)
- .WithOptions(ConfigOptions.JoinSummary)
+ //.WithOptions(ConfigOptions.JoinSummary)
.WithOrderer(new DefaultOrderer(SummaryOrderPolicy.Method))
.HideColumns(Column.UnrollFactor, Column.Type, Column.InvocationCount, Column.Error, Column.StdDev, Column.Median, Column.Job)
.AddDiagnoser(MemoryDiagnoser.Default) // https://benchmarkdotnet.org/articles/configs/diagnosers.html
//.AddDiagnoser(new EtwProfiler()) // https://adamsitnik.com/ETW-Profiler/
.AddJob(
Job.Default
- .WithId("Job-PerfTests")));
+ .WithId("Job-PerfTests")), args);
}
catch (Exception ex)
{
diff --git a/tests/Microsoft.Identity.Test.Performance/TokenCacheTests.cs b/tests/Microsoft.Identity.Test.Performance/TokenCacheTests.cs
index 56b7b16542..09d2358d29 100644
--- a/tests/Microsoft.Identity.Test.Performance/TokenCacheTests.cs
+++ b/tests/Microsoft.Identity.Test.Performance/TokenCacheTests.cs
@@ -37,7 +37,6 @@ public class TokenCacheTests
// This is a workaround to specify the exact param combinations to be used.
public IEnumerable<(int, int)> CacheSizeSource => new[] {
(1, 10),
- (1, 1000),
(10000, 10),
};