Add Action to run performance benchmarks (#4285)

* Pass console args to BDN.

* Comment tests temporarily.

* Add perf benchmark.

* Update perf project to net6.0

* Trigger action.

* Fix path

* Trigger.

* Fix artifact path. Use .NET Core 3.1.

* Comment out build path temporarily.

* Install .NET.

* Fix

* Fix warnings.

* Enable graphs.

* Fix push gh pages

* Fix graphs.

* Added tests.

* Add all tests to benchmark action. Remove (1, 1000) test case.

* Fix test naming.

* Update run command in ADO perf yml.

* Change threshold.

* Update path.

* Temporary change.

* Update yml, trigger on pull request, but don't publish the charts.

* Test trigger.

* Test.

* Test3. Fix ADO perf yml.

* Fix push GH pages.

* Revert.

* Revert ADO yml.

* Update perf alert threshold.

* Cleanup.

* Try upload to GH pages.

* Fix for testing.

* Fix for test.

* Fix for tests.

* Revert.

.github/workflows/benchmark-action.yml

@@ -0,0 +1,96 @@
+name: Run performance benchmarks
+on:
+  workflow_dispatch: # Allows manual triggering of workflow
+  push:
+    branches:
+      - main
+    paths:
+      - src/client/Microsoft.Identity.Client/**/*.cs
+
+permissions:
+  # Deployments permission to deploy GitHub pages website
+  deployments: write
+  # Contents permission to update benchmark contents in gh-pages branch
+  contents: write
+
+jobs:
+  benchmark:
+    name: Run performance benchmarks
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-dotnet@v3
+        with:
+          dotnet-version: '3.1.426'
+      - name: Run benchmark
+        run: cd tests/Microsoft.Identity.Test.Performance && dotnet run -c release -f netcoreapp3.1 --exporters json 
+
+      - name: Download previous benchmark data
+        uses: actions/cache@v3
+        with:
+          path: ./cache
+          key: ${{ runner.os }}-benchmark
+
+      - name: Store AcquireTokenNoCacheTests result
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          name: AcquireTokenNoCache
+          tool: 'benchmarkdotnet'
+          output-file-path: tests/Microsoft.Identity.Test.Performance/BenchmarkDotNet.Artifacts/results/Microsoft.Identity.Test.Performance.AcquireTokenNoCacheTests-report-full-compressed.json
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          auto-push: true
+          # Show alert with commit comment on detecting possible performance regression
+          alert-threshold: '130%'
+          comment-on-alert: true
+          fail-on-alert: true
+          summary-always: true
+          gh-pages-branch: gh-pages
+          benchmark-data-dir-path: benchmarks
+
+      - name: Store AcquireTokenForClientCacheTests result
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          name: AcquireTokenForClientWithCache
+          tool: 'benchmarkdotnet'
+          output-file-path: tests/Microsoft.Identity.Test.Performance/BenchmarkDotNet.Artifacts/results/Microsoft.Identity.Test.Performance.AcquireTokenForClientCacheTests-report-full-compressed.json
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          auto-push: true
+          # Show alert with commit comment on detecting possible performance regression
+          alert-threshold: '130%'
+          comment-on-alert: true
+          fail-on-alert: true
+          summary-always: true
+          gh-pages-branch: gh-pages
+          benchmark-data-dir-path: benchmarks
+
+      - name: Store AcquireTokenForOboCacheTests result
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          name: AcquireTokenForOboWithCache
+          tool: 'benchmarkdotnet'
+          output-file-path: tests/Microsoft.Identity.Test.Performance/BenchmarkDotNet.Artifacts/results/Microsoft.Identity.Test.Performance.AcquireTokenForOboCacheTests-report-full-compressed.json
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          auto-push: true
+          # Show alert with commit comment on detecting possible performance regression
+          alert-threshold: '130%'
+          comment-on-alert: true
+          fail-on-alert: true
+          summary-always: true
+          gh-pages-branch: gh-pages
+          benchmark-data-dir-path: benchmarks
+
+      - name: Store TokenCacheTests result
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          name: TokenCacheTestsWithCache
+          tool: 'benchmarkdotnet'
+          output-file-path: tests/Microsoft.Identity.Test.Performance/BenchmarkDotNet.Artifacts/results/Microsoft.Identity.Test.Performance.TokenCacheTests-report-full-compressed.json
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          auto-push: true
+          # Show alert with commit comment on detecting possible performance regression
+          alert-threshold: '130%'
+          comment-on-alert: true
+          fail-on-alert: true
+          summary-always: true
+          gh-pages-branch: gh-pages
+          benchmark-data-dir-path: benchmarks

tests/Microsoft.Identity.Test.Performance/AcquireTokenForClientCacheTests.cs

@@ -41,7 +41,6 @@ public class AcquireTokenForClientCacheTests
         // This is a workaround to specify the exact param combinations to be used.
         public IEnumerable<(int, int)> CacheSizeSource => new[] {
             (1, 10),
-            (1, 1000),
             (10000, 10),
         };
 

tests/Microsoft.Identity.Test.Performance/AcquireTokenForOboCacheTests.cs

@@ -43,7 +43,6 @@ public class AcquireTokenForOboCacheTests
         // This is a workaround to specify the exact param combinations to be used.
         public IEnumerable<(int, int)> CacheSizeSource => new[] {
             (1, 10),
-            (1, 1000),
             (10000, 10),
         };
 

tests/Microsoft.Identity.Test.Performance/Microsoft.Identity.Test.Performance.csproj

@@ -20,7 +20,6 @@
   <ItemGroup Condition="'$(UseIdentityWeb)' == 'true'">
     <PackageReference Include="Microsoft.Identity.Web.TokenCache" Version="*" />
     <PackageReference Include="Microsoft.Identity.Client" Version="*" />
-    <PackageReference Include="Microsoft.Identity.Client.Desktop" Version="*" />
   </ItemGroup>
 
   <ItemGroup Condition="'$(UseIdentityWeb)' != 'true'">

tests/Microsoft.Identity.Test.Performance/Program.cs

@@ -33,14 +33,14 @@ public static void Main(string[] args)
 #endif
                         .WithOptions(ConfigOptions.DisableLogFile)
                         .WithOptions(ConfigOptions.StopOnFirstError)
-                        .WithOptions(ConfigOptions.JoinSummary)
+                        //.WithOptions(ConfigOptions.JoinSummary)
                         .WithOrderer(new DefaultOrderer(SummaryOrderPolicy.Method))
                         .HideColumns(Column.UnrollFactor, Column.Type, Column.InvocationCount, Column.Error, Column.StdDev, Column.Median, Column.Job)
                         .AddDiagnoser(MemoryDiagnoser.Default) // https://benchmarkdotnet.org/articles/configs/diagnosers.html
                         //.AddDiagnoser(new EtwProfiler()) // https://adamsitnik.com/ETW-Profiler/
                 .AddJob(
                     Job.Default
-                        .WithId("Job-PerfTests")));
+                        .WithId("Job-PerfTests")), args);
             }
             catch (Exception ex)
             {

tests/Microsoft.Identity.Test.Performance/TokenCacheTests.cs

@@ -37,7 +37,6 @@ public class TokenCacheTests
         // This is a workaround to specify the exact param combinations to be used.
         public IEnumerable<(int, int)> CacheSizeSource => new[] {
             (1, 10),
-            (1, 1000),
             (10000, 10),
         };