Add Action to run performance benchmarks

.github/workflows/benchmark-action.yml

@@ -0,0 +1,112 @@
+name: Run performance benchmarks
+on:
+  workflow_dispatch: # Allows manual triggering of workflow
+  push:
+    branches:
+      - main
+    paths:
+      - src/client/Microsoft.Identity.Client/**/*.cs
+  #pull_request: # Will re-enable once the code to not push to gh-pages is fixed
+  #  types: [opened, reopened, synchronize, ready_for_review]
+  #  paths:
+  #    - src/client/Microsoft.Identity.Client/**/*.cs
+
+permissions:
+  # Deployments permission to deploy GitHub pages website
+  deployments: write
+  # Contents permission to update benchmark contents in gh-pages branch
+  contents: write
+
+jobs:
+  benchmark:
+    name: Run performance benchmarks
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-dotnet@v3
+        with:
+          dotnet-version: '3.1.426'
+      - name: Run benchmark
+        run: cd tests/Microsoft.Identity.Test.Performance && dotnet run -c release -f netcoreapp3.1 --exporters json 
+
+      - name: Download previous benchmark data
+        uses: actions/cache@v3
+        with:
+          path: ./cache
+          key: ${{ runner.os }}-benchmark
+
+      - name: Store AcquireTokenNoCacheTests result
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          name: AcquireTokenNoCache
+          tool: 'benchmarkdotnet'
+          output-file-path: tests/Microsoft.Identity.Test.Performance/BenchmarkDotNet.Artifacts/results/Microsoft.Identity.Test.Performance.AcquireTokenNoCacheTests-report-full-compressed.json
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          auto-push: true
+          #external-data-json-path: ./cache/benchmark-data.json
+          # Show alert with commit comment on detecting possible performance regression
+          alert-threshold: '140%'
+          comment-on-alert: true
+          fail-on-alert: true
+          alert-comment-cc-users: '@pmaytak'
+          summary-always: true
+          gh-pages-branch: gh-pages
+          benchmark-data-dir-path: benchmarks
+
+      - name: Store AcquireTokenForClientCacheTests result
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          name: AcquireTokenForClientWithCache
+          tool: 'benchmarkdotnet'
+          output-file-path: tests/Microsoft.Identity.Test.Performance/BenchmarkDotNet.Artifacts/results/Microsoft.Identity.Test.Performance.AcquireTokenForClientCacheTests-report-full-compressed.json
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          auto-push: true
+          #external-data-json-path: ./cache/benchmark-data.json
+          # Show alert with commit comment on detecting possible performance regression
+          alert-threshold: '140%'
+          comment-on-alert: true
+          fail-on-alert: true
+          alert-comment-cc-users: '@pmaytak'
+          summary-always: true
+          gh-pages-branch: gh-pages
+          benchmark-data-dir-path: benchmarks
+
+      - name: Store AcquireTokenForOboCacheTests result
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          name: AcquireTokenForOboWithCache
+          tool: 'benchmarkdotnet'
+          output-file-path: tests/Microsoft.Identity.Test.Performance/BenchmarkDotNet.Artifacts/results/Microsoft.Identity.Test.Performance.AcquireTokenForOboCacheTests-report-full-compressed.json
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          auto-push: true
+          #external-data-json-path: ./cache/benchmark-data.json
+          # Show alert with commit comment on detecting possible performance regression
+          alert-threshold: '140%'
+          comment-on-alert: true
+          fail-on-alert: true
+          alert-comment-cc-users: '@pmaytak'
+          summary-always: true
+          gh-pages-branch: gh-pages
+          benchmark-data-dir-path: benchmarks
+
+      - name: Store TokenCacheTests result
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          name: TokenCacheTestsWithCache
+          tool: 'benchmarkdotnet'
+          output-file-path: tests/Microsoft.Identity.Test.Performance/BenchmarkDotNet.Artifacts/results/Microsoft.Identity.Test.Performance.TokenCacheTests-report-full-compressed.json
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          auto-push: true
+          #external-data-json-path: ./cache/benchmark-data.json
+          # Show alert with commit comment on detecting possible performance regression
+          alert-threshold: '140%'
+          comment-on-alert: true
+          fail-on-alert: true
+          alert-comment-cc-users: '@pmaytak'
+          summary-always: true
+          gh-pages-branch: gh-pages
+          benchmark-data-dir-path: benchmarks
+
+      #- name: Publish GibHub Pages
+      #  run: git push origin gh-pages:gh-pages
+      #  if: github.event_name != 'pull_request'

tests/Microsoft.Identity.Test.Performance/AcquireTokenForClientCacheTests.cs

@@ -41,7 +41,6 @@ public class AcquireTokenForClientCacheTests
         // This is a workaround to specify the exact param combinations to be used.
         public IEnumerable<(int, int)> CacheSizeSource => new[] {
             (1, 10),
-            (1, 1000),
             (10000, 10),
         };
 

tests/Microsoft.Identity.Test.Performance/AcquireTokenForOboCacheTests.cs

@@ -43,7 +43,6 @@ public class AcquireTokenForOboCacheTests
         // This is a workaround to specify the exact param combinations to be used.
         public IEnumerable<(int, int)> CacheSizeSource => new[] {
             (1, 10),
-            (1, 1000),
             (10000, 10),
         };
 

tests/Microsoft.Identity.Test.Performance/Microsoft.Identity.Test.Performance.csproj

@@ -20,7 +20,6 @@
   <ItemGroup Condition="'$(UseIdentityWeb)' == 'true'">
     <PackageReference Include="Microsoft.Identity.Web.TokenCache" Version="*" />
     <PackageReference Include="Microsoft.Identity.Client" Version="*" />
-    <PackageReference Include="Microsoft.Identity.Client.Desktop" Version="*" />
   </ItemGroup>
 
   <ItemGroup Condition="'$(UseIdentityWeb)' != 'true'">

tests/Microsoft.Identity.Test.Performance/Program.cs

@@ -33,14 +33,14 @@ public static void Main(string[] args)
 #endif
                         .WithOptions(ConfigOptions.DisableLogFile)
                         .WithOptions(ConfigOptions.StopOnFirstError)
-                        .WithOptions(ConfigOptions.JoinSummary)
+                        //.WithOptions(ConfigOptions.JoinSummary)
                         .WithOrderer(new DefaultOrderer(SummaryOrderPolicy.Method))
                         .HideColumns(Column.UnrollFactor, Column.Type, Column.InvocationCount, Column.Error, Column.StdDev, Column.Median, Column.Job)
                         .AddDiagnoser(MemoryDiagnoser.Default) // https://benchmarkdotnet.org/articles/configs/diagnosers.html
                         //.AddDiagnoser(new EtwProfiler()) // https://adamsitnik.com/ETW-Profiler/
                 .AddJob(
                     Job.Default
-                        .WithId("Job-PerfTests")));
+                        .WithId("Job-PerfTests")), args);
             }
             catch (Exception ex)
             {

tests/Microsoft.Identity.Test.Performance/TokenCacheTests.cs

@@ -37,7 +37,6 @@ public class TokenCacheTests
         // This is a workaround to specify the exact param combinations to be used.
         public IEnumerable<(int, int)> CacheSizeSource => new[] {
             (1, 10),
-            (1, 1000),
             (10000, 10),
         };