ci: run the tests as a separate build stage (#12141)

This commit moves the unit and feature tests to a separate stage, which can be retried more easily. Closes #11858
microsoft · Jan 12, 2022 · 1a62f47 · 1a62f47
1 parent f1baa31
commit 1a62f47
Show file tree

Hide file tree

Showing 4 changed files with 111 additions and 53 deletions.
diff --git a/build/pipelines/ci.yml b/build/pipelines/ci.yml
@@ -61,6 +61,21 @@ stages:
       - template: ./templates/build-console-ci.yml
         parameters:
           platform: ARM64
+  - stage: Test_x64
+    displayName: Test x64
+    dependsOn: [Build_x64]
+    condition: succeeded()
+    jobs:
+      - template: ./templates/test-console-ci.yml
+        parameters:
+          platform: x64
+  - stage: Test_x86
+    displayName: Test x86
+    dependsOn: [Build_x86]
+    jobs:
+      - template: ./templates/test-console-ci.yml
+        parameters:
+          platform: x86
   - stage: Scripts
     displayName: Code Health Scripts
     dependsOn: []

diff --git a/build/pipelines/templates/build-console-steps.yml b/build/pipelines/templates/build-console-steps.yml
@@ -1,6 +1,5 @@
 parameters:
   additionalBuildArguments: ''
-  testLogPath: '$(Build.BinariesDirectory)\$(BuildPlatform)\$(BuildConfiguration)\testsOnBuildMachine.wtl'
 
 steps:
 - checkout: self
@@ -90,55 +89,6 @@ steps:
       If ($Arch -Eq "x86") { $Arch = "Win32" }
       Write-Host "##vso[task.setvariable variable=RationalizedBuildPlatform]${Arch}"
 
-- task: PowerShell@2
-  displayName: 'Run Unit Tests'
-  inputs:
-    targetType: filePath
-    filePath: build\scripts\Run-Tests.ps1
-    arguments: -MatchPattern '*unit.test*.dll' -Platform '$(RationalizedBuildPlatform)' -Configuration '$(BuildConfiguration)' -LogPath '${{ parameters.testLogPath }}'
-  condition: and(and(succeeded(), ne(variables['PGOBuildMode'], 'Instrument')), or(eq(variables['BuildPlatform'], 'x64'), eq(variables['BuildPlatform'], 'x86')))
-
-- task: PowerShell@2
-  displayName: 'Run Feature Tests (x64 only)'
-  inputs:
-    targetType: filePath
-    filePath: build\scripts\Run-Tests.ps1
-    arguments: -MatchPattern '*feature.test*.dll' -Platform '$(RationalizedBuildPlatform)' -Configuration '$(BuildConfiguration)' -LogPath '${{ parameters.testLogPath }}'
-  condition: and(and(succeeded(), ne(variables['PGOBuildMode'], 'Instrument')), eq(variables['BuildPlatform'], 'x64'))
-
-- task: PowerShell@2
-  displayName: 'Convert Test Logs from WTL to xUnit format'
-  inputs:
-    targetType: filePath
-    filePath: build\Helix\ConvertWttLogToXUnit.ps1
-    arguments: -WttInputPath '${{ parameters.testLogPath }}' -WttSingleRerunInputPath 'unused.wtl' -WttMultipleRerunInputPath 'unused2.wtl' -XUnitOutputPath 'onBuildMachineResults.xml' -TestNamePrefix '$(BuildConfiguration).$(BuildPlatform)'
-  condition: and(ne(variables['PGOBuildMode'], 'Instrument'),or(eq(variables['BuildPlatform'], 'x64'), eq(variables['BuildPlatform'], 'x86')))
-
-- task: PublishTestResults@2
-  displayName: 'Upload converted test logs'
-  condition: ne(variables['PGOBuildMode'], 'Instrument')
-  inputs:
-    testResultsFormat: 'xUnit' # Options: JUnit, NUnit, VSTest, xUnit, cTest
-    testResultsFiles: '**/onBuildMachineResults.xml'
-    #searchFolder: '$(System.DefaultWorkingDirectory)' # Optional
-    #mergeTestResults: false # Optional
-    #failTaskOnFailedTests: false # Optional
-    testRunTitle: 'On Build Machine Tests' # Optional
-    buildPlatform: $(BuildPlatform) # Optional
-    buildConfiguration: $(BuildConfiguration) # Optional
-    #publishRunAttachments: true # Optional
-
-- task: CopyFiles@2
-  displayName: 'Copy result logs to Artifacts'
-  inputs:
-    Contents: |
-     **/*.wtl
-     **/*onBuildMachineResults.xml
-     ${{ parameters.testLogPath }}
-    TargetFolder: '$(Build.ArtifactStagingDirectory)/$(BuildConfiguration)/$(BuildPlatform)/test'
-    OverWrite: true
-    flattenFolders: true
-
 - task: CopyFiles@2
   displayName: 'Copy *.appx/*.msix to Artifacts (Non-PR builds only)'
   inputs:

diff --git a/build/pipelines/templates/test-console-ci.yml b/build/pipelines/templates/test-console-ci.yml
@@ -0,0 +1,92 @@
+parameters:
+  configuration: 'Release'
+  platform: ''
+  additionalBuildArguments: ''
+  minimumExpectedTestsExecutedCount: 10  # Sanity check for minimum expected tests to be reported
+  rerunPassesRequiredToAvoidFailure: 5
+  artifactName: 'drop'
+  testLogPath: '$(Build.BinariesDirectory)\$(BuildPlatform)\$(BuildConfiguration)\testsOnBuildMachine.wtl'
+
+jobs:
+- job: Test${{ parameters.platform }}${{ parameters.configuration }}
+  displayName: Test ${{ parameters.platform }} ${{ parameters.configuration }}
+  variables:
+    BuildConfiguration: ${{ parameters.configuration }}
+    BuildPlatform: ${{ parameters.platform }}
+  pool: 
+    ${{ if eq(variables['System.CollectionUri'], 'https://dev.azure.com/ms/') }}:
+      name: WinDevPoolOSS-L
+    ${{ if ne(variables['System.CollectionUri'], 'https://dev.azure.com/ms/') }}:
+      name: WinDevPool-L
+    demands: ImageOverride -equals WinDevVS16-latest
+
+  steps:
+  - checkout: self
+    submodules: true
+    clean: true
+    fetchDepth: 1
+
+  - task: DownloadBuildArtifacts@0
+    inputs:
+      artifactName: ${{ parameters.artifactName }}
+
+  - task: PowerShell@2
+    displayName: 'Rationalize build platform'
+    inputs:
+      targetType: inline
+      script: |
+        $Arch = "$(BuildPlatform)"
+        If ($Arch -Eq "x86") { $Arch = "Win32" }
+        Write-Host "##vso[task.setvariable variable=RationalizedBuildPlatform]${Arch}"
+
+  - task: PowerShell@2
+    displayName: 'Run Unit Tests'
+    inputs:
+      targetType: filePath
+      filePath: build\scripts\Run-Tests.ps1
+      arguments: -MatchPattern '*unit.test*.dll' -Platform '$(RationalizedBuildPlatform)' -Configuration '$(BuildConfiguration)' -LogPath '${{ parameters.testLogPath }}' -Root "$(System.ArtifactsDirectory)\\${{ parameters.artifactName }}\\$(BuildConfiguration)\\$(BuildPlatform)\\test"
+    condition: and(and(succeeded(), ne(variables['PGOBuildMode'], 'Instrument')), or(eq(variables['BuildPlatform'], 'x64'), eq(variables['BuildPlatform'], 'x86')))
+
+  - task: PowerShell@2
+    displayName: 'Run Feature Tests (x64 only)'
+    inputs:
+      targetType: filePath
+      filePath: build\scripts\Run-Tests.ps1
+      arguments: -MatchPattern '*feature.test*.dll' -Platform '$(RationalizedBuildPlatform)' -Configuration '$(BuildConfiguration)' -LogPath '${{ parameters.testLogPath }}' -Root "$(System.ArtifactsDirectory)\\${{ parameters.artifactName }}\\$(BuildConfiguration)\\$(BuildPlatform)\\test"
+    condition: and(and(succeeded(), ne(variables['PGOBuildMode'], 'Instrument')), eq(variables['BuildPlatform'], 'x64'))
+
+  - task: PowerShell@2
+    displayName: 'Convert Test Logs from WTL to xUnit format'
+    inputs:
+      targetType: filePath
+      filePath: build\Helix\ConvertWttLogToXUnit.ps1
+      arguments: -WttInputPath '${{ parameters.testLogPath }}' -WttSingleRerunInputPath 'unused.wtl' -WttMultipleRerunInputPath 'unused2.wtl' -XUnitOutputPath 'onBuildMachineResults.xml' -TestNamePrefix '$(BuildConfiguration).$(BuildPlatform)'
+    condition: and(ne(variables['PGOBuildMode'], 'Instrument'),or(eq(variables['BuildPlatform'], 'x64'), eq(variables['BuildPlatform'], 'x86')))
+
+  - task: PublishTestResults@2
+    displayName: 'Upload converted test logs'
+    condition: ne(variables['PGOBuildMode'], 'Instrument')
+    inputs:
+      testResultsFormat: 'xUnit' # Options: JUnit, NUnit, VSTest, xUnit, cTest
+      testResultsFiles: '**/onBuildMachineResults.xml'
+      #searchFolder: '$(System.DefaultWorkingDirectory)' # Optional
+      #mergeTestResults: false # Optional
+      #failTaskOnFailedTests: false # Optional
+      testRunTitle: 'On Build Machine Tests' # Optional
+      buildPlatform: $(BuildPlatform) # Optional
+      buildConfiguration: $(BuildConfiguration) # Optional
+      #publishRunAttachments: true # Optional
+
+  - task: CopyFiles@2
+    displayName: 'Copy result logs to Artifacts'
+    inputs:
+      Contents: |
+       **/*.wtl
+       **/*onBuildMachineResults.xml
+       ${{ parameters.testLogPath }}
+      TargetFolder: '$(Build.ArtifactStagingDirectory)/$(BuildConfiguration)/$(BuildPlatform)/test-logs'
+      OverWrite: true
+      flattenFolders: true
+
+  - publish: '$(Build.ArtifactStagingDirectory)/$(BuildConfiguration)/$(BuildPlatform)/test-logs'
+    artifact: TestLogs$(BuildPlatform)$(BuildConfiguration)
diff --git a/build/scripts/Run-Tests.ps1 b/build/scripts/Run-Tests.ps1
@@ -3,10 +3,11 @@ Param(
     [Parameter(Mandatory=$true, Position=0)][string]$MatchPattern,
     [Parameter(Mandatory=$true, Position=1)][string]$Platform,
     [Parameter(Mandatory=$true, Position=2)][string]$Configuration,
-    [Parameter(Mandatory=$false, Position=3)][string]$LogPath
+    [Parameter(Mandatory=$false, Position=3)][string]$LogPath,
+    [Parameter(Mandatory=$false)][string]$Root = ".\bin\$Platform\$Configuration"
 )
 
-$testdlls = Get-ChildItem -Path ".\bin\$Platform\$Configuration" -Recurse -Filter $MatchPattern
+$testdlls = Get-ChildItem -Path "$Root" -Recurse -Filter $MatchPattern
 
 
 $args = @();
@@ -19,7 +20,7 @@ if ($LogPath)
     Write-Host "Wtt Logging Enabled";
 }
 
-&".\bin\$Platform\$Configuration\te.exe" $args $testdlls.FullName
+&"$Root\te.exe" $args $testdlls.FullName
 
 if ($lastexitcode -Ne 0) { Exit $lastexitcode }