[release/8.0.4xx] Backport maestro and artifact drop infra improvements

Directory.Build.props

@@ -39,6 +39,8 @@
     <AndroidPackVersionMajorMinor>$(AndroidPackVersion.Substring(0,$(AndroidPackVersionPatchIndex)))</AndroidPackVersionMajorMinor>
     <AndroidPackVersionPatch>$(AndroidPackVersion.Substring($([MSBuild]::Add($(AndroidPackVersionPatchIndex), 1))))</AndroidPackVersionPatch>
     <AndroidPackVersionSuffix>rtm</AndroidPackVersionSuffix>
+    <IsStableBuild>false</IsStableBuild>
+    <IsStableBuild Condition=" '$(AndroidPackVersionSuffix)' == 'rtm' ">true</IsStableBuild>
   </PropertyGroup>
 
   <!-- Common <PackageReference/> versions -->

build-tools/automation/azure-pipelines.yaml

@@ -496,7 +496,7 @@ extends:
           use1ESTemplate: true
 
       # Check - "Xamarin.Android (Prepare .NET Release Convert NuGet to MSI)"
-      - template: nuget-msi-convert/job/v3.yml@yaml-templates
+      - template: nuget-msi-convert/job/v4.yml@yaml-templates
         parameters:
           yamlResourceName: yaml-templates
           dependsOn: sign_net_mac_win
@@ -505,126 +505,125 @@ extends:
             !*Darwin*
           propsArtifactName: $(NuGetArtifactName)
           signType: $(MicroBuildSignType)
-          use1ESTemplate: true
+          preConvertSteps:
+          - task: DownloadPipelineArtifact@2
+            inputs:
+              artifactName: nuget-signed
+              artifactPatterns: |
+                *Darwin*
+              downloadPath: $(Build.StagingDirectory)\drop\nugets
+          - task: DownloadPipelineArtifact@2
+            inputs:
+              artifactName: nuget-linux-signed
+              downloadPath: $(Build.StagingDirectory)\drop\nugets
           postConvertSteps:
           - task: DownloadPipelineArtifact@2
             inputs:
               artifactName: $(NuGetArtifactName)
               downloadPath: $(Build.StagingDirectory)\sign-verify
               patterns: |
                 **/SignVerifyIgnore.txt
-
           - task: MicroBuildCodesignVerify@3
             displayName: verify signed msi content
             inputs:
               TargetFolders: |
                 $(Build.ArtifactStagingDirectory)\bin\manifests
-                $(Build.ArtifactStagingDirectory)\bin\manifests-multitarget
+                $(Build.ArtifactStagingDirectory)\bin\manifests-packs
+                $(Build.ArtifactStagingDirectory)\bin\msi-nupkgs
               ExcludeSNVerify: true
               ApprovalListPathForCerts: $(Build.StagingDirectory)\sign-verify\SignVerifyIgnore.txt
 
-      # Check - "Xamarin.Android (Prepare .NET Release Push Internal)"
+      # Check - "Xamarin.Android (Prepare .NET Release Publish symbols and Push to Maestro)"
       - job: push_signed_nugets
-        displayName: Push Internal
+        displayName: Publish symbols and Push to Maestro
         dependsOn:
         - nuget_convert
         - sign_net_linux
         condition: and(eq(dependencies.nuget_convert.result, 'Succeeded'), eq(dependencies.sign_net_linux.result, 'Succeeded'))
-        timeoutInMinutes: 60
+        timeoutInMinutes: 90
         pool:
           name: AzurePipelines-EO
           image: $(WindowsPoolImage1ESPT)
           os: windows
         workspace:
           clean: all
-        variables:
-        - ${{ if eq(variables['MicroBuildSignType'], 'Real') }}:
-          - group: Publish-Build-Assets
         templateContext:
           outputs:
-          - output: nuget
-            condition: and(succeeded(), eq('${{ parameters.pushXAPackagesToMaestro }}', 'true'))
-            useDotNetTask: false # The default is false to use the NuGetCommand task. Set to true to use the DotNetCoreCLI task to publish packages.
-            packagesToPush: $(Build.StagingDirectory)\nuget-signed\*.nupkg
-            packageParentPath: $(Build.StagingDirectory)\nuget-signed
-            nuGetFeedType: external
-            publishFeedCredentials: $(DotNetFeedCredential)
+          - output: artifactsDrop
+            dropServiceURI: https://devdiv.artifacts.visualstudio.com/DefaultCollection
+            buildNumber: $(ReleaseDropPrefix)/symbols
+            dropMetadataContainerName: DropMetadata-$(Build.BuildId)-symbols-$(System.JobAttempt)
+            sourcePath: $(Build.StagingDirectory)\symbols
+            retentionDays: 120
+            toLowerCase: false
         steps:
         - checkout: self
+          clean: true
+          submodules: recursive
 
-        - task: DownloadPipelineArtifact@2
+        - task: UseDotNet@2
+          displayName: Install .NET 9.x
           inputs:
-            artifactName: nuget-signed
-            downloadPath: $(Build.StagingDirectory)\nuget-signed
+            version: 9.x
+            includePreviewVersions: true
 
+        # Download symbols to be published to the symbols artifact drop declared above
         - task: DownloadPipelineArtifact@2
           inputs:
-            artifactName: nuget-linux-signed
-            downloadPath: $(Build.StagingDirectory)\nuget-signed
+            artifactName: $(WindowsToolchainPdbArtifactName)
+            downloadPath: $(Build.StagingDirectory)\symbols
 
         - task: DownloadPipelineArtifact@2
           inputs:
-            artifactName: vs-msi-nugets
-            downloadPath: $(Build.StagingDirectory)\nuget-signed
+            artifactName: DropMetadata-$(Build.BuildId)-nugets-$(System.JobAttempt)
+            downloadPath: $(Build.StagingDirectory)\metadata
+          displayName: Download nugets drop metadata
 
-        - task: DownloadPipelineArtifact@2
+        - powershell: |
+            $jsonContent = Get-Content -Path "$(Build.StagingDirectory)\metadata\VSTSDrop.json" -Raw | ConvertFrom-Json
+            $dropPrefix = $jsonContent.VstsDropBuildArtifact.VstsDropUrl -replace 'https://devdiv.artifacts.visualstudio.com/DefaultCollection/_apis/drop/drops/' -replace '/nugets'
+            Write-Host "##vso[task.setvariable variable=ReleaseDropPrefix]$dropPrefix"
+          displayName: Set variable ReleaseDropPrefix
+
+        # Download nugets drop created by nuget-msi-convert/job/v4.yml and publish to maestro
+        - task: ms-vscs-artifact.build-tasks.artifactDropDownloadTask-1.artifactDropDownloadTask@1
+          displayName: Download $(ReleaseDropPrefix)/nugets
           inputs:
-            artifactName: $(WindowsToolchainPdbArtifactName)
-            downloadPath: $(Build.StagingDirectory)\nuget-signed
+            dropServiceURI: https://devdiv.artifacts.visualstudio.com/DefaultCollection
+            buildNumber: $(ReleaseDropPrefix)/nugets
+            destinationPath: $(Build.StagingDirectory)\nuget-signed
 
-        - template: templates\common\upload-vs-insertion-artifacts.yml@sdk-insertions
-          parameters:
-            githubToken: $(GitHub.Token)
-            githubContext: $(NupkgCommitStatusName)
-            blobName: $(NupkgCommitStatusName)
-            packagePrefix: xamarin-android
-            artifactsPath: $(Build.StagingDirectory)\nuget-signed
-            yamlResourceName: yaml-templates
-
-        - template: templates\common\upload-vs-insertion-artifacts.yml@sdk-insertions
-          parameters:
-            githubToken: $(GitHub.Token)
-            githubContext: $(VSDropCommitStatusName)
-            blobName: $(VSDropCommitStatusName)
-            packagePrefix: xamarin-android
-            artifactsPath: $(Build.StagingDirectory)\$(VSDropCommitStatusName)
-            yamlResourceName: yaml-templates
-            downloadSteps:
-            - task: DownloadPipelineArtifact@2
-              inputs:
-                artifactName: vsdrop-signed
-                downloadPath: $(Build.StagingDirectory)\$(VSDropCommitStatusName)
-
-        - template: templates\common\upload-vs-insertion-artifacts.yml@sdk-insertions
-          parameters:
-            githubToken: $(GitHub.Token)
-            githubContext: $(MultiTargetVSDropCommitStatusName)
-            blobName: $(MultiTargetVSDropCommitStatusName)
-            packagePrefix: xamarin-android
-            artifactsPath: $(Build.StagingDirectory)\$(MultiTargetVSDropCommitStatusName)
-            yamlResourceName: yaml-templates
-            downloadSteps:
-            - task: DownloadPipelineArtifact@2
-              inputs:
-                artifactName: vsdrop-multitarget-signed
-                downloadPath: $(Build.StagingDirectory)\$(MultiTargetVSDropCommitStatusName)
-
-        - powershell: >-
-            & dotnet build -v:n -c $(XA.Build.Configuration)
-            -t:PushManifestToBuildAssetRegistry
-            -p:BuildAssetRegistryToken=$(MaestroAccessToken)
-            -p:OutputPath=$(Build.StagingDirectory)\nuget-signed\
-            $(System.DefaultWorkingDirectory)\build-tools\create-packs\Microsoft.Android.Sdk.proj
-            -bl:$(System.DefaultWorkingDirectory)\bin\Build$(XA.Build.Configuration)\push-bar-manifest.binlog
+        - task: DotNetCoreCLI@2
+          displayName: build Xamarin.Android.Tools.BootstrapTasks.sln
+          inputs:
+            projects: $(System.DefaultWorkingDirectory)\Xamarin.Android.BootstrapTasks.sln
+            arguments: -c $(XA.Build.Configuration) -bl:$(System.DefaultWorkingDirectory)\bin\Build$(XA.Build.Configuration)\bootstrap.binlog
+          condition: and(succeeded(), eq('${{ parameters.pushXAPackagesToMaestro }}', 'true'))
+
+        - task: AzureCLI@2
           displayName: generate and publish BAR manifest
+          inputs:
+            azureSubscription: "Darc: Maestro Production"
+            scriptType: ps
+            scriptLocation: inlineScript
+            inlineScript: >-
+              dotnet build $(System.DefaultWorkingDirectory)\build-tools\create-packs\Microsoft.Android.Sdk.proj
+              -t:PushManifestToBuildAssetRegistry
+              -p:OutputPath=$(Build.StagingDirectory)\nuget-signed\
+              -c $(XA.Build.Configuration) -bl:$(System.DefaultWorkingDirectory)\bin\Build$(XA.Build.Configuration)\push-bar-manifest.binlog
           condition: and(succeeded(), eq('${{ parameters.pushXAPackagesToMaestro }}', 'true'))
 
-        - powershell: |
-            $versionEndpoint = 'https://maestro-prod.westus2.cloudapp.azure.com/api/assets/darc-version?api-version=2019-01-16'
-            $darcVersion = $(Invoke-WebRequest -Uri $versionEndpoint -UseBasicParsing).Content
-            $arcadeServicesSource = 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json'
-            & dotnet tool update microsoft.dotnet.darc --version "$darcVersion" --add-source "$arcadeServicesSource" --tool-path $(Agent.ToolsDirectory)\darc -v n
-            & $(Agent.ToolsDirectory)\darc\darc add-build-to-channel --default-channels --id $(BARBuildId) --publishing-infra-version 3 --skip-assets-publishing --password $(MaestroAccessToken) --azdev-pat $(publishing-dnceng-devdiv-code-r-build-re)
+        - task: AzureCLI@2
+          inputs:
+            azureSubscription: "Darc: Maestro Production"
+            scriptType: ps
+            scriptLocation: inlineScript
+            inlineScript: |
+              $versionEndpoint = 'https://maestro.dot.net/api/assets/darc-version?api-version=2019-01-16'
+              $darcVersion = $(Invoke-WebRequest -Uri $versionEndpoint -UseBasicParsing).Content
+              $arcadeServicesSource = 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json'
+              & dotnet tool update microsoft.dotnet.darc --version "$darcVersion" --add-source "$arcadeServicesSource" --tool-path $(Agent.ToolsDirectory)\darc -v n
+              & $(Agent.ToolsDirectory)\darc\darc add-build-to-channel --default-channels --id $(BARBuildId) --ci --publishing-infra-version 3 --azdev-pat $(System.AccessToken)
           displayName: add build to default darc channel
           condition: and(succeeded(), eq('${{ parameters.pushXAPackagesToMaestro }}', 'true'))
 

build-tools/create-packs/Directory.Build.targets

@@ -7,10 +7,9 @@
 
   <Import Project="License.targets" />
   <Import Project="..\..\build-tools\installers\create-installers.targets" />
-  <Import Project="Sdk.props" Sdk="Microsoft.DotNet.SharedFramework.Sdk" Version="$(MicrosoftDotNetBuildTasksFeedPackageVersion)" />
+  <Import Project="Sdk.props" Sdk="Microsoft.DotNet.SharedFramework.Sdk" Version="8.0.0-beta.24413.2" />
 
   <UsingTask TaskName="CreateFrameworkListFile" AssemblyFile="$(DotNetSharedFrameworkTaskFile)"/>
-  <UsingTask TaskName="GenerateBuildManifest" AssemblyFile="$(_MicrosoftDotNetBuildTasksFeedTaskDir)Microsoft.DotNet.Build.Tasks.Feed.dll" />
   <UsingTask TaskName="Xamarin.Android.BuildTools.PrepTasks.ReplaceFileContents" AssemblyFile="$(PrepTasksAssembly)" />
 
   <!-- https://github.com/dotnet/runtime/blob/0647ec314948904319da5eb15e9931f7c85ed1e2/src/installer/pkg/projects/Directory.Build.targets#L281 -->
@@ -164,15 +163,27 @@
     <RemoveDir Directories="@(_PackFoldersToDelete)" />
   </Target>
 
-  <Target Name="PushManifestToBuildAssetRegistry" >
+  <!-- https://github.com/dotnet/arcade/blob/00d6decc59f5030c2399a64fd3e4f6e8e11bacca/Documentation/DependencyFlowOnboardingWithoutArcade.md -->
+  <Target Name="PushManifestToBuildAssetRegistry" 
+      DependsOnTargets="GetXAVersionInfo" >
+    <PropertyGroup>
+      <ArtifactsLogDir>$(OutputPath)</ArtifactsLogDir>
+      <AssetManifestFileName>Assets.xml</AssetManifestFileName>
+      <AssetManifestPath>$(ArtifactsLogDir)AssetManifest\$(AssetManifestFileName)</AssetManifestPath>
+    </PropertyGroup>
+
+    <Error Condition="Exists($(AssetManifestPath))" Text="The manifest file '$(AssetManifestPath)' already exists." />
+
     <ItemGroup>
-      <BuildArtifacts Include="$(OutputPath)*.nupkg" />
+      <ItemsToPush Include="$(OutputPath)*.nupkg" />
     </ItemGroup>
 
-    <Error Condition="'@(BuildArtifacts)' == ''" Text="No packages to create manifest from." />
+    <Error Condition="'@(ItemsToPush)' == ''" Text="No packages to push." />
+
+    <Message Text="Publishing %(ItemsToPush.Identity)" Importance="normal" />
 
     <ItemGroup>
-      <ManifestBuildData Include="InitialAssetsLocation=https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet7/nuget/v3/index.json" />
+      <ManifestBuildData Include="InitialAssetsLocation=" />
       <ManifestBuildData Include="AzureDevOpsBuildId=$(BUILD_BUILDID)" />
       <ManifestBuildData Include="AzureDevOpsBuildDefinitionId=$(SYSTEM_DEFINITIONID)" />
       <ManifestBuildData Include="AzureDevOpsProject=$(SYSTEM_TEAMPROJECT)" />
@@ -181,25 +192,26 @@
       <ManifestBuildData Include="AzureDevOpsBranch=$(BUILD_SOURCEBRANCH)" />
     </ItemGroup>
 
-    <GenerateBuildManifest
-        Artifacts="@(BuildArtifacts)"
-        OutputPath="$(OutputPath)bar-manifests\AssetManifest.xml"
-        BuildId="$(BUILD_BUILDNUMBER)"
-        BuildData="@(ManifestBuildData)"
-        RepoUri="$(BUILD_REPOSITORY_URI)"
-        RepoBranch="$(BUILD_SOURCEBRANCH)"
-        RepoCommit="$(BUILD_SOURCEVERSION)"
+    <PushToBuildStorage
+        ItemsToPush="@(ItemsToPush)"
+        IsStableBuild="$(IsStableBuild)"
+        ManifestBuildData="@(ManifestBuildData)"
+        ManifestRepoUri="$(BUILD_REPOSITORY_NAME)"
+        ManifestBranch="$(BUILD_SOURCEBRANCH)"
+        ManifestBuildId="$(BUILD_BUILDNUMBER)"
+        ManifestCommit="$(BUILD_SOURCEVERSION)"
+        AssetManifestPath="$(AssetManifestPath)"
         PublishingVersion="3" />
 
     <MSBuild
         Targets="Restore"
         Projects="$(PkgMicrosoft_DotNet_Arcade_Sdk)\tools\SdkTasks\PublishBuildAssets.proj"
-        Properties="Configuration=$(Configuration);RepoRoot=$(XamarinAndroidSourcePath);VersionPrefix=$(AndroidPackVersion)"
+        Properties="Configuration=$(Configuration);RepoRoot=$(XamarinAndroidSourcePath);VersionPrefix=$(AndroidPackVersionLong)"
     />
 
     <MSBuild
         Projects="$(PkgMicrosoft_DotNet_Arcade_Sdk)\tools\SdkTasks\PublishBuildAssets.proj"
-        Properties="Configuration=$(Configuration);RepoRoot=$(XamarinAndroidSourcePath);VersionPrefix=$(AndroidPackVersion);ManifestsPath=$(OutputPath)bar-manifests;MaestroApiEndpoint=https://maestro-prod.westus2.cloudapp.azure.com"
+        Properties="Configuration=$(Configuration);RepoRoot=$(XamarinAndroidSourcePath);VersionPrefix=$(AndroidPackVersionLong);ManifestsPath=$(ArtifactsLogDir)AssetManifest;MaestroApiEndpoint=https://maestro.dot.net"
     />
   </Target>
 

eng/Version.Details.xml

@@ -30,9 +30,9 @@
       <Uri>https://github.com/dotnet/arcade</Uri>
       <Sha>70831f0d126fe88b81d7dc8de11358e17a5ce364</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.Build.Tasks.Feed" Version="7.0.0-beta.22103.1">
+    <Dependency Name="Microsoft.DotNet.Build.Tasks.Feed" Version="9.0.0-beta.24408.2">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>70831f0d126fe88b81d7dc8de11358e17a5ce364</Sha>
+      <Sha>60ae233c3d77f11c5fdb53e570b64d503b13ba59</Sha>
     </Dependency>
     <Dependency Name="Microsoft.TemplateEngine.Tasks" Version="7.0.100-rc.1.22410.7">
       <Uri>https://github.com/dotnet/templating</Uri>

eng/Versions.props

@@ -6,7 +6,7 @@
     <MicrosoftNETILLinkTasksPackageVersion>8.0.8</MicrosoftNETILLinkTasksPackageVersion>
     <MicrosoftNETCoreAppRefPackageVersion>8.0.8</MicrosoftNETCoreAppRefPackageVersion>
     <MicrosoftDotNetApiCompatPackageVersion>7.0.0-beta.22103.1</MicrosoftDotNetApiCompatPackageVersion>
-    <MicrosoftDotNetBuildTasksFeedPackageVersion>7.0.0-beta.22103.1</MicrosoftDotNetBuildTasksFeedPackageVersion>
+    <MicrosoftDotNetBuildTasksFeedPackageVersion>9.0.0-beta.24408.2</MicrosoftDotNetBuildTasksFeedPackageVersion>
     <MicrosoftNETWorkloadEmscriptenCurrentManifest80100Version>8.0.8</MicrosoftNETWorkloadEmscriptenCurrentManifest80100Version>
     <MicrosoftNETWorkloadEmscriptenPackageVersion>$(MicrosoftNETWorkloadEmscriptenCurrentManifest80100Version)</MicrosoftNETWorkloadEmscriptenPackageVersion>
     <MicrosoftTemplateEngineTasksPackageVersion>7.0.100-rc.1.22410.7</MicrosoftTemplateEngineTasksPackageVersion>