[ci] Add signing to the internal pipeline

build-tools/automation/azure-pipelines-internal.yaml

@@ -10,10 +10,9 @@
 # branch policies for main, release/*, and feature/*. Azure Repos does not
 # support YAML `pr` triggers.
 #
-# Signing is intentionally NOT enabled yet. We will eventually release &
-# sign from here; when we do, convert this to `extends` the 1ES
-# `MicroBuild.1ES.Official.yml` template and flip the `use1ESTemplate`
-# parameters below to `true` (see azure-pipelines.yaml for reference).
+# Signing uses the in-repo Arcade templates and adapter under
+# build-tools/automation/dnceng. Do not add a Xamarin.yaml-templates
+# repository resource: dnceng/internal cannot access that repository.
 
 name: $(Build.SourceBranchName)-$(Build.SourceVersion)-$(Rev:r)
 
@@ -46,6 +45,22 @@ resources:
 # Global variables
 variables:
 - template: /build-tools/automation/yaml-templates/variables.yaml@self
+- name: _TeamName
+  value: Maui
+- ${{ if and(startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), ne(variables['Build.Reason'], 'PullRequest')) }}:
+  - name: _SignType
+    value: real
+- ${{ else }}:
+  - name: _SignType
+    value: test
+- name: _BuildConfig
+  value: Release
+- name: _BuildOfficialId
+  value: $[ format('{0}.{1}', format('{0:yyyyMMdd}', pipeline.startTime), counter(format('{0:yyyyMMdd}', pipeline.startTime), 1)) ]
+- name: DOTNET_CLI_WORKLOAD_UPDATE_NOTIFY_DISABLE
+  value: 'true'
+- name: DOTNET_SDK_VULNERABILITY_CHECK_DISABLE
+  value: 'true'
 # Override for internal pipeline
 - name: DefaultJavaSdkMajorVersion
   value: 21
@@ -270,6 +285,329 @@ stages:
 
     - template: /build-tools/automation/yaml-templates/fail-on-issue.yaml
 
+# Signing and Visual Studio workload insertion artifacts
+- stage: dotnet_prepare_release
+  displayName: Prepare .NET Release
+  dependsOn:
+  - mac_build
+  - linux_build
+  condition: and(
+      succeeded(),
+      ne(variables['Build.Reason'], 'PullRequest'),
+      ne(variables['Build.Reason'], 'Schedule')
+    )
+  jobs:
+  - template: /eng/common/templates/jobs/jobs.yml@self
+    parameters:
+      runAsPublic: false
+      enableMicrobuild: true
+      microbuildUseESRP: ${{ and(startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), ne(variables['Build.Reason'], 'PullRequest')) }}
+      jobs:
+      - job: prepare_release
+        displayName: Sign NuGets and workload MSIs
+        timeoutInMinutes: 240
+        pool:
+          name: $(NetCoreInternalPoolName)
+          demands:
+          - ImageOverride -equals $(WindowsPoolImageNetCoreInternal)
+        workspace:
+          clean: all
+        steps:
+        - checkout: self
+          clean: true
+          fetchDepth: 1
+          fetchTags: false
+          path: s/android
+
+        - checkout: android-tools
+          path: s/android-tools
+
+        - task: DownloadPipelineArtifact@2
+          displayName: Download unsigned macOS and Windows packages
+          inputs:
+            artifactName: $(NuGetArtifactName)
+            downloadPath: $(Build.StagingDirectory)\unsigned\mac-win
+
+        - task: DownloadPipelineArtifact@2
+          displayName: Download unsigned Linux packages
+          inputs:
+            artifactName: $(LinuxNuGetArtifactName)
+            downloadPath: $(Build.StagingDirectory)\unsigned\linux
+
+        - task: UseDotNet@2
+          displayName: Install .NET 8 SDK for SwixBuild
+          inputs:
+            version: 8.0.x
+
+        - task: UseDotNet@2
+          displayName: Install .NET SDK for Arcade signing
+          inputs:
+            version: $(DotNetSdkVersion).x
+            includePreviewVersions: true
+
+        - pwsh: |
+            $unsigned = "$(Build.StagingDirectory)\unsigned"
+            $artifacts = "$(Build.SourcesDirectory)\android\artifacts"
+            $shipping = Join-Path $artifacts "packages\Release\Shipping"
+            $logs = Join-Path $artifacts "log\Release"
+            $signingLogs = "$(Build.StagingDirectory)\signing-logs"
+
+            Remove-Item $artifacts -Recurse -Force -ErrorAction Ignore
+            New-Item $shipping -ItemType Directory -Force | Out-Null
+            New-Item $logs -ItemType Directory -Force | Out-Null
+            New-Item $signingLogs -ItemType Directory -Force | Out-Null
+
+            $packages = @(Get-ChildItem $unsigned -Filter "*.nupkg" -File -Recurse)
+            if ($packages.Count -eq 0) {
+              throw "The build did not publish any NuGet packages."
+            }
+            Copy-Item $packages.FullName $shipping
+
+            $workloadProps = Get-ChildItem $unsigned -Filter "vs-workload.props" -File -Recurse | Select-Object -First 1
+            if (-not $workloadProps) {
+              throw "The build did not publish vs-workload.props."
+            }
+            Copy-Item $workloadProps.FullName $shipping
+
+            $signList = Get-ChildItem $unsigned -Filter "SignList.xml" -File -Recurse | Select-Object -First 1
+            if (-not $signList) {
+              throw "The build did not publish SignList.xml."
+            }
+
+            [xml] $arcadeSignList = Get-Content $signList.FullName
+            $packageEntries = foreach ($package in $packages) {
+              $archive = [IO.Compression.ZipFile]::OpenRead($package.FullName)
+              try {
+                foreach ($entry in $archive.Entries) {
+                  [pscustomobject]@{
+                    FullName = $entry.FullName.Replace("\", "/")
+                    Name = $entry.Name
+                  }
+                }
+              } finally {
+                $archive.Dispose()
+              }
+            }
+
+            foreach ($item in @($arcadeSignList.SelectNodes("/Project/ItemGroup/*[@Include]"))) {
+              $include = $item.GetAttribute("Include").Replace("\", "/")
+              if ($include.Contains("*") -or $include.Contains("?")) {
+                $matchedEntries = if ($include.Contains("/")) {
+                  @($packageEntries | Where-Object { $_.FullName -like $include -or $_.FullName -like "*/$include" })
+                } else {
+                  @($packageEntries | Where-Object { $_.Name -like $include })
+                }
+                $names = @($matchedEntries.Name | Where-Object { $_ } | Sort-Object -Unique)
+                if ($names.Count -eq 0) {
+                  throw "Signing pattern '$include' did not match any package entries."
+                }
+                foreach ($name in $names) {
+                  $expandedItem = $item.Clone()
+                  $escapedName = $name.Replace("%", "%25").Replace(";", "%3B")
+                  $expandedItem.SetAttribute("Include", $escapedName)
+                  [void] $item.ParentNode.InsertBefore($expandedItem, $item)
+                }
+                [void] $item.ParentNode.RemoveChild($item)
+              } else {
+                if ($item.LocalName -eq "MacDeveloperSign" -or $item.LocalName -eq "MacDeveloperSignHarden") {
+                  $include = [IO.Path]::GetFileName($include)
+                }
+                $include = $include.Replace("%", "%25").Replace(";", "%3B")
+                $item.SetAttribute("Include", $include)
+              }
+            }
+
+            $skipNames = @($arcadeSignList.SelectNodes("/Project/ItemGroup/Skip") | ForEach-Object Include)
+            foreach ($item in @($arcadeSignList.SelectNodes("/Project/ItemGroup/MacDeveloperSign | /Project/ItemGroup/MacDeveloperSignHarden"))) {
+              if ($item.GetAttribute("Include") -in $skipNames) {
+                [void] $item.ParentNode.RemoveChild($item)
+              }
+            }
+
+            $certificatesByName = @{}
+            foreach ($item in $arcadeSignList.SelectNodes("/Project/ItemGroup/ThirdParty | /Project/ItemGroup/ThirdPartyScript | /Project/ItemGroup/MacDeveloperSign | /Project/ItemGroup/MacDeveloperSignHarden")) {
+              $name = $item.GetAttribute("Include")
+              $certificateGroup = $item.LocalName
+              if ($certificatesByName.ContainsKey($name) -and $certificatesByName[$name] -ne $certificateGroup) {
+                throw "Signing entry '$name' is assigned to both '$($certificatesByName[$name])' and '$certificateGroup'."
+              }
+              $certificatesByName[$name] = $certificateGroup
+            }
+
+            $arcadeSignListPath = "$(Build.StagingDirectory)\ArcadeSignList.xml"
+            $arcadeSignList.Save($arcadeSignListPath)
+            Write-Host "##vso[task.setvariable variable=AndroidSignList]$arcadeSignListPath"
+
+            $signVerify = Get-ChildItem $unsigned -Filter "SignVerifyIgnore.txt" -File -Recurse | Select-Object -First 1
+            if (-not $signVerify) {
+              throw "The build did not publish SignVerifyIgnore.txt."
+            }
+            Copy-Item $signVerify.FullName "$(Build.StagingDirectory)\SignVerifyIgnore.txt"
+
+            $inputNames = $packages.Name | Sort-Object -Unique
+            $inputNames | ConvertTo-Json | Set-Content "$(Build.StagingDirectory)\unsigned-package-names.json"
+
+            $shortHash = "$(Build.SourceVersion)".Substring(0, 7)
+            Write-Host "##vso[task.setvariable variable=BUILD_SOURCEVERSIONSHORT]$shortHash"
+          displayName: Normalize unsigned artifacts for Arcade
+
+        - pwsh: |
+            $adapter = "$(Agent.TempDirectory)\android-arcade"
+            Remove-Item $adapter -Recurse -Force -ErrorAction Ignore
+            Copy-Item "$(Build.SourcesDirectory)\android\build-tools\automation\dnceng\arcade" $adapter -Recurse
+            Copy-Item "$(Build.SourcesDirectory)\android\eng\common" "$adapter\eng\common" -Recurse
+
+            $globalJson = Get-Content "$(Build.SourcesDirectory)\android\global.json" -Raw | ConvertFrom-Json -AsHashtable
+            $globalJson["tools"] = [ordered]@{
+              dotnet = (& dotnet --version)
+            }
+            $globalJson | ConvertTo-Json -Depth 5 | Set-Content "$adapter\global.json"
+
+            Write-Host "##vso[task.setvariable variable=ArcadeAdapterRoot]$adapter"
+          displayName: Prepare isolated Arcade adapter
+
+        - pwsh: |
+            $properties = @(
+              "/p:RepositoryEngineeringDir=$(ArcadeAdapterRoot)\eng\",
+              "/p:AndroidRepoRoot=$(Build.SourcesDirectory)\android",
+              "/p:AndroidSignList=$(AndroidSignList)",
+              "/p:ArtifactsDir=$(Build.SourcesDirectory)\android\artifacts\",
+              "/p:NuGetPackageRoot=$(Build.SourcesDirectory)\android\.packages\",
+              "/p:RestorePackagesPath=$(Build.SourcesDirectory)\android\.packages\",
+              "/p:RestoreConfigFile=$(Build.SourcesDirectory)\android\NuGet.config",
+              "/p:OfficialBuildId=$(_BuildOfficialId)",
+              "/p:DotNetSignType=$(_SignType)",
+              "/p:MicroBuildOverridePluginDirectory=$(Agent.TempDirectory)\MicroBuild\Plugins",
+              "/p:RepositoryName=dotnet/android"
+            )
+
+            & "$(ArcadeAdapterRoot)\eng\common\build.ps1" `
+              -configuration Release `
+              -msbuildEngine dotnet `
+              -projects "$(ArcadeAdapterRoot)\Workloads.proj" `
+              -restore `
+              -sign `
+              -ci `
+              -warnAsError:$false `
+              -binaryLogName "$(Build.StagingDirectory)\signing-logs\sign-nugets.binlog" `
+              @properties
+          displayName: Sign prebuilt NuGet packages with Arcade
+          env:
+            BUILD_REPOSITORY_NAME: dotnet/android
+            BUILD_REPOSITORY_URI: https://github.com/dotnet/android
+            NUGET_CONFIG: $(Build.SourcesDirectory)\android\NuGet.config
+            NUGET_PACKAGES: $(Build.SourcesDirectory)\android\.packages
+            SYSTEM_ACCESSTOKEN: $(System.AccessToken)
+
+        - pwsh: |
+            $properties = @(
+              "/p:RepositoryEngineeringDir=$(ArcadeAdapterRoot)\eng\",
+              "/p:AndroidRepoRoot=$(Build.SourcesDirectory)\android",
+              "/p:AndroidSignList=$(AndroidSignList)",
+              "/p:ArtifactsDir=$(Build.SourcesDirectory)\android\artifacts\",
+              "/p:NuGetPackageRoot=$(Build.SourcesDirectory)\android\.packages\",
+              "/p:RestorePackagesPath=$(Build.SourcesDirectory)\android\.packages\",
+              "/p:RestoreConfigFile=$(Build.SourcesDirectory)\android\NuGet.config",
+              "/p:OfficialBuildId=$(_BuildOfficialId)",
+              "/p:DotNetSignType=$(_SignType)",
+              "/p:SignWorkloadMsis=true",
+              "/p:MicroBuildOverridePluginDirectory=$(Agent.TempDirectory)\MicroBuild\Plugins",
+              "/p:RepositoryName=dotnet/android"
+            )
+
+            & "$(ArcadeAdapterRoot)\eng\common\build.ps1" `
+              -configuration Release `
+              -msbuildEngine dotnet `
+              -projects "$(ArcadeAdapterRoot)\Workloads.proj" `
+              -build `
+              -sign `
+              -ci `
+              -warnAsError:$false `
+              -binaryLogName "$(Build.StagingDirectory)\signing-logs\workload-msis.binlog" `
+              @properties
+          displayName: Build and sign workload MSIs with Arcade
+          env:
+            BUILD_REPOSITORY_NAME: dotnet/android
+            BUILD_REPOSITORY_URI: https://github.com/dotnet/android
+            NUGET_CONFIG: $(Build.SourcesDirectory)\android\NuGet.config
+            NUGET_PACKAGES: $(Build.SourcesDirectory)\android\.packages
+            SYSTEM_ACCESSTOKEN: $(System.AccessToken)
+
+        - pwsh: |
+            $properties = @(
+              "/p:RepositoryEngineeringDir=$(ArcadeAdapterRoot)\eng\",
+              "/p:AndroidRepoRoot=$(Build.SourcesDirectory)\android",
+              "/p:AndroidSignList=$(AndroidSignList)",
+              "/p:ArtifactsDir=$(Build.SourcesDirectory)\android\artifacts\",
+              "/p:NuGetPackageRoot=$(Build.SourcesDirectory)\android\.packages\",
+              "/p:RestorePackagesPath=$(Build.SourcesDirectory)\android\.packages\",
+              "/p:RestoreConfigFile=$(Build.SourcesDirectory)\android\NuGet.config",
+              "/p:OfficialBuildId=$(_BuildOfficialId)",
+              "/p:DotNetSignType=$(_SignType)",
+              "/p:MicroBuildOverridePluginDirectory=$(Agent.TempDirectory)\MicroBuild\Plugins",
+              "/p:RepositoryName=dotnet/android"
+            )
+
+            & "$(ArcadeAdapterRoot)\eng\common\build.ps1" `
+              -configuration Release `
+              -msbuildEngine dotnet `
+              -projects "$(ArcadeAdapterRoot)\Workloads.proj" `
+              -sign `
+              -ci `
+              -warnAsError:$false `
+              -binaryLogName "$(Build.StagingDirectory)\signing-logs\sign-msi-nugets.binlog" `
+              @properties
+          displayName: Sign workload MSI NuGet packages with Arcade
+          env:
+            BUILD_REPOSITORY_NAME: dotnet/android
+            BUILD_REPOSITORY_URI: https://github.com/dotnet/android
+            NUGET_CONFIG: $(Build.SourcesDirectory)\android\NuGet.config
+            NUGET_PACKAGES: $(Build.SourcesDirectory)\android\.packages
+            SYSTEM_ACCESSTOKEN: $(System.AccessToken)
+
+        - pwsh: |
+            $shipping = "$(Build.SourcesDirectory)\android\artifacts\packages\Release\Shipping"
+            $inputNames = @(Get-Content "$(Build.StagingDirectory)\unsigned-package-names.json" -Raw | ConvertFrom-Json)
+            foreach ($name in $inputNames) {
+              if (-not (Test-Path (Join-Path $shipping $name) -PathType Leaf)) {
+                throw "Signed output is missing '$name'."
+              }
+            }
+
+            $msiPackages = @(Get-ChildItem $shipping -Filter "*.nupkg" -File |
+              Where-Object { $_.Name -notin $inputNames })
+            if ($msiPackages.Count -eq 0) {
+              throw "The workload build did not generate any MSI NuGet packages."
+            }
+
+            $msiDirectory = "$(Build.StagingDirectory)\msi-nupkgs"
+            New-Item $msiDirectory -ItemType Directory -Force | Out-Null
+            Copy-Item $msiPackages.FullName $msiDirectory
+          displayName: Validate signed package outputs
+
+        - task: MicroBuildCodesignVerify@3
+          displayName: Verify signed workload artifacts
+          inputs:
+            TargetFolders: |
+              $(Build.SourcesDirectory)\android\artifacts\packages\Release\Shipping\manifests
+              $(Build.SourcesDirectory)\android\artifacts\packages\Release\Shipping\manifests-packs
+              $(Build.StagingDirectory)\msi-nupkgs
+            ExcludeSNVerify: true
+            ApprovalListPathForCerts: $(Build.StagingDirectory)\SignVerifyIgnore.txt
+
+        - task: PublishPipelineArtifact@1
+          displayName: Publish signed NuGet packages
+          inputs:
+            artifactName: nuget-signed
+            targetPath: $(Build.SourcesDirectory)\android\artifacts\packages\Release\Shipping
+
+        - task: PublishPipelineArtifact@1
+          displayName: Publish signing logs
+          condition: always()
+          inputs:
+            artifactName: signing-logs-$(System.StageAttempt)-$(System.JobAttempt)
+            targetPath: $(Build.StagingDirectory)\signing-logs
+
 # MSBuild Tests Stage
 - stage: msbuild_dotnet
   displayName: MSBuild Tests