diff --git a/FSharp.sln b/FSharp.sln index 094d4302e5c..83933b62da3 100644 --- a/FSharp.sln +++ b/FSharp.sln @@ -1,7 +1,7 @@  Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Version 17 -VisualStudioVersion = 17.1.32113.165 +# Visual Studio Version 18 +VisualStudioVersion = 18.0.11104.47 d18.0 MinimumVisualStudioVersion = 10.0.40219.1 Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "FSharp.Core", "src\FSharp.Core\FSharp.Core.fsproj", "{DED3BBD7-53F4-428A-8C9F-27968E768605}" EndProject @@ -166,6 +166,13 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = ".VisualStudio", ".VisualStu docs\release-notes\.VisualStudio\17.9.md = docs\release-notes\.VisualStudio\17.9.md EndProjectSection EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "eng", "eng", "{79E058E4-79E9-4178-AFA4-A87C45373379}" + ProjectSection(SolutionItems) = preProject + eng\Version.Details.props = eng\Version.Details.props + eng\Version.Details.xml = eng\Version.Details.xml + eng\Versions.props = eng\Versions.props + EndProjectSection +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU diff --git a/azure-pipelines-PR.yml b/azure-pipelines-PR.yml index cfa5411ee33..4b76589ea7a 100644 --- a/azure-pipelines-PR.yml +++ b/azure-pipelines-PR.yml @@ -5,6 +5,7 @@ trigger: - main - dev16.1 - feature/* + - feature* - release/* paths: include: @@ -31,6 +32,7 @@ pr: - main - dev16.1 - feature/* + - feature* - release/* paths: include: @@ -55,6 +57,10 @@ variables: value: Release - name: _PublishUsingPipelines value: true + - name: _WindowsMachineQueueName + value: windows.vs2026preview.scout.amd64.open + - name: LinuxMachineQueueName + value: build.azurelinux.3.amd64.open - name: VisualStudioDropName value: Products/$(System.TeamProject)/$(Build.Repository.Name)/$(Build.SourceBranchName)/$(Build.BuildNumber) - name: Codeql.Enabled @@ -101,7 +107,7 @@ stages: value: Test pool: name: $(DncEngPublicBuildPool) - demands: ImageOverride -equals $(WindowsMachineQueueName) + demands: ImageOverride -equals $(_WindowsMachineQueueName) timeoutInMinutes: 90 strategy: maxParallel: 2 @@ -138,7 +144,8 @@ stages: # Check FSComp.txt error code sorting and code formatting - job: CheckCodeFormatting pool: - vmImage: $(UbuntuMachineQueueName) + name: $(DncEngPublicBuildPool) + demands: ImageOverride -equals $(LinuxMachineQueueName) steps: - checkout: self clean: true @@ -157,6 +164,10 @@ stages: env: DOTNET_ROLL_FORWARD_TO_PRERELEASE: 1 displayName: Check code formatting (run 'dotnet fantomas .' to fix) + - script: ./eng/common/dotnet.sh fsi eng/tests/TestSplit.fsx --validate + env: + DOTNET_ROLL_FORWARD_TO_PRERELEASE: 1 + displayName: Validate test projects are registered in TestSplit.fsx # Check whether package with current version has been published to nuget.org # We will try to restore both FSharp.Core and FCS and if restore is _successful_, package version needs to be bumped. @@ -165,7 +176,8 @@ stages: # It is also helping the release notes automation to be up to date with packages versions. - job: Check_Published_Package_Versions pool: - vmImage: $(UbuntuMachineQueueName) + name: $(DncEngPublicBuildPool) + demands: ImageOverride -equals $(LinuxMachineQueueName) strategy: maxParallel: 2 matrix: @@ -176,17 +188,8 @@ stages: steps: - checkout: self clean: true - # We first download a publicly available .NET SDK. That one has support for `path` in global.json. dotnet.cmd script can then download a version which is not yet shipped, but matches global.json. - - task: UseDotNet@2 - displayName: install SDK - inputs: - packageType: sdk - version: '10.x' - includePreviewVersions: true - workingDirectory: $(Build.SourcesDirectory) - installationPath: $(Build.SourcesDirectory)/.dotnet - script: ./eng/common/dotnet.sh - - pwsh: ./check.ps1 -project $(_project) + - script: ./check.sh $(_project) workingDirectory: $(Build.SourcesDirectory)/buildtools/checkpackages env: DOTNET_ROLL_FORWARD_TO_PRERELEASE: 1 @@ -210,18 +213,17 @@ stages: - name: 'Managed' container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9' buildArguments: '--source-build' + - name: 'Managed_MonoRuntime' + container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9' + buildArguments: '--source-build /p:DotNetBuildUseMonoRuntime=true' enableTelemetry: true helixRepo: dotnet/fsharp jobs: - job: WindowsLangVersionPreview pool: - # The PR build definition sets this variable: - # WindowsMachineQueueName=Windows.vs2022.amd64.open - # and there is an alternate build definition that sets this to a queue that is always scouting the - # next preview of Visual Studio. name: $(DncEngPublicBuildPool) - demands: ImageOverride -equals $(WindowsMachineQueueName) + demands: ImageOverride -equals $(_WindowsMachineQueueName) timeoutInMinutes: 120 steps: - checkout: self @@ -230,7 +232,7 @@ stages: - script: eng\CIBuildNoPublish.cmd -compressallmetadata -configuration Release /p:FSharpLangVersion=preview env: DOTNET_DbgEnableMiniDump: 1 - DOTNET_DbgMiniDumpType: 3 # Triage dump, 1 for mini, 2 for Heap, 3 for triage, 4 for full. Don't use 4 unless you know what you're doing. + DOTNET_DbgMiniDumpType: 2 # 1=mini, 2=heap, 3=triage, 4=full. Heap dumps include managed object data for debugging. DOTNET_DbgMiniDumpName: $(Build.SourcesDirectory)\artifacts\log\Release\$(Build.BuildId)-%e-%p-%t.dmp NativeToolsOnMachine: true displayName: Build @@ -240,7 +242,7 @@ stages: condition: always() continueOnError: true inputs: - PathToPublish: '$(Build.SourcesDirectory)\artifacts\log/Release\Build.VisualFSharp.sln.binlog' + PathToPublish: '$(Build.SourcesDirectory)\artifacts\log/Release\Build.VisualFSharp.slnx.binlog' ArtifactName: 'Windows Release build binlogs' ArtifactType: Container parallel: true @@ -256,12 +258,8 @@ stages: - job: WindowsNoRealsig_testCoreclr pool: - # The PR build definition sets this variable: - # WindowsMachineQueueName=Windows.vs2022.amd64.open - # and there is an alternate build definition that sets this to a queue that is always scouting the - # next preview of Visual Studio. name: $(DncEngPublicBuildPool) - demands: ImageOverride -equals $(WindowsMachineQueueName) + demands: ImageOverride -equals $(_WindowsMachineQueueName) timeoutInMinutes: 120 steps: - checkout: self @@ -270,7 +268,7 @@ stages: - script: eng\CIBuildNoPublish.cmd -compressallmetadata -buildnorealsig -testCoreclr -configuration Release env: DOTNET_DbgEnableMiniDump: 1 - DOTNET_DbgMiniDumpType: 3 # Triage dump, 1 for mini, 2 for Heap, 3 for triage, 4 for full. Don't use 4 unless you know what you're doing. + DOTNET_DbgMiniDumpType: 2 # 1=mini, 2=heap, 3=triage, 4=full. Heap dumps include managed object data for debugging. DOTNET_DbgMiniDumpName: $(Build.SourcesDirectory)\artifacts\log\Release\$(Build.BuildId)-%e-%p-%t.dmp NativeToolsOnMachine: true displayName: Build @@ -290,7 +288,7 @@ stages: condition: always() continueOnError: true inputs: - PathToPublish: '$(Build.SourcesDirectory)\artifacts\log/Release\Build.VisualFSharp.sln.binlog' + PathToPublish: '$(Build.SourcesDirectory)\artifacts\log/Release\Build.VisualFSharp.slnx.binlog' ArtifactName: 'Windows Release build binlogs' ArtifactType: Container parallel: true @@ -306,24 +304,18 @@ stages: - job: WindowsNoRealsig_testDesktop pool: - # The PR build definition sets this variable: - # WindowsMachineQueueName=Windows.vs2022.amd64.open - # and there is an alternate build definition that sets this to a queue that is always scouting the - # next preview of Visual Studio. name: $(DncEngPublicBuildPool) - demands: ImageOverride -equals $(WindowsMachineQueueName) + demands: ImageOverride -equals $(_WindowsMachineQueueName) timeoutInMinutes: 120 - strategy: - parallel: 4 steps: - checkout: self clean: true - - script: eng\CIBuildNoPublish.cmd -compressallmetadata -buildnorealsig -testDesktop -configuration Release -testBatch $(System.JobPositionInPhase) + - script: eng\CIBuildNoPublish.cmd -compressallmetadata -buildnorealsig -testDesktop -configuration Release env: FSharp_CacheEvictionImmediate: true DOTNET_DbgEnableMiniDump: 1 - DOTNET_DbgMiniDumpType: 3 # Triage dump, 1 for mini, 2 for Heap, 3 for triage, 4 for full. Don't use 4 unless you know what you're doing. + DOTNET_DbgMiniDumpType: 2 # 1=mini, 2=heap, 3=triage, 4=full. Heap dumps include managed object data for debugging. DOTNET_DbgMiniDumpName: $(Build.SourcesDirectory)\artifacts\log\Release\$(Build.BuildId)-%e-%p-%t.dmp NativeToolsOnMachine: true displayName: Build @@ -332,7 +324,7 @@ stages: displayName: Publish Test Results inputs: testResultsFormat: 'XUnit' - testRunTitle: WindowsNoRealsig_testDesktop batch $(System.JobPositionInPhase) + testRunTitle: WindowsNoRealsig_testDesktop mergeTestResults: true testResultsFiles: '*.xml' searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/Release' @@ -340,10 +332,9 @@ stages: continueOnError: true - task: PublishBuildArtifacts@1 displayName: Publish Build BinLog - condition: eq(variables['System.JobPositionInPhase'], 1) continueOnError: true inputs: - PathToPublish: '$(Build.SourcesDirectory)\artifacts\log/Release\Build.VisualFSharp.sln.binlog' + PathToPublish: '$(Build.SourcesDirectory)\artifacts\log/Release\Build.VisualFSharp.slnx.binlog' ArtifactName: 'Windows Release build binlogs' ArtifactType: Container parallel: true @@ -353,18 +344,14 @@ stages: continueOnError: true inputs: PathToPublish: '$(Build.SourcesDirectory)\artifacts\log\Release' - ArtifactName: 'Windows Release WindowsNoRealsig_testDesktop process dumps $(System.JobPositionInPhase)' + ArtifactName: 'Windows Release WindowsNoRealsig_testDesktop process dumps' ArtifactType: Container parallel: true - job: WindowsStrictIndentation pool: - # The PR build definition sets this variable: - # WindowsMachineQueueName=Windows.vs2022.amd64.open - # and there is an alternate build definition that sets this to a queue that is always scouting the - # next preview of Visual Studio. name: $(DncEngPublicBuildPool) - demands: ImageOverride -equals $(WindowsMachineQueueName) + demands: ImageOverride -equals $(_WindowsMachineQueueName) timeoutInMinutes: 120 steps: - checkout: self @@ -373,7 +360,7 @@ stages: - script: eng\CIBuildNoPublish.cmd -compressallmetadata -configuration Release /p:AdditionalFscCmdFlags=--strict-indentation+ env: DOTNET_DbgEnableMiniDump: 1 - DOTNET_DbgMiniDumpType: 3 # Triage dump, 1 for mini, 2 for Heap, 3 for triage, 4 for full. Don't use 4 unless you know what you're doing. + DOTNET_DbgMiniDumpType: 2 # 1=mini, 2=heap, 3=triage, 4=full. Heap dumps include managed object data for debugging. DOTNET_DbgMiniDumpName: $(Build.SourcesDirectory)\artifacts\log\Release\$(Build.BuildId)-%e-%p-%t.dmp NativeToolsOnMachine: true displayName: Build @@ -383,7 +370,7 @@ stages: condition: always() continueOnError: true inputs: - PathToPublish: '$(Build.SourcesDirectory)\artifacts\log/Release\Build.VisualFSharp.sln.binlog' + PathToPublish: '$(Build.SourcesDirectory)\artifacts\log/Release\Build.VisualFSharp.slnx.binlog' ArtifactName: 'Windows Release build binlogs' ArtifactType: Container parallel: true @@ -400,7 +387,7 @@ stages: - job: WindowsNoStrictIndentation pool: name: $(DncEngPublicBuildPool) - demands: ImageOverride -equals $(WindowsMachineQueueName) + demands: ImageOverride -equals $(_WindowsMachineQueueName) timeoutInMinutes: 120 steps: - checkout: self @@ -409,7 +396,7 @@ stages: - script: eng\CIBuildNoPublish.cmd -compressallmetadata -configuration Release /p:AdditionalFscCmdFlags=--strict-indentation- env: DOTNET_DbgEnableMiniDump: 1 - DOTNET_DbgMiniDumpType: 3 # Triage dump, 1 for mini, 2 for Heap, 3 for triage, 4 for full. Don't use 4 unless you know what you're doing. + DOTNET_DbgMiniDumpType: 2 # 1=mini, 2=heap, 3=triage, 4=full. Heap dumps include managed object data for debugging. DOTNET_DbgMiniDumpName: $(Build.SourcesDirectory)\artifacts\log\Release\$(Build.BuildId)-%e-%p-%t.dmp NativeToolsOnMachine: true displayName: Build @@ -419,7 +406,7 @@ stages: condition: always() continueOnError: true inputs: - PathToPublish: '$(Build.SourcesDirectory)\artifacts\log/Release\Build.VisualFSharp.sln.binlog' + PathToPublish: '$(Build.SourcesDirectory)\artifacts\log/Release\Build.VisualFSharp.slnx.binlog' ArtifactName: 'Windows Release build binlogs' ArtifactType: Container parallel: true @@ -441,12 +428,8 @@ stages: - name: __VSNeverShowWhatsNew value: 1 pool: - # The PR build definition sets this variable: - # WindowsMachineQueueName=Windows.vs2022.amd64.open - # and there is an alternate build definition that sets this to a queue that is always scouting the - # next preview of Visual Studio. name: $(DncEngPublicBuildPool) - demands: ImageOverride -equals $(WindowsMachineQueueName) + demands: ImageOverride -equals $(_WindowsMachineQueueName) timeoutInMinutes: 120 strategy: matrix: @@ -454,11 +437,6 @@ stages: _configuration: Release _testKind: testCoreclr transparentCompiler: # Empty display name part. - fsharpqa_release: - _configuration: Release - _testKind: testFSharpQA - FSharp_CacheEvictionImmediate: true - transparentCompiler: vs_release: _configuration: Release _testKind: testVs @@ -485,7 +463,7 @@ stages: - script: eng\CIBuildNoPublish.cmd -compressallmetadata -configuration $(_configuration) -$(_testKind) env: DOTNET_DbgEnableMiniDump: 1 - DOTNET_DbgMiniDumpType: 3 # Triage dump, 1 for mini, 2 for Heap, 3 for triage, 4 for full. Don't use 4 unless you know what you're doing. + DOTNET_DbgMiniDumpType: 2 # 1=mini, 2=heap, 3=triage, 4=full. Heap dumps include managed object data for debugging. DOTNET_DbgMiniDumpName: $(Build.SourcesDirectory)\artifacts\log\$(_configuration)\$(Build.BuildId)-%e-%p-%t.dmp NativeToolsOnMachine: true displayName: Build and Test $(_testKind) $(transparentCompiler) @@ -499,13 +477,13 @@ stages: testResultsFiles: '*.xml' searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_configuration)' continueOnError: true - condition: succeededOrFailed() # ne(variables['_testKind'], 'testFSharpQA') + condition: succeededOrFailed() - task: PublishBuildArtifacts@1 displayName: Publish Tests BinLog condition: always() continueOnError: true inputs: - PathToPublish: '$(Build.SourcesDirectory)\artifacts\log/$(_configuration)\Build.VisualFSharp.sln.binlog' + PathToPublish: '$(Build.SourcesDirectory)\artifacts\log/$(_configuration)\Build.VisualFSharp.slnx.binlog' ArtifactName: Windows $(_configuration) $(_testKind) $(transparentCompiler) test binlogs ArtifactType: Container parallel: true @@ -538,91 +516,47 @@ stages: continueOnError: true condition: failed() - # Windows With Compressed Metadata Desktop + # Windows With Compressed Metadata Desktop (split into 3 batches) - job: WindowsCompressedMetadata_Desktop + strategy: + matrix: + Batch1: + batchNumber: 1 + Batch2: + batchNumber: 2 + Batch3: + batchNumber: 3 variables: - name: XUNIT_LOGS value: $(Build.SourcesDirectory)\artifacts\TestResults\Release - name: __VSNeverShowWhatsNew value: 1 pool: - # The PR build definition sets this variable: - # WindowsMachineQueueName=Windows.vs2022.amd64.open - # and there is an alternate build definition that sets this to a queue that is always scouting the - # next preview of Visual Studio. name: $(DncEngPublicBuildPool) - demands: ImageOverride -equals $(WindowsMachineQueueName) + demands: ImageOverride -equals $(_WindowsMachineQueueName) timeoutInMinutes: 120 - strategy: - parallel: 4 steps: - - checkout: self - clean: true - - - script: eng\CIBuildNoPublish.cmd -compressallmetadata -configuration Release -testDesktop -testBatch $(System.JobPositionInPhase) - env: - FSharp_CacheEvictionImmediate: true - DOTNET_DbgEnableMiniDump: 1 - DOTNET_DbgMiniDumpType: 3 # Triage dump, 1 for mini, 2 for Heap, 3 for triage, 4 for full. Don't use 4 unless you know what you're doing. - DOTNET_DbgMiniDumpName: $(Build.SourcesDirectory)\artifacts\log\Release\$(Build.BuildId)-%e-%p-%t.dmp - NativeToolsOnMachine: true - displayName: Build / Test - - - task: PublishTestResults@2 - displayName: Publish Test Results - inputs: - testResultsFormat: 'XUnit' - testRunTitle: WindowsCompressedMetadata testDesktop batch $(System.JobPositionInPhase) - mergeTestResults: true - testResultsFiles: '*.xml' - searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/Release' - continueOnError: true - condition: succeededOrFailed() - - - task: PublishBuildArtifacts@1 - displayName: Publish BinLog - condition: eq(variables['System.JobPositionInPhase'], 1) - continueOnError: true - inputs: - PathToPublish: '$(Build.SourcesDirectory)\artifacts\log/Release\Build.VisualFSharp.sln.binlog' - ArtifactName: 'Windows testDesktop binlogs' - ArtifactType: Container - parallel: true - - task: PublishBuildArtifacts@1 - displayName: Publish Dumps - condition: failed() - continueOnError: true - inputs: - PathToPublish: '$(Build.SourcesDirectory)\artifacts\log\Release' - ArtifactName: 'Windows testDesktop process dumps $(System.JobPositionInPhase)' - ArtifactType: Container - parallel: true - - task: PublishBuildArtifacts@1 - displayName: Publish Test Logs - inputs: - PathtoPublish: '$(Build.SourcesDirectory)\artifacts\TestResults\Release' - ArtifactName: 'Windows testDesktop test logs batch $(System.JobPositionInPhase)' - publishLocation: Container - continueOnError: true - condition: always() - - script: dotnet build $(Build.SourcesDirectory)/eng/DumpPackageRoot/DumpPackageRoot.csproj - displayName: Dump NuGet cache contents - condition: failed() - - task: PublishBuildArtifacts@1 - displayName: Publish NuGet cache contents - inputs: - PathtoPublish: '$(Build.SourcesDirectory)\artifacts\NugetPackageRootContents' - ArtifactName: 'NuGetPackageContents Windows testDesktop $(System.JobPositionInPhase)' - publishLocation: Container - continueOnError: true - condition: failed() + - template: /eng/templates/batched-test-steps.yml + parameters: + buildCommand: eng\CIBuildNoPublish.cmd -compressallmetadata -configuration Release -testDesktopBatch $(batchNumber) + buildEnv: + FSharp_CacheEvictionImmediate: true + DOTNET_DbgEnableMiniDump: 1 + DOTNET_DbgMiniDumpType: 2 + DOTNET_DbgMiniDumpName: $(Build.SourcesDirectory)\artifacts\log\Release\$(Build.BuildId)-%e-%p-%t.dmp + NativeToolsOnMachine: true + testRunTitlePrefix: 'WindowsCompressedMetadata testDesktop' + artifactNamePrefix: 'Windows testDesktop' + publishBinLog: true + binLogPath: '$(Build.SourcesDirectory)\artifacts\log/Release\Build.VisualFSharp.slnx.binlog' + publishDumps: true # Mock official build - job: MockOfficial pool: name: $(DncEngPublicBuildPool) - demands: ImageOverride -equals $(WindowsMachineQueueName) + demands: ImageOverride -equals $(_WindowsMachineQueueName) steps: - checkout: self clean: true @@ -632,7 +566,8 @@ stages: # Linux - job: Linux pool: - vmImage: $(UbuntuMachineQueueName) + name: $(DncEngPublicBuildPool) + demands: ImageOverride -equals $(LinuxMachineQueueName) timeoutInMinutes: 120 variables: - name: _SignType @@ -672,8 +607,14 @@ stages: continueOnError: true condition: failed() - # MacOS + # MacOS (split into 2 batches; batch 3 is desktop-only FSharpSuite) - job: MacOS + strategy: + matrix: + Batch1: + batchNumber: 1 + Batch2: + batchNumber: 2 pool: vmImage: macos-latest timeoutInMinutes: 120 @@ -681,65 +622,64 @@ stages: - name: _SignType value: Test steps: - - checkout: self - clean: true - - script: ./eng/cibuild.sh --configuration $(_BuildConfig) --testcoreclr - env: - COMPlus_DefaultStackSize: 1000000 - displayName: Build / Test - - task: PublishTestResults@2 - displayName: Publish Test Results - inputs: - testResultsFormat: 'XUnit' - testResultsFiles: '*.xml' - testRunTitle: MacOS - mergeTestResults: true - searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' - continueOnError: true - condition: succeededOrFailed() - - task: PublishBuildArtifacts@1 - displayName: Publish Test Logs - inputs: - PathtoPublish: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' - ArtifactName: 'MacOS $(_BuildConfig) test logs' - publishLocation: Container - continueOnError: true - condition: failed() - - script: dotnet build $(Build.SourcesDirectory)/eng/DumpPackageRoot/DumpPackageRoot.csproj - displayName: Dump NuGet cache contents - condition: failed() - - task: PublishBuildArtifacts@1 - displayName: Publish NuGet cache contents - inputs: - PathtoPublish: '$(Build.SourcesDirectory)/artifacts/NugetPackageRootContents' - ArtifactName: 'NuGetPackageContents Mac' - publishLocation: Container - continueOnError: true - condition: failed() + - template: /eng/templates/batched-test-steps.yml + parameters: + buildCommand: ./eng/cibuild.sh --configuration $(_BuildConfig) --testcoreclrbatch $(batchNumber) + buildEnv: + COMPlus_DefaultStackSize: 1000000 + testRunTitlePrefix: 'MacOS' + artifactNamePrefix: 'MacOS' + publishBinLog: true + binLogPath: '$(Build.SourcesDirectory)/artifacts/log/Release/Build.binlog' # End to end build - job: EndToEndBuildTests pool: name: $(DncEngPublicBuildPool) - demands: ImageOverride -equals $(WindowsMachineQueueName) - strategy: - maxParallel: 2 - matrix: - regular: - _experimental_flag: '' - experimental_features: - _experimental_flag: '' + demands: ImageOverride -equals $(_WindowsMachineQueueName) steps: - checkout: self clean: true - script: .\Build.cmd -c Release -pack env: NativeToolsOnMachine: true - FSHARP_EXPERIMENTAL_FEATURES: $(_experimental_flag) - script: .\tests\EndToEndBuildTests\EndToEndBuildTests.cmd -c Release - env: - FSHARP_EXPERIMENTAL_FEATURES: $(_experimental_flag) displayName: End to end build tests + + # Publish artifacts for regression testing + - task: PublishPipelineArtifact@1 + displayName: Publish F# Compiler FSC Artifacts for Regression Tests + inputs: + targetPath: '$(Build.SourcesDirectory)/artifacts/bin/fsc' + artifactName: 'FSharpCompilerFscArtifacts' + publishLocation: pipeline + condition: succeeded() + + - task: PublishPipelineArtifact@1 + displayName: Publish F# Core Artifacts for Regression Tests + inputs: + targetPath: '$(Build.SourcesDirectory)/artifacts/bin/FSharp.Core' + artifactName: 'FSharpCoreArtifacts' + publishLocation: pipeline + condition: succeeded() + + - pwsh: | + # Stage UseLocalCompiler props and TargetFrameworks.props together + $stagingDir = "$(Build.SourcesDirectory)/UseLocalCompilerPropsStaging" + New-Item -ItemType Directory -Force -Path $stagingDir | Out-Null + Copy-Item "$(Build.SourcesDirectory)/UseLocalCompiler.Directory.Build.props" -Destination $stagingDir + Copy-Item "$(Build.SourcesDirectory)/eng/TargetFrameworks.props" -Destination $stagingDir + Write-Host "Staged files for UseLocalCompilerProps artifact:" + Get-ChildItem $stagingDir -Name + displayName: Stage UseLocalCompiler props files + + - task: PublishPipelineArtifact@1 + displayName: Publish UseLocalCompiler props file for Regression Tests + inputs: + targetPath: '$(Build.SourcesDirectory)/UseLocalCompilerPropsStaging' + artifactName: 'UseLocalCompilerProps' + publishLocation: pipeline + condition: succeeded() # Up-to-date - disabled due to it being flaky #- job: UpToDate_Windows @@ -761,7 +701,7 @@ stages: - job: Plain_Build_Windows pool: name: $(DncEngPublicBuildPool) - demands: ImageOverride -equals $(WindowsMachineQueueName) + demands: ImageOverride -equals $(_WindowsMachineQueueName) variables: - name: _BuildConfig value: Debug @@ -770,16 +710,17 @@ stages: clean: true - script: dotnet --list-sdks displayName: Report dotnet SDK versions - - script: .\eng\common\dotnet.cmd build .\FSharp.Compiler.Service.sln /bl:\"artifacts/log/$(_BuildConfig)/ServiceRegularBuild.binlog\" + - script: .\eng\common\dotnet.cmd build .\FSharp.Compiler.Service.slnx /bl:\"artifacts/log/$(_BuildConfig)/ServiceRegularBuild.binlog\" workingDirectory: $(Build.SourcesDirectory) - displayName: Regular rebuild of FSharp.Compiler.Service.sln + displayName: Regular rebuild of FSharp.Compiler.Service.slnx continueOnError: false condition: always() # Plain FCS build Linux - job: Plain_Build_Linux pool: - vmImage: $(UbuntuMachineQueueName) + name: $(DncEngPublicBuildPool) + demands: ImageOverride -equals $(LinuxMachineQueueName) variables: - name: _BuildConfig value: Debug @@ -788,9 +729,9 @@ stages: clean: true - script: dotnet --list-sdks displayName: Report dotnet SDK versions - - script: ./eng/common/dotnet.sh build ./FSharp.Compiler.Service.sln /bl:\"artifacts/log/$(_BuildConfig)/ServiceRegularBuild.binlog\" + - script: ./eng/common/dotnet.sh build ./FSharp.Compiler.Service.slnx /bl:\"artifacts/log/$(_BuildConfig)/ServiceRegularBuild.binlog\" workingDirectory: $(Build.SourcesDirectory) - displayName: Regular rebuild of FSharp.Compiler.Service.sln + displayName: Regular rebuild of FSharp.Compiler.Service.slnx continueOnError: false condition: always() @@ -806,33 +747,17 @@ stages: clean: true - script: dotnet --list-sdks displayName: Report dotnet SDK versions - - script: ./eng/common/dotnet.sh build ./FSharp.Compiler.Service.sln /bl:\"artifacts/log/$(_BuildConfig)/ServiceRegularBuild.binlog\" + - script: ./eng/common/dotnet.sh build ./FSharp.Compiler.Service.slnx /bl:\"artifacts/log/$(_BuildConfig)/ServiceRegularBuild.binlog\" workingDirectory: $(Build.SourcesDirectory) - displayName: Regular rebuild of FSharp.Compiler.Service.sln + displayName: Regular rebuild of FSharp.Compiler.Service.slnx continueOnError: false condition: always() - # Build and run fast benchmarks - - job: Benchmarks - pool: - name: $(DncEngPublicBuildPool) - demands: ImageOverride -equals $(WindowsMachineQueueName) - variables: - - name: _BuildConfig - value: Release - steps: - - checkout: self - clean: true - - script: eng\CIBuild.cmd -configuration $(_BuildConfig) -testBenchmarks - displayName: Smoke test fast benchmarks - continueOnError: true - condition: always() - # Test trimming on Windows - job: Build_And_Test_AOT_Windows pool: name: $(DncEngPublicBuildPool) - demands: ImageOverride -equals $(WindowsMachineQueueName) + demands: ImageOverride -equals $(_WindowsMachineQueueName) strategy: maxParallel: 2 matrix: @@ -873,7 +798,7 @@ stages: - job: ILVerify pool: name: $(DncEngPublicBuildPool) - demands: ImageOverride -equals $(WindowsMachineQueueName) + demands: ImageOverride -equals $(_WindowsMachineQueueName) steps: - checkout: self clean: true @@ -892,3 +817,84 @@ stages: - pwsh: .\tests\ILVerify\ilverify.ps1 displayName: Run ILVerify workingDirectory: $(Build.SourcesDirectory) + + #-------------------------------------------------------------------------------------------------------------------# + # F# Compiler Regression Tests # + #-------------------------------------------------------------------------------------------------------------------# + - ${{ if eq(variables['System.TeamProject'], 'public') }}: + - template: /eng/templates/regression-test-jobs.yml + parameters: + testMatrix: + - repo: marklam/SlowBuildRepro + commit: bbe2dec4d0379b5d7d0480997858c30d442fbb42 + buildScript: dotnet build -bl + displayName: UMX_Slow_Repro + - repo: fsprojects/FSharpPlus + commit: f614035b75922aba41ed6a36c2fc986a2171d2b8 + buildScript: build.cmd + displayName: FSharpPlus_Windows + - repo: fsprojects/FSharpPlus + commit: f614035b75922aba41ed6a36c2fc986a2171d2b8 + buildScript: build.sh + displayName: FSharpPlus_Linux + useVmImage: $(LinuxMachineQueueName) + usePool: $(DncEngPublicBuildPool) + - repo: fsprojects/FSharpPlus + commit: 2648efe + buildScript: dotnet build tests/FSharpPlus.Tests/FSharpPlus.Tests.fsproj -c Release -bl + displayName: FsharpPlus_NET10_Build_Lib_Tests + # remove this before merging + - repo: fsprojects/FSharpPlus + commit: 2648efe + buildScript: dotnet msbuild build.proj -t:Build;Test -bl + displayName: FsharpPlus_NET10_Test_Debug + - repo: fsprojects/FSharpPlus + commit: 2648efe + buildScript: dotnet msbuild build.proj -t:Build;Test -p:Configuration=Release -bl + displayName: FsharpPlus_NET10_Test_Release + - repo: fsprojects/FSharpPlus + commit: 2648efe + buildScript: dotnet msbuild build.proj -t:Build;AllDocs -bl + displayName: FsharpPlus_NET10_Docs + - repo: fsprojects/FSharpPlus + commit: 2648efe + buildScript: build.sh + displayName: FsharpPlus_Net10_Linux + useVmImage: $(LinuxMachineQueueName) + usePool: $(DncEngPublicBuildPool) + - repo: TheAngryByrd/IcedTasks + commit: 5453025da1cd3db8aebe9283a9ff190062a703e0 + buildScript: dotnet build IcedTasks.sln -bl + displayName: IcedTasks_Build + - repo: TheAngryByrd/IcedTasks + commit: 5453025da1cd3db8aebe9283a9ff190062a703e0 + buildScript: dotnet test IcedTasks.sln -bl + displayName: IcedTasks_Test_Debug + - repo: TheAngryByrd/IcedTasks + commit: 5453025da1cd3db8aebe9283a9ff190062a703e0 + buildScript: dotnet test IcedTasks.sln -c Release -bl + displayName: IcedTasks_Test_Release + - repo: demystifyfp/FsToolkit.ErrorHandling + commit: a7119caabe329817bee49d7094610c45db7725c1 + buildScript: dotnet build FsToolkit.ErrorHandling.sln -bl + displayName: FsToolkit_ErrorHandling_Build + - repo: demystifyfp/FsToolkit.ErrorHandling + commit: a7119caabe329817bee49d7094610c45db7725c1 + buildScript: dotnet test FsToolkit.ErrorHandling.sln -bl + displayName: FsToolkit_ErrorHandling_Test_Debug + - repo: demystifyfp/FsToolkit.ErrorHandling + commit: a7119caabe329817bee49d7094610c45db7725c1 + buildScript: dotnet test FsToolkit.ErrorHandling.sln -c Release -bl + displayName: FsToolkit_ErrorHandling_Test_Release + - repo: opentk/opentk + commit: d4b1b18be03859dc066298ee191f6efa521e7e28 + buildScript: dotnet build tests/OpenTK.Tests/OpenTK.Tests.fsproj -c Release -bl:build_1.binlog ;; dotnet build tests/OpenTK.Tests.Integration/OpenTK.Tests.Integration.fsproj -c Release -bl:build_2.binlog + displayName: OpenTK_FSharp_Build + - repo: bryanedds/Prime + commit: 6ddc28d46f81447eacb241b96e16ce693b210c96 + buildScript: dotnet build Prime.sln --configuration Release + displayName: Prime_Build + - repo: bryanedds/Nu + commit: b321cb41a0bea0dab6b4509f895e6cd4d024e9e5 + buildScript: dotnet build Nu.sln --configuration Release + displayName: Nu_Build diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 403f7373f1f..19ed86c2d3f 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -10,10 +10,10 @@ trigger: - '*' exclude: - .github/* - - docs/ + - docs/* - .vscode/* - .devcontainer/* - - tests/scripts/ + - tests/scripts/* - attributions.md - CODE_OF_CONDUCT.md - DEVGUIDE.md @@ -43,11 +43,11 @@ variables: - name: _SignType value: Real - name: _PublishUsingPipelines - value: true + value: "true" - name: VisualStudioDropName value: Products/$(System.TeamProject)/$(Build.Repository.Name)/$(Build.SourceBranchName)/$(Build.BuildNumber) - name: Codeql.Enabled - value: true + value: "true" - group: DotNet-FSharp-SDLValidation-Params - template: /eng/common/templates-official/variables/pool-providers.yml@self @@ -112,7 +112,7 @@ extends: - job: Full_Signed pool: name: NetCore1ESPool-Svc-Internal - image: windows.vs2022preview.amd64 + image: windows.vs2026preview.scout.amd64 timeoutInMinutes: 300 templateContext: mb: @@ -148,6 +148,7 @@ extends: env: NativeToolsOnMachine: true FSHARP_CACHE_OVERRIDE: 256 + FSharp_CacheEvictionImmediate: true - task: PublishTestResults@2 displayName: Publish Test Results inputs: @@ -188,16 +189,25 @@ extends: targetPath: '$(Build.SourcesDirectory)\artifacts\SymStore\$(_BuildConfig)' artifactName: 'NativeSymbols' condition: succeeded() + - task: AzureCLI@2 + displayName: Get DevDiv Drop Access Token + inputs: + azureSubscription: 'dnceng-devdiv-drop-rw-code-rw-wif' + scriptType: pscore + scriptLocation: inlineScript + inlineScript: | + $token = az account get-access-token --resource 499b84ac-1321-427f-aa17-267ca6975798 --query accessToken -o tsv + Write-Host "##vso[task.setvariable variable=DevDivDropAccessToken;issecret=true]$token" + condition: succeeded() - task: 1ES.MicroBuildVstsDrop@1 displayName: Upload VSTS Drop inputs: dropName: $(VisualStudioDropName) dropFolder: '$(Build.SourcesDirectory)\artifacts\VSSetup\$(_BuildConfig)\Insertion' dropRetentionDays: 90 - accessToken: $(dn-bot-devdiv-drop-rw-code-rw) + accessToken: $(DevDivDropAccessToken) dropServiceUri: 'https://devdiv.artifacts.visualstudio.com' vsDropServiceUri: 'https://vsdrop.corp.microsoft.com/file/v1' - condition: succeeded() #---------------------------------------------------------------------------------------------------------------------# @@ -238,3 +248,4 @@ extends: insertTeamEmail: fsharpteam@microsoft.com insertTeamName: 'F#' completeInsertion: 'auto' + diff --git a/docs/release-notes/.FSharp.Compiler.Service/10.0.100.md b/docs/release-notes/.FSharp.Compiler.Service/10.0.100.md index fe1ddaf787d..89afb46101b 100644 --- a/docs/release-notes/.FSharp.Compiler.Service/10.0.100.md +++ b/docs/release-notes/.FSharp.Compiler.Service/10.0.100.md @@ -36,6 +36,7 @@ * Caches: type subsumption cache key perf regression ([Issue #18925](https://github.com/dotnet/fsharp/issues/18925) [PR #18926](https://github.com/dotnet/fsharp/pull/18926)) * Ensure that line directives are applied to source identifiers (issue [#18908](https://github.com/dotnet/fsharp/issues/18908), PR [#18918](https://github.com/dotnet/fsharp/pull/18918)) * Fix expected and actual types in ErrorFromAddingTypeEquation message and extended diagnostic data. ([PR #18915](https://github.com/dotnet/fsharp/pull/18915)) +* Editor: Fix Record fields completion in update record with partial field name. ([PR #18946](https://github.com/dotnet/fsharp/pull/18946)) ### Changed * Use `errorR` instead of `error` in `CheckDeclarations.fs` when possible. ([PR #18645](https://github.com/dotnet/fsharp/pull/18645)) diff --git a/docs/release-notes/.FSharp.Compiler.Service/11.0.0.md b/docs/release-notes/.FSharp.Compiler.Service/11.0.0.md new file mode 100644 index 00000000000..8b137891791 --- /dev/null +++ b/docs/release-notes/.FSharp.Compiler.Service/11.0.0.md @@ -0,0 +1 @@ + diff --git a/eng/TargetFrameworks.props b/eng/TargetFrameworks.props new file mode 100644 index 00000000000..d384e5fbcaa --- /dev/null +++ b/eng/TargetFrameworks.props @@ -0,0 +1,20 @@ + + + + + + + net10.0 + + + $([System.Text.RegularExpressions.Regex]::Replace('$(FSharpNetCoreProductTargetFramework)', '^net(\d+)\.0$', '$1')) + + + diff --git a/eng/Version.Details.props b/eng/Version.Details.props index e0f3350befd..7f6cdf3d02c 100644 --- a/eng/Version.Details.props +++ b/eng/Version.Details.props @@ -5,48 +5,62 @@ This file should be imported by eng/Versions.props --> - - 18.0.0-preview-25476-03 - 18.0.0-preview-25476-03 - 18.0.0-preview-25476-03 - 18.0.0-preview-25476-03 - - 9.0.0 - 9.0.0 - 9.0.0 - 9.0.0 - 9.0.0 - 9.0.0 - - 10.0.0-beta.25426.3 - - 1.0.0-prerelease.25467.1 - 1.0.0-prerelease.25467.1 - 1.0.0-prerelease.25467.1 - 1.0.0-prerelease.25467.1 - 1.0.0-prerelease.25467.1 + + 10.0.0-beta.26269.2 + + 18.8.0-preview-26269-11 + 18.8.0-preview-26269-11 + 18.8.0-preview-26269-11 + 18.8.0-preview-26269-11 + + 1.0.0-prerelease.26180.1 + 1.0.0-prerelease.26180.1 + 1.0.0-prerelease.26180.1 + 1.0.0-prerelease.26180.1 + 1.0.0-prerelease.26180.1 + + 5.8.0-1.26268.6 + 5.8.0-1.26268.6 + 5.8.0-1.26268.6 + 5.8.0-1.26268.6 + 5.8.0-1.26268.6 + 5.8.0-1.26268.6 + 5.8.0-1.26268.6 + 5.8.0-1.26268.6 + + 10.0.2 + 10.0.2 + 10.0.2 + 10.0.2 - + + $(MicrosoftDotNetArcadeSdkPackageVersion) + $(MicrosoftBuildPackageVersion) $(MicrosoftBuildFrameworkPackageVersion) $(MicrosoftBuildTasksCorePackageVersion) $(MicrosoftBuildUtilitiesCorePackageVersion) - - $(SystemCollectionsImmutablePackageVersion) - $(SystemComponentModelCompositionPackageVersion) - $(SystemCompositionPackageVersion) - $(SystemDiagnosticsDiagnosticSourcePackageVersion) - $(SystemReflectionMetadataPackageVersion) - $(SystemThreadingTasksDataflowPackageVersion) - - $(MicrosoftDotNetArcadeSdkPackageVersion) - + $(optimizationlinuxarm64MIBCRuntimePackageVersion) $(optimizationlinuxx64MIBCRuntimePackageVersion) $(optimizationwindows_ntarm64MIBCRuntimePackageVersion) $(optimizationwindows_ntx64MIBCRuntimePackageVersion) $(optimizationwindows_ntx86MIBCRuntimePackageVersion) + + $(MicrosoftCodeAnalysisPackageVersion) + $(MicrosoftCodeAnalysisCompilersPackageVersion) + $(MicrosoftCodeAnalysisCSharpPackageVersion) + $(MicrosoftCodeAnalysisEditorFeaturesPackageVersion) + $(MicrosoftCodeAnalysisEditorFeaturesTextPackageVersion) + $(MicrosoftCodeAnalysisExternalAccessFSharpPackageVersion) + $(MicrosoftCodeAnalysisFeaturesPackageVersion) + $(MicrosoftVisualStudioLanguageServicesPackageVersion) + + $(SystemCollectionsImmutablePackageVersion) + $(SystemCompositionPackageVersion) + $(SystemDiagnosticsDiagnosticSourcePackageVersion) + $(SystemReflectionMetadataPackageVersion) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 4c2103c42d6..d1e5b5c3bfe 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,72 +1,104 @@ - + - + https://github.com/dotnet/msbuild - 5480b47bbc592c6feedca7c57c9863ba01e0c52c + 911bea0b57d3613eb9c29f49ff9858d03884c397 - + https://github.com/dotnet/msbuild - 5480b47bbc592c6feedca7c57c9863ba01e0c52c + 911bea0b57d3613eb9c29f49ff9858d03884c397 - + https://github.com/dotnet/msbuild - 5480b47bbc592c6feedca7c57c9863ba01e0c52c + 911bea0b57d3613eb9c29f49ff9858d03884c397 - + https://github.com/dotnet/msbuild - 5480b47bbc592c6feedca7c57c9863ba01e0c52c + 911bea0b57d3613eb9c29f49ff9858d03884c397 - - https://github.com/dotnet/runtime - 9d5a6a9aa463d6d10b0b0ba6d5982cc82f363dc3 + + https://github.com/dotnet/roslyn + 256895f3327e7e2f3f7e94248f227a47c698c1e5 - - https://github.com/dotnet/runtime - 9d5a6a9aa463d6d10b0b0ba6d5982cc82f363dc3 + + https://github.com/dotnet/roslyn + 256895f3327e7e2f3f7e94248f227a47c698c1e5 + + + https://github.com/dotnet/roslyn + 256895f3327e7e2f3f7e94248f227a47c698c1e5 + + + https://github.com/dotnet/roslyn + 256895f3327e7e2f3f7e94248f227a47c698c1e5 + + + https://github.com/dotnet/roslyn + 256895f3327e7e2f3f7e94248f227a47c698c1e5 + + + https://github.com/dotnet/roslyn + 256895f3327e7e2f3f7e94248f227a47c698c1e5 + + + https://github.com/dotnet/roslyn + 256895f3327e7e2f3f7e94248f227a47c698c1e5 + + + https://github.com/dotnet/roslyn + 256895f3327e7e2f3f7e94248f227a47c698c1e5 - + + https://github.com/dotnet/runtime - 9d5a6a9aa463d6d10b0b0ba6d5982cc82f363dc3 + + - + + https://github.com/dotnet/runtime - 9d5a6a9aa463d6d10b0b0ba6d5982cc82f363dc3 + + - + + https://github.com/dotnet/runtime - 9d5a6a9aa463d6d10b0b0ba6d5982cc82f363dc3 + + - + + https://github.com/dotnet/runtime - 9d5a6a9aa463d6d10b0b0ba6d5982cc82f363dc3 + + - + https://github.com/dotnet/arcade - 5db998e02282e63bc375948a237bcdfef534a5c5 + 96a58d029003171c2302b951688fc87464b197f5 - + https://dev.azure.com/dnceng/internal/_git/dotnet-optimization - 59dc6a9bf1b3e3ab71c73d94160c2049fb104cd1 + 9f55ee44f6d99b78f3e80f77e2ed73fb73b8f63b - + https://dev.azure.com/dnceng/internal/_git/dotnet-optimization - 59dc6a9bf1b3e3ab71c73d94160c2049fb104cd1 + 9f55ee44f6d99b78f3e80f77e2ed73fb73b8f63b - + https://dev.azure.com/dnceng/internal/_git/dotnet-optimization - 59dc6a9bf1b3e3ab71c73d94160c2049fb104cd1 + 9f55ee44f6d99b78f3e80f77e2ed73fb73b8f63b - + https://dev.azure.com/dnceng/internal/_git/dotnet-optimization - 59dc6a9bf1b3e3ab71c73d94160c2049fb104cd1 + 9f55ee44f6d99b78f3e80f77e2ed73fb73b8f63b - + https://dev.azure.com/dnceng/internal/_git/dotnet-optimization - 59dc6a9bf1b3e3ab71c73d94160c2049fb104cd1 + 9f55ee44f6d99b78f3e80f77e2ed73fb73b8f63b diff --git a/eng/Versions.props b/eng/Versions.props index 1af06f79f5d..64e545f6118 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -1,39 +1,44 @@ - + + + + - false - true true true + - 2 - rc$(FSharpPreReleaseIteration) - + 5 + preview$(FSharpPreReleaseIteration) - 10 + 11 0 - 100 + 101 0 $(FSMajorVersion).$(FSMinorVersion) + + 1 $(FSMajorVersion).$(FSMinorVersion) $(FSMajorVersion).$(FSMinorVersion).$(FSBuildVersion) $(FSMajorVersion).$(FSMinorVersion).$(FSBuildVersion) $(FSMajorVersion).$(FSMinorVersion).0.0 - 10.0.0.0 + 10.0.101.0 + + 0 43 - 10 + 12 $(FSBuildVersion) $(FSRevisionVersion) $(FCSMajorVersion).$(FCSMinorVersion).$(FCSBuildVersion) @@ -41,32 +46,35 @@ $(FSMajorVersion).$(FSMinorVersion).$(FSBuildVersion) - 9.0.300 + 10.0.101 $(FSCorePackageVersionValue)-$(PreReleaseVersionLabel).* - 14 - 0 + 15 + 2 $(FSBuildVersion) $(FSRevisionVersion) $(FSToolsMajorVersion).$(FSToolsMinorVersion).$(FSToolsBuildVersion) $(FSToolsMajorVersion).$(FSToolsMinorVersion).$(FSToolsBuildVersion).$(FSToolsRevisionVersion) + 18 - 0 + 8 $(VSMajorVersion).0 $(VSMajorVersion).$(VSMinorVersion).0 $(VSAssemblyVersionPrefix).0 + $(FSCoreVersionPrefix) $(FSCoreVersion) + $(FSCoreVersionPrefix) $(FSProductVersionPrefix) @@ -75,18 +83,15 @@ $(VersionPrefix).0 - - + + + 4.6.1 4.6.3 6.1.2 + - - - 4.6.0 - 4.6.0 - 6.1.0 - + - 5.0.0-1.25276.102 - 17.14.188 - 17.14.40268 - 17.14.79 - 17.14.40254 - 17.14.15 - - $(RoslynVersion) - $(RoslynVersion) - $(RoslynVersion) - 5.0.0-1.25275.2 - $(RoslynVersion) - $(RoslynVersion) - $(RoslynVersion) - $(RoslynVersion) + 18.0.404-preview + 18.0.2188-preview.1 + 18.0.1237-pre + 18.0.2077-preview.1 + 18.0.5 + + 2.0.28 - $(RoslynVersion) - $(MicrosoftVisualStudioShellPackagesVersion) - $(MicrosoftVisualStudioShellPackagesVersion) $(VisualStudioShellProjectsPackages) - $(MicrosoftVisualStudioShellPackagesVersion) $(MicrosoftVisualStudioShellPackagesVersion) $(MicrosoftVisualStudioShellPackagesVersion) $(MicrosoftVisualStudioShellPackagesVersion) - $(MicrosoftVisualStudioShellPackagesVersion) $(MicrosoftVisualStudioShellPackagesVersion) - $(MicrosoftVisualStudioShellPackagesVersion) $(MicrosoftVisualStudioShellPackagesVersion) $(MicrosoftVisualStudioShellPackagesVersion) $(VisualStudioShellProjectsPackages) - $(VisualStudioShellProjectsPackages) - $(MicrosoftVisualStudioShellPackagesVersion) - 17.14.40270 $(MicrosoftVisualStudioShellPackagesVersion) - 17.14.40270 10.0.30319 11.0.50727 15.0.25123-Dev15Preview + - $(VisualStudioEditorPackagesVersion) $(VisualStudioEditorPackagesVersion) $(VisualStudioEditorPackagesVersion) - $(VisualStudioEditorPackagesVersion) $(VisualStudioEditorPackagesVersion) $(VisualStudioEditorPackagesVersion) - $(VisualStudioEditorPackagesVersion) $(VisualStudioEditorPackagesVersion) - $(VisualStudioEditorPackagesVersion) - $(VisualStudioEditorPackagesVersion) - 17.14.106 - 5.6.0 - 0.1.169-beta + 17.14.0 + 0.1.800-beta $(MicrosoftVisualStudioExtensibilityTestingVersion) - $(MicrosoftVisualStudioExtensibilityTestingVersion) - - + + $(MicrosoftVisualStudioThreadingPackagesVersion) - + $(VisualStudioProjectSystemPackagesVersion) 2.3.6152103 - + - 17.10.2179 - 17.14.20 + 17.14.2120 17.0.0 - 17.8.8 - 12.0.4 - 7.0.4 - 8.0.4 - 11.0.4 - 7.0.4 0.2.0 1.0.0 - 1.1.33 + 1.1.87 0.13.10 - 2.16.5 - 4.3.0.0 - 1.0.31 + 2.16.6 4.3.0-1.22220.8 - 3.1.0 + 5.0.0-preview.7.20364.11 5.0.0-preview.7.20364.11 - 17.11.1 - 13.0.3 - 1.0.0-beta2-dev3 - 2.22.11 - 2.12.87 - 2.9.0 - 2.8.2 - 3.1.17 - 2.2.0 + 17.14.1 + 2.0.2 + 13.0.4 + 3.2.2 + 3.2.2 + 8.0.0 + diff --git a/eng/common/SetupNugetSources.ps1 b/eng/common/SetupNugetSources.ps1 index 792b60b49d4..65ed3a8adef 100644 --- a/eng/common/SetupNugetSources.ps1 +++ b/eng/common/SetupNugetSources.ps1 @@ -1,13 +1,14 @@ # This script adds internal feeds required to build commits that depend on internal package sources. For instance, -# dotnet6-internal would be added automatically if dotnet6 was found in the nuget.config file. In addition also enables -# disabled internal Maestro (darc-int*) feeds. +# dotnet6-internal would be added automatically if dotnet6 was found in the nuget.config file. Similarly, +# dotnet-eng-internal and dotnet-tools-internal are added if dotnet-eng and dotnet-tools are present. +# In addition, this script also enables disabled internal Maestro (darc-int*) feeds. # # Optionally, this script also adds a credential entry for each of the internal feeds if supplied. # # See example call for this script below. # # - task: PowerShell@2 -# displayName: Setup Private Feeds Credentials +# displayName: Setup internal Feeds Credentials # condition: eq(variables['Agent.OS'], 'Windows_NT') # inputs: # filePath: $(System.DefaultWorkingDirectory)/eng/common/SetupNugetSources.ps1 @@ -34,19 +35,28 @@ Set-StrictMode -Version 2.0 . $PSScriptRoot\tools.ps1 +# Adds or enables the package source with the given name +function AddOrEnablePackageSource($sources, $disabledPackageSources, $SourceName, $SourceEndPoint, $creds, $Username, $pwd) { + if ($disabledPackageSources -eq $null -or -not (EnableInternalPackageSource -DisabledPackageSources $disabledPackageSources -Creds $creds -PackageSourceName $SourceName)) { + AddPackageSource -Sources $sources -SourceName $SourceName -SourceEndPoint $SourceEndPoint -Creds $creds -Username $userName -pwd $Password + } +} + # Add source entry to PackageSources function AddPackageSource($sources, $SourceName, $SourceEndPoint, $creds, $Username, $pwd) { $packageSource = $sources.SelectSingleNode("add[@key='$SourceName']") if ($packageSource -eq $null) { + Write-Host "Adding package source $SourceName" + $packageSource = $doc.CreateElement("add") $packageSource.SetAttribute("key", $SourceName) $packageSource.SetAttribute("value", $SourceEndPoint) $sources.AppendChild($packageSource) | Out-Null } else { - Write-Host "Package source $SourceName already present." + Write-Host "Package source $SourceName already present and enabled." } AddCredential -Creds $creds -Source $SourceName -Username $Username -pwd $pwd @@ -59,6 +69,8 @@ function AddCredential($creds, $source, $username, $pwd) { return; } + Write-Host "Inserting credential for feed: " $source + # Looks for credential configuration for the given SourceName. Create it if none is found. $sourceElement = $creds.SelectSingleNode($Source) if ($sourceElement -eq $null) @@ -91,24 +103,27 @@ function AddCredential($creds, $source, $username, $pwd) { $passwordElement.SetAttribute("value", $pwd) } -function InsertMaestroPrivateFeedCredentials($Sources, $Creds, $Username, $pwd) { - $maestroPrivateSources = $Sources.SelectNodes("add[contains(@key,'darc-int')]") - - Write-Host "Inserting credentials for $($maestroPrivateSources.Count) Maestro's private feeds." - - ForEach ($PackageSource in $maestroPrivateSources) { - Write-Host "`tInserting credential for Maestro's feed:" $PackageSource.Key - AddCredential -Creds $creds -Source $PackageSource.Key -Username $Username -pwd $pwd +# Enable all darc-int package sources. +function EnableMaestroInternalPackageSources($DisabledPackageSources, $Creds) { + $maestroInternalSources = $DisabledPackageSources.SelectNodes("add[contains(@key,'darc-int')]") + ForEach ($DisabledPackageSource in $maestroInternalSources) { + EnableInternalPackageSource -DisabledPackageSources $DisabledPackageSources -Creds $Creds -PackageSourceName $DisabledPackageSource.key } } -function EnablePrivatePackageSources($DisabledPackageSources) { - $maestroPrivateSources = $DisabledPackageSources.SelectNodes("add[contains(@key,'darc-int')]") - ForEach ($DisabledPackageSource in $maestroPrivateSources) { - Write-Host "`tEnsuring private source '$($DisabledPackageSource.key)' is enabled by deleting it from disabledPackageSource" +# Enables an internal package source by name, if found. Returns true if the package source was found and enabled, false otherwise. +function EnableInternalPackageSource($DisabledPackageSources, $Creds, $PackageSourceName) { + $DisabledPackageSource = $DisabledPackageSources.SelectSingleNode("add[@key='$PackageSourceName']") + if ($DisabledPackageSource) { + Write-Host "Enabling internal source '$($DisabledPackageSource.key)'." + # Due to https://github.com/NuGet/Home/issues/10291, we must actually remove the disabled entries $DisabledPackageSources.RemoveChild($DisabledPackageSource) + + AddCredential -Creds $creds -Source $DisabledPackageSource.Key -Username $userName -pwd $Password + return $true } + return $false } if (!(Test-Path $ConfigFile -PathType Leaf)) { @@ -121,15 +136,17 @@ $doc = New-Object System.Xml.XmlDocument $filename = (Get-Item $ConfigFile).FullName $doc.Load($filename) -# Get reference to or create one if none exist already +# Get reference to - fail if none exist $sources = $doc.DocumentElement.SelectSingleNode("packageSources") if ($sources -eq $null) { - $sources = $doc.CreateElement("packageSources") - $doc.DocumentElement.AppendChild($sources) | Out-Null + Write-PipelineTelemetryError -Category 'Build' -Message "Eng/common/SetupNugetSources.ps1 returned a non-zero exit code. NuGet config file must contain a packageSources section: $ConfigFile" + ExitWithExitCode 1 } $creds = $null +$feedSuffix = "v3/index.json" if ($Password) { + $feedSuffix = "v2" # Looks for a node. Create it if none is found. $creds = $doc.DocumentElement.SelectSingleNode("packageSourceCredentials") if ($creds -eq $null) { @@ -138,34 +155,35 @@ if ($Password) { } } +$userName = "dn-bot" + # Check for disabledPackageSources; we'll enable any darc-int ones we find there $disabledSources = $doc.DocumentElement.SelectSingleNode("disabledPackageSources") if ($disabledSources -ne $null) { Write-Host "Checking for any darc-int disabled package sources in the disabledPackageSources node" - EnablePrivatePackageSources -DisabledPackageSources $disabledSources + EnableMaestroInternalPackageSources -DisabledPackageSources $disabledSources -Creds $creds } - -$userName = "dn-bot" - -# Insert credential nodes for Maestro's private feeds -InsertMaestroPrivateFeedCredentials -Sources $sources -Creds $creds -Username $userName -pwd $Password - -# 3.1 uses a different feed url format so it's handled differently here -$dotnet31Source = $sources.SelectSingleNode("add[@key='dotnet3.1']") -if ($dotnet31Source -ne $null) { - AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v2" -Creds $creds -Username $userName -pwd $Password - AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2" -Creds $creds -Username $userName -pwd $Password -} - -$dotnetVersions = @('5','6','7','8','9') +$dotnetVersions = @('5','6','7','8','9','10') foreach ($dotnetVersion in $dotnetVersions) { $feedPrefix = "dotnet" + $dotnetVersion; $dotnetSource = $sources.SelectSingleNode("add[@key='$feedPrefix']") if ($dotnetSource -ne $null) { - AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal/nuget/v2" -Creds $creds -Username $userName -pwd $Password - AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal-transport/nuget/v2" -Creds $creds -Username $userName -pwd $Password + AddOrEnablePackageSource -Sources $sources -DisabledPackageSources $disabledSources -SourceName "$feedPrefix-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal/nuget/$feedSuffix" -Creds $creds -Username $userName -pwd $Password + AddOrEnablePackageSource -Sources $sources -DisabledPackageSources $disabledSources -SourceName "$feedPrefix-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal-transport/nuget/$feedSuffix" -Creds $creds -Username $userName -pwd $Password } } +# Check for dotnet-eng and add dotnet-eng-internal if present +$dotnetEngSource = $sources.SelectSingleNode("add[@key='dotnet-eng']") +if ($dotnetEngSource -ne $null) { + AddOrEnablePackageSource -Sources $sources -DisabledPackageSources $disabledSources -SourceName "dotnet-eng-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet-eng-internal/nuget/$feedSuffix" -Creds $creds -Username $userName -pwd $Password +} + +# Check for dotnet-tools and add dotnet-tools-internal if present +$dotnetToolsSource = $sources.SelectSingleNode("add[@key='dotnet-tools']") +if ($dotnetToolsSource -ne $null) { + AddOrEnablePackageSource -Sources $sources -DisabledPackageSources $disabledSources -SourceName "dotnet-tools-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet-tools-internal/nuget/$feedSuffix" -Creds $creds -Username $userName -pwd $Password +} + $doc.Save($filename) diff --git a/eng/common/SetupNugetSources.sh b/eng/common/SetupNugetSources.sh index facb415ca6f..b2163abbe71 100755 --- a/eng/common/SetupNugetSources.sh +++ b/eng/common/SetupNugetSources.sh @@ -1,8 +1,9 @@ #!/usr/bin/env bash # This script adds internal feeds required to build commits that depend on internal package sources. For instance, -# dotnet6-internal would be added automatically if dotnet6 was found in the nuget.config file. In addition also enables -# disabled internal Maestro (darc-int*) feeds. +# dotnet6-internal would be added automatically if dotnet6 was found in the nuget.config file. Similarly, +# dotnet-eng-internal and dotnet-tools-internal are added if dotnet-eng and dotnet-tools are present. +# In addition, this script also enables disabled internal Maestro (darc-int*) feeds. # # Optionally, this script also adds a credential entry for each of the internal feeds if supplied. # @@ -52,81 +53,139 @@ if [[ `uname -s` == "Darwin" ]]; then TB='' fi -# Ensure there is a ... section. -grep -i "" $ConfigFile -if [ "$?" != "0" ]; then - echo "Adding ... section." - ConfigNodeHeader="" - PackageSourcesTemplate="${TB}${NL}${TB}" +# Enables an internal package source by name, if found. Returns 0 if found and enabled, 1 if not found. +EnableInternalPackageSource() { + local PackageSourceName="$1" + + # Check if disabledPackageSources section exists + grep -i "" "$ConfigFile" > /dev/null + if [ "$?" != "0" ]; then + return 1 # No disabled sources section + fi + + # Check if this source name is disabled + grep -i " /dev/null + if [ "$?" == "0" ]; then + echo "Enabling internal source '$PackageSourceName'." + # Remove the disabled entry (including any surrounding comments or whitespace on the same line) + sed -i.bak "//d" "$ConfigFile" + + # Add the source name to PackageSources for credential handling + PackageSources+=("$PackageSourceName") + return 0 # Found and enabled + fi + + return 1 # Not found in disabled sources +} + +# Add source entry to PackageSources +AddPackageSource() { + local SourceName="$1" + local SourceEndPoint="$2" + + # Check if source already exists + grep -i " /dev/null + if [ "$?" == "0" ]; then + echo "Package source $SourceName already present and enabled." + PackageSources+=("$SourceName") + return + fi + + echo "Adding package source $SourceName" + PackageSourcesNodeFooter="" + PackageSourceTemplate="${TB}" + + sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" "$ConfigFile" + PackageSources+=("$SourceName") +} + +# Adds or enables the package source with the given name +AddOrEnablePackageSource() { + local SourceName="$1" + local SourceEndPoint="$2" + + # Try to enable if disabled, if not found then add new source + EnableInternalPackageSource "$SourceName" + if [ "$?" != "0" ]; then + AddPackageSource "$SourceName" "$SourceEndPoint" + fi +} - sed -i.bak "s|$ConfigNodeHeader|$ConfigNodeHeader${NL}$PackageSourcesTemplate|" $ConfigFile -fi +# Enable all darc-int package sources +EnableMaestroInternalPackageSources() { + # Check if disabledPackageSources section exists + grep -i "" "$ConfigFile" > /dev/null + if [ "$?" != "0" ]; then + return # No disabled sources section + fi + + # Find all darc-int disabled sources + local DisabledDarcIntSources=() + DisabledDarcIntSources+=$(grep -oh '"darc-int-[^"]*" value="true"' "$ConfigFile" | tr -d '"') + + for DisabledSourceName in ${DisabledDarcIntSources[@]} ; do + if [[ $DisabledSourceName == darc-int* ]]; then + EnableInternalPackageSource "$DisabledSourceName" + fi + done +} -# Ensure there is a ... section. -grep -i "" $ConfigFile +# Ensure there is a ... section. +grep -i "" $ConfigFile if [ "$?" != "0" ]; then - echo "Adding ... section." - - PackageSourcesNodeFooter="" - PackageSourceCredentialsTemplate="${TB}${NL}${TB}" - - sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourcesNodeFooter${NL}$PackageSourceCredentialsTemplate|" $ConfigFile + Write-PipelineTelemetryError -Category 'Build' "Error: Eng/common/SetupNugetSources.sh returned a non-zero exit code. NuGet config file must contain a packageSources section: $ConfigFile" + ExitWithExitCode 1 fi PackageSources=() -# Ensure dotnet3.1-internal and dotnet3.1-internal-transport are in the packageSources if the public dotnet3.1 feeds are present -grep -i "... section. + grep -i "" $ConfigFile if [ "$?" != "0" ]; then - echo "Adding dotnet3.1-internal to the packageSources." - PackageSourcesNodeFooter="" - PackageSourceTemplate="${TB}" + echo "Adding ... section." - sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile - fi - PackageSources+=('dotnet3.1-internal') - - grep -i "" $ConfigFile - if [ "$?" != "0" ]; then - echo "Adding dotnet3.1-internal-transport to the packageSources." PackageSourcesNodeFooter="" - PackageSourceTemplate="${TB}" + PackageSourceCredentialsTemplate="${TB}${NL}${TB}" - sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile + sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourcesNodeFooter${NL}$PackageSourceCredentialsTemplate|" $ConfigFile fi - PackageSources+=('dotnet3.1-internal-transport') fi -DotNetVersions=('5' '6' '7' '8' '9') +# Check for disabledPackageSources; we'll enable any darc-int ones we find there +grep -i "" $ConfigFile > /dev/null +if [ "$?" == "0" ]; then + echo "Checking for any darc-int disabled package sources in the disabledPackageSources node" + EnableMaestroInternalPackageSources +fi + +DotNetVersions=('5' '6' '7' '8' '9' '10') for DotNetVersion in ${DotNetVersions[@]} ; do FeedPrefix="dotnet${DotNetVersion}"; - grep -i " /dev/null if [ "$?" == "0" ]; then - grep -i "" - - sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile - fi - PackageSources+=("$FeedPrefix-internal") - - grep -i "" $ConfigFile - if [ "$?" != "0" ]; then - echo "Adding $FeedPrefix-internal-transport to the packageSources." - PackageSourcesNodeFooter="" - PackageSourceTemplate="${TB}" - - sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile - fi - PackageSources+=("$FeedPrefix-internal-transport") + AddOrEnablePackageSource "$FeedPrefix-internal" "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$FeedPrefix-internal/nuget/$FeedSuffix" + AddOrEnablePackageSource "$FeedPrefix-internal-transport" "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$FeedPrefix-internal-transport/nuget/$FeedSuffix" fi done +# Check for dotnet-eng and add dotnet-eng-internal if present +grep -i " /dev/null +if [ "$?" == "0" ]; then + AddOrEnablePackageSource "dotnet-eng-internal" "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet-eng-internal/nuget/$FeedSuffix" +fi + +# Check for dotnet-tools and add dotnet-tools-internal if present +grep -i " /dev/null +if [ "$?" == "0" ]; then + AddOrEnablePackageSource "dotnet-tools-internal" "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet-tools-internal/nuget/$FeedSuffix" +fi + # I want things split line by line PrevIFS=$IFS IFS=$'\n' @@ -139,29 +198,12 @@ if [ "$CredToken" ]; then # Check if there is no existing credential for this FeedName grep -i "<$FeedName>" $ConfigFile if [ "$?" != "0" ]; then - echo "Adding credentials for $FeedName." + echo " Inserting credential for feed: $FeedName" PackageSourceCredentialsNodeFooter="" - NewCredential="${TB}${TB}<$FeedName>${NL}${NL}${NL}" + NewCredential="${TB}${TB}<$FeedName>${NL}${TB}${NL}${TB}${TB}${NL}${TB}${TB}" sed -i.bak "s|$PackageSourceCredentialsNodeFooter|$NewCredential${NL}$PackageSourceCredentialsNodeFooter|" $ConfigFile fi done fi - -# Re-enable any entries in disabledPackageSources where the feed name contains darc-int -grep -i "" $ConfigFile -if [ "$?" == "0" ]; then - DisabledDarcIntSources=() - echo "Re-enabling any disabled \"darc-int\" package sources in $ConfigFile" - DisabledDarcIntSources+=$(grep -oh '"darc-int-[^"]*" value="true"' $ConfigFile | tr -d '"') - for DisabledSourceName in ${DisabledDarcIntSources[@]} ; do - if [[ $DisabledSourceName == darc-int* ]] - then - OldDisableValue="" - NewDisableValue="" - sed -i.bak "s|$OldDisableValue|$NewDisableValue|" $ConfigFile - echo "Neutralized disablePackageSources entry for '$DisabledSourceName'" - fi - done -fi diff --git a/eng/common/core-templates/job/job.yml b/eng/common/core-templates/job/job.yml index 5ce51840619..eaed6d87e65 100644 --- a/eng/common/core-templates/job/job.yml +++ b/eng/common/core-templates/job/job.yml @@ -24,12 +24,12 @@ parameters: enablePublishBuildArtifacts: false enablePublishBuildAssets: false enablePublishTestResults: false + enablePublishing: false enableBuildRetry: false mergeTestResults: false testRunTitle: '' testResultsFormat: '' name: '' - componentGovernanceSteps: [] preSteps: [] artifactPublishSteps: [] runAsPublic: false @@ -146,9 +146,6 @@ jobs: - ${{ each step in parameters.steps }}: - ${{ step }} - - ${{ each step in parameters.componentGovernanceSteps }}: - - ${{ step }} - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - template: /eng/common/core-templates/steps/cleanup-microbuild.yml parameters: diff --git a/eng/common/core-templates/job/onelocbuild.yml b/eng/common/core-templates/job/onelocbuild.yml index c5788829a87..eefed3b667a 100644 --- a/eng/common/core-templates/job/onelocbuild.yml +++ b/eng/common/core-templates/job/onelocbuild.yml @@ -52,13 +52,13 @@ jobs: # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: name: AzurePipelines-EO - image: 1ESPT-Windows2022 + image: 1ESPT-Windows2025 demands: Cmd os: windows # If it's not devdiv, it's dnceng ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: name: $(DncEngInternalBuildPool) - image: 1es-windows-2022 + image: windows.vs2026.amd64 os: windows steps: diff --git a/eng/common/core-templates/job/publish-build-assets.yml b/eng/common/core-templates/job/publish-build-assets.yml index 348cd16376f..06f2eed0323 100644 --- a/eng/common/core-templates/job/publish-build-assets.yml +++ b/eng/common/core-templates/job/publish-build-assets.yml @@ -40,6 +40,8 @@ parameters: repositoryAlias: self + officialBuildId: '' + jobs: - job: Asset_Registry_Publish @@ -62,18 +64,23 @@ jobs: value: false # unconditional - needed for logs publishing (redactor tool version) - template: /eng/common/core-templates/post-build/common-variables.yml + - name: OfficialBuildId + ${{ if ne(parameters.officialBuildId, '') }}: + value: ${{ parameters.officialBuildId }} + ${{ else }}: + value: $(Build.BuildNumber) pool: # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: name: AzurePipelines-EO - image: 1ESPT-Windows2022 + image: 1ESPT-Windows2025 demands: Cmd os: windows # If it's not devdiv, it's dnceng ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: name: NetCore1ESPool-Publishing-Internal - image: windows.vs2019.amd64 + image: windows.vs2026.amd64 os: windows steps: - ${{ if eq(parameters.is1ESPipeline, '') }}: @@ -113,6 +120,14 @@ jobs: - task: NuGetAuthenticate@1 + # Populate internal runtime variables. + - template: /eng/common/templates/steps/enable-internal-sources.yml + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + parameters: + legacyCredential: $(dn-bot-dnceng-artifact-feeds-rw) + + - template: /eng/common/templates/steps/enable-internal-runtimes.yml + - task: AzureCLI@2 displayName: Publish Build Assets inputs: @@ -124,7 +139,10 @@ jobs: /p:ManifestsPath='$(Build.StagingDirectory)/AssetManifests' /p:IsAssetlessBuild=${{ parameters.isAssetlessBuild }} /p:MaestroApiEndpoint=https://maestro.dot.net - /p:OfficialBuildId=$(Build.BuildNumber) + /p:OfficialBuildId=$(OfficialBuildId) + -runtimeSourceFeed https://ci.dot.net/internal + -runtimeSourceFeedKey '$(dotnetbuilds-internal-container-read-token-base64)' + condition: ${{ parameters.condition }} continueOnError: ${{ parameters.continueOnError }} @@ -154,17 +172,18 @@ jobs: targetPath: '$(Build.ArtifactStagingDirectory)/MergedManifest.xml' artifactName: AssetManifests displayName: 'Publish Merged Manifest' - retryCountOnTaskFailure: 10 # for any logs being locked - sbomEnabled: false # we don't need SBOM for logs + retryCountOnTaskFailure: 10 # for any files being locked + isProduction: false # just metadata for publishing - - template: /eng/common/core-templates/steps/publish-build-artifacts.yml + - template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml parameters: is1ESPipeline: ${{ parameters.is1ESPipeline }} args: displayName: Publish ReleaseConfigs Artifact - pathToPublish: '$(Build.StagingDirectory)/ReleaseConfigs' - publishLocation: Container + targetPath: '$(Build.StagingDirectory)/ReleaseConfigs' artifactName: ReleaseConfigs + retryCountOnTaskFailure: 10 # for any files being locked + isProduction: false # just metadata for publishing - ${{ if or(eq(parameters.publishAssetsImmediately, 'true'), eq(parameters.isAssetlessBuild, 'true')) }}: - template: /eng/common/core-templates/post-build/setup-maestro-vars.yml @@ -173,6 +192,11 @@ jobs: PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} is1ESPipeline: ${{ parameters.is1ESPipeline }} + # Darc is targeting 8.0, so make sure it's installed + - task: UseDotNet@2 + inputs: + version: 8.0.x + - task: AzureCLI@2 displayName: Publish Using Darc inputs: @@ -188,9 +212,12 @@ jobs: -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' -SkipAssetsPublishing '${{ parameters.isAssetlessBuild }}' + -runtimeSourceFeed https://ci.dot.net/internal + -runtimeSourceFeedKey '$(dotnetbuilds-internal-container-read-token-base64)' - ${{ if eq(parameters.enablePublishBuildArtifacts, 'true') }}: - template: /eng/common/core-templates/steps/publish-logs.yml parameters: is1ESPipeline: ${{ parameters.is1ESPipeline }} - JobLabel: 'Publish_Artifacts_Logs' + StageLabel: 'BuildAssetRegistry' + JobLabel: 'Publish_Artifacts_Logs' diff --git a/eng/common/core-templates/job/source-build.yml b/eng/common/core-templates/job/source-build.yml index d805d5faeb9..1997c2ae00d 100644 --- a/eng/common/core-templates/job/source-build.yml +++ b/eng/common/core-templates/job/source-build.yml @@ -60,19 +60,19 @@ jobs: pool: ${{ if eq(variables['System.TeamProject'], 'public') }}: name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore-Svc-Public' ), False, 'NetCore-Public')] - demands: ImageOverride -equals build.ubuntu.2004.amd64 + demands: ImageOverride -equals build.azurelinux.3.amd64.open ${{ if eq(variables['System.TeamProject'], 'internal') }}: name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore1ESPool-Svc-Internal'), False, 'NetCore1ESPool-Internal')] - image: 1es-mariner-2 + image: build.azurelinux.3.amd64 os: linux ${{ else }}: pool: ${{ if eq(variables['System.TeamProject'], 'public') }}: name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore-Svc-Public' ), False, 'NetCore-Public')] - demands: ImageOverride -equals Build.Ubuntu.2204.Amd64.Open + demands: ImageOverride -equals build.azurelinux.3.amd64.open ${{ if eq(variables['System.TeamProject'], 'internal') }}: name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore1ESPool-Svc-Internal'), False, 'NetCore1ESPool-Internal')] - demands: ImageOverride -equals Build.Ubuntu.2204.Amd64 + demands: ImageOverride -equals build.azurelinux.3.amd64 ${{ if ne(parameters.platform.pool, '') }}: pool: ${{ parameters.platform.pool }} diff --git a/eng/common/core-templates/job/source-index-stage1.yml b/eng/common/core-templates/job/source-index-stage1.yml index 30530359a5d..76baf5c2725 100644 --- a/eng/common/core-templates/job/source-index-stage1.yml +++ b/eng/common/core-templates/job/source-index-stage1.yml @@ -3,7 +3,7 @@ parameters: sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci" preSteps: [] binlogPath: artifacts/log/Debug/Build.binlog - condition: '' + condition: eq(variables['Build.SourceBranch'], 'refs/heads/main') dependsOn: '' pool: '' is1ESPipeline: '' @@ -25,10 +25,10 @@ jobs: pool: ${{ if eq(variables['System.TeamProject'], 'public') }}: name: $(DncEngPublicBuildPool) - image: windows.vs2022.amd64.open + image: windows.vs2026preview.scout.amd64.open ${{ if eq(variables['System.TeamProject'], 'internal') }}: name: $(DncEngInternalBuildPool) - image: windows.vs2022.amd64 + image: windows.vs2026preview.scout.amd64 steps: - ${{ if eq(parameters.is1ESPipeline, '') }}: @@ -41,4 +41,4 @@ jobs: - template: /eng/common/core-templates/steps/source-index-stage1-publish.yml parameters: - binLogPath: ${{ parameters.binLogPath }} \ No newline at end of file + binLogPath: ${{ parameters.binLogPath }} diff --git a/eng/common/core-templates/jobs/jobs.yml b/eng/common/core-templates/jobs/jobs.yml index b637cb6e948..cc8cce45278 100644 --- a/eng/common/core-templates/jobs/jobs.yml +++ b/eng/common/core-templates/jobs/jobs.yml @@ -43,7 +43,12 @@ parameters: artifacts: {} is1ESPipeline: '' + + # Publishing version w/default. + publishingVersion: 3 + repositoryAlias: self + officialBuildId: '' # Internal resources (telemetry, microbuild) can only be accessed from non-public projects, # and some (Microbuild) should only be applied to non-PR cases for internal builds. @@ -101,6 +106,7 @@ jobs: parameters: is1ESPipeline: ${{ parameters.is1ESPipeline }} continueOnError: ${{ parameters.continueOnError }} + publishingVersion: ${{ parameters.publishingVersion }} dependsOn: - ${{ if ne(parameters.publishBuildAssetsDependsOn, '') }}: - ${{ each job in parameters.publishBuildAssetsDependsOn }}: @@ -116,3 +122,4 @@ jobs: artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} signingValidationAdditionalParameters: ${{ parameters.signingValidationAdditionalParameters }} repositoryAlias: ${{ parameters.repositoryAlias }} + officialBuildId: ${{ parameters.officialBuildId }} diff --git a/eng/common/core-templates/post-build/post-build.yml b/eng/common/core-templates/post-build/post-build.yml index f6f87fe5c67..905a6315e2d 100644 --- a/eng/common/core-templates/post-build/post-build.yml +++ b/eng/common/core-templates/post-build/post-build.yml @@ -9,6 +9,7 @@ parameters: default: 3 values: - 3 + - 4 - name: BARBuildId displayName: BAR Build Id @@ -120,18 +121,18 @@ stages: # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: name: AzurePipelines-EO - image: 1ESPT-Windows2022 + image: 1ESPT-Windows2025 demands: Cmd os: windows # If it's not devdiv, it's dnceng ${{ else }}: ${{ if eq(parameters.is1ESPipeline, true) }}: name: $(DncEngInternalBuildPool) - image: windows.vs2022.amd64 + image: windows.vs2026preview.scout.amd64 os: windows ${{ else }}: name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals windows.vs2022.amd64 + demands: ImageOverride -equals windows.vs2026preview.scout.amd64 steps: - template: /eng/common/core-templates/post-build/setup-maestro-vars.yml @@ -140,16 +141,30 @@ stages: PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} is1ESPipeline: ${{ parameters.is1ESPipeline }} - - task: DownloadBuildArtifacts@0 - displayName: Download Package Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: PackageArtifacts - checkDownloadedFiles: true + - ${{ if ne(parameters.publishingInfraVersion, 4) }}: + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: PackageArtifacts + checkDownloadedFiles: true + - ${{ if eq(parameters.publishingInfraVersion, 4) }}: + - task: DownloadPipelineArtifact@2 + displayName: Download Pipeline Artifacts (V4) + inputs: + itemPattern: '*/packages/**/*.nupkg' + targetPath: '$(Build.ArtifactStagingDirectory)/PipelineArtifactsDownload' + - task: CopyFiles@2 + displayName: Flatten packages to PackageArtifacts + inputs: + SourceFolder: '$(Build.ArtifactStagingDirectory)/PipelineArtifactsDownload' + Contents: '**/*.nupkg' + TargetFolder: '$(Build.ArtifactStagingDirectory)/PackageArtifacts' + flattenFolders: true - task: PowerShell@2 displayName: Validate @@ -164,18 +179,18 @@ stages: # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: name: AzurePipelines-EO - image: 1ESPT-Windows2022 + image: 1ESPT-Windows2025 demands: Cmd os: windows # If it's not devdiv, it's dnceng ${{ else }}: ${{ if eq(parameters.is1ESPipeline, true) }}: name: $(DncEngInternalBuildPool) - image: 1es-windows-2022 + image: windows.vs2026.amd64 os: windows ${{ else }}: name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals windows.vs2022.amd64 + demands: ImageOverride -equals windows.vs2026preview.scout.amd64 steps: - template: /eng/common/core-templates/post-build/setup-maestro-vars.yml parameters: @@ -183,16 +198,30 @@ stages: PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} is1ESPipeline: ${{ parameters.is1ESPipeline }} - - task: DownloadBuildArtifacts@0 - displayName: Download Package Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: PackageArtifacts - checkDownloadedFiles: true + - ${{ if ne(parameters.publishingInfraVersion, 4) }}: + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: PackageArtifacts + checkDownloadedFiles: true + - ${{ if eq(parameters.publishingInfraVersion, 4) }}: + - task: DownloadPipelineArtifact@2 + displayName: Download Pipeline Artifacts (V4) + inputs: + itemPattern: '*/packages/**/*.nupkg' + targetPath: '$(Build.ArtifactStagingDirectory)/PipelineArtifactsDownload' + - task: CopyFiles@2 + displayName: Flatten packages to PackageArtifacts + inputs: + SourceFolder: '$(Build.ArtifactStagingDirectory)/PipelineArtifactsDownload' + Contents: '**/*.nupkg' + TargetFolder: '$(Build.ArtifactStagingDirectory)/PackageArtifacts' + flattenFolders: true # This is necessary whenever we want to publish/restore to an AzDO private feed # Since sdk-task.ps1 tries to restore packages we need to do this authentication here @@ -225,18 +254,18 @@ stages: # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: name: AzurePipelines-EO - image: 1ESPT-Windows2022 + image: 1ESPT-Windows2025 demands: Cmd os: windows # If it's not devdiv, it's dnceng ${{ else }}: ${{ if eq(parameters.is1ESPipeline, true) }}: name: $(DncEngInternalBuildPool) - image: 1es-windows-2022 + image: windows.vs2026.amd64 os: windows ${{ else }}: name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals windows.vs2022.amd64 + demands: ImageOverride -equals windows.vs2026preview.scout.amd64 steps: - template: /eng/common/core-templates/post-build/setup-maestro-vars.yml parameters: @@ -244,16 +273,30 @@ stages: PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} is1ESPipeline: ${{ parameters.is1ESPipeline }} - - task: DownloadBuildArtifacts@0 - displayName: Download Blob Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: BlobArtifacts - checkDownloadedFiles: true + - ${{ if ne(parameters.publishingInfraVersion, 4) }}: + - task: DownloadBuildArtifacts@0 + displayName: Download Blob Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: BlobArtifacts + checkDownloadedFiles: true + - ${{ if eq(parameters.publishingInfraVersion, 4) }}: + - task: DownloadPipelineArtifact@2 + displayName: Download Pipeline Artifacts (V4) + inputs: + itemPattern: '*/assets/**' + targetPath: '$(Build.ArtifactStagingDirectory)/PipelineArtifactsDownload' + - task: CopyFiles@2 + displayName: Flatten assets to BlobArtifacts + inputs: + SourceFolder: '$(Build.ArtifactStagingDirectory)/PipelineArtifactsDownload' + Contents: '**/*' + TargetFolder: '$(Build.ArtifactStagingDirectory)/BlobArtifacts' + flattenFolders: true - task: PowerShell@2 displayName: Validate @@ -286,18 +329,18 @@ stages: # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: name: AzurePipelines-EO - image: 1ESPT-Windows2022 + image: 1ESPT-Windows2025 demands: Cmd os: windows # If it's not devdiv, it's dnceng ${{ else }}: ${{ if eq(parameters.is1ESPipeline, true) }}: name: NetCore1ESPool-Publishing-Internal - image: windows.vs2019.amd64 + image: windows.vs2026.amd64 os: windows ${{ else }}: name: NetCore1ESPool-Publishing-Internal - demands: ImageOverride -equals windows.vs2019.amd64 + demands: ImageOverride -equals windows.vs2026.amd64 steps: - template: /eng/common/core-templates/post-build/setup-maestro-vars.yml parameters: @@ -307,6 +350,18 @@ stages: - task: NuGetAuthenticate@1 + # Populate internal runtime variables. + - template: /eng/common/templates/steps/enable-internal-sources.yml + parameters: + legacyCredential: $(dn-bot-dnceng-artifact-feeds-rw) + + - template: /eng/common/templates/steps/enable-internal-runtimes.yml + + # Darc is targeting 8.0, so make sure it's installed + - task: UseDotNet@2 + inputs: + version: 8.0.x + - task: AzureCLI@2 displayName: Publish Using Darc inputs: @@ -316,10 +371,12 @@ stages: scriptPath: $(System.DefaultWorkingDirectory)/eng/common/post-build/publish-using-darc.ps1 arguments: > -BuildId $(BARBuildId) - -PublishingInfraVersion ${{ parameters.publishingInfraVersion }} + -PublishingInfraVersion 3 -AzdoToken '$(System.AccessToken)' -WaitPublishingFinish true -RequireDefaultChannels ${{ parameters.requireDefaultChannels }} -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' -SkipAssetsPublishing '${{ parameters.isAssetlessBuild }}' + -runtimeSourceFeed https://ci.dot.net/internal + -runtimeSourceFeedKey '$(dotnetbuilds-internal-container-read-token-base64)' diff --git a/eng/common/core-templates/post-build/setup-maestro-vars.yml b/eng/common/core-templates/post-build/setup-maestro-vars.yml index a7abd58c4bb..6dfa99ec5e3 100644 --- a/eng/common/core-templates/post-build/setup-maestro-vars.yml +++ b/eng/common/core-templates/post-build/setup-maestro-vars.yml @@ -8,12 +8,11 @@ steps: - 'Illegal entry point, is1ESPipeline is not defined. Repository yaml should not directly reference templates in core-templates folder.': error - ${{ if eq(coalesce(parameters.PromoteToChannelIds, 0), 0) }}: - - task: DownloadBuildArtifacts@0 + - task: DownloadPipelineArtifact@2 displayName: Download Release Configs inputs: - buildType: current artifactName: ReleaseConfigs - checkDownloadedFiles: true + targetPath: '$(Build.StagingDirectory)/ReleaseConfigs' - task: AzureCLI@2 name: setReleaseVars diff --git a/eng/common/core-templates/steps/component-governance.yml b/eng/common/core-templates/steps/component-governance.yml deleted file mode 100644 index cf0649aa956..00000000000 --- a/eng/common/core-templates/steps/component-governance.yml +++ /dev/null @@ -1,16 +0,0 @@ -parameters: - disableComponentGovernance: false - componentGovernanceIgnoreDirectories: '' - is1ESPipeline: false - displayName: 'Component Detection' - -steps: -- ${{ if eq(parameters.disableComponentGovernance, 'true') }}: - - script: echo "##vso[task.setvariable variable=skipComponentGovernanceDetection]true" - displayName: Set skipComponentGovernanceDetection variable -- ${{ if ne(parameters.disableComponentGovernance, 'true') }}: - - task: ComponentGovernanceComponentDetection@0 - continueOnError: true - displayName: ${{ parameters.displayName }} - inputs: - ignoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} diff --git a/eng/common/core-templates/steps/generate-sbom.yml b/eng/common/core-templates/steps/generate-sbom.yml index c05f6502797..aad0a8aeda3 100644 --- a/eng/common/core-templates/steps/generate-sbom.yml +++ b/eng/common/core-templates/steps/generate-sbom.yml @@ -1,54 +1,14 @@ -# BuildDropPath - The root folder of the drop directory for which the manifest file will be generated. -# PackageName - The name of the package this SBOM represents. -# PackageVersion - The version of the package this SBOM represents. -# ManifestDirPath - The path of the directory where the generated manifest files will be placed -# IgnoreDirectories - Directories to ignore for SBOM generation. This will be passed through to the CG component detector. - parameters: - PackageVersion: 10.0.0 - BuildDropPath: '$(System.DefaultWorkingDirectory)/artifacts' - PackageName: '.NET' - ManifestDirPath: $(Build.ArtifactStagingDirectory)/sbom - IgnoreDirectories: '' - sbomContinueOnError: true - is1ESPipeline: false - # disable publishArtifacts if some other step is publishing the artifacts (like job.yml). - publishArtifacts: true + PackageVersion: unused + BuildDropPath: unused + PackageName: unused + ManifestDirPath: unused + IgnoreDirectories: unused + sbomContinueOnError: unused + is1ESPipeline: unused + publishArtifacts: unused steps: -- task: PowerShell@2 - displayName: Prep for SBOM generation in (Non-linux) - condition: or(eq(variables['Agent.Os'], 'Windows_NT'), eq(variables['Agent.Os'], 'Darwin')) - inputs: - filePath: ./eng/common/generate-sbom-prep.ps1 - arguments: ${{parameters.manifestDirPath}} - -# Chmodding is a workaround for https://github.com/dotnet/arcade/issues/8461 - script: | - chmod +x ./eng/common/generate-sbom-prep.sh - ./eng/common/generate-sbom-prep.sh ${{parameters.manifestDirPath}} - displayName: Prep for SBOM generation in (Linux) - condition: eq(variables['Agent.Os'], 'Linux') - continueOnError: ${{ parameters.sbomContinueOnError }} - -- task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 - displayName: 'Generate SBOM manifest' - continueOnError: ${{ parameters.sbomContinueOnError }} - inputs: - PackageName: ${{ parameters.packageName }} - BuildDropPath: ${{ parameters.buildDropPath }} - PackageVersion: ${{ parameters.packageVersion }} - ManifestDirPath: ${{ parameters.manifestDirPath }}/$(ARTIFACT_NAME) - ${{ if ne(parameters.IgnoreDirectories, '') }}: - AdditionalComponentDetectorArgs: '--IgnoreDirectories ${{ parameters.IgnoreDirectories }}' - -- ${{ if eq(parameters.publishArtifacts, 'true')}}: - - template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml - parameters: - is1ESPipeline: ${{ parameters.is1ESPipeline }} - args: - displayName: Publish SBOM manifest - continueOnError: ${{parameters.sbomContinueOnError}} - targetPath: '${{ parameters.manifestDirPath }}' - artifactName: $(ARTIFACT_NAME) - + echo "##vso[task.logissue type=warning]Including generate-sbom.yml is deprecated, SBOM generation is handled 1ES PT now. Remove this include." + displayName: Issue generate-sbom.yml deprecation warning diff --git a/eng/common/core-templates/steps/install-microbuild.yml b/eng/common/core-templates/steps/install-microbuild.yml index d6b9878f54d..553fce66b94 100644 --- a/eng/common/core-templates/steps/install-microbuild.yml +++ b/eng/common/core-templates/steps/install-microbuild.yml @@ -11,9 +11,8 @@ parameters: # Unfortunately, _SignType can't be used to exclude the use of the service connection in non-real sign scenarios. The # variable is not available in template expression. _SignType has a very large proliferation across .NET, so replacing it is tough. microbuildUseESRP: true - # Location of the MicroBuild output folder - # NOTE: There's something that relies on this being in the "default" source directory for tasks such as Signing to work properly. - microBuildOutputFolder: '$(Build.SourcesDirectory)' + # Microbuild installation directory + microBuildOutputFolder: $(Agent.TempDirectory)/MicroBuild continueOnError: false @@ -26,8 +25,27 @@ steps: inputs: packageType: sdk version: 8.0.x - installationPath: ${{ parameters.microBuildOutputFolder }}/.dotnet - workingDirectory: ${{ parameters.microBuildOutputFolder }} + installationPath: ${{ parameters.microBuildOutputFolder }}/.dotnet-microbuild + condition: and(succeeded(), ne(variables['Agent.Os'], 'Windows_NT')) + + - script: | + set -euo pipefail + + # UseDotNet@2 prepends the dotnet executable path to the PATH variable, so we can call dotnet directly + version=$(dotnet --version) + cat << 'EOF' > ${{ parameters.microBuildOutputFolder }}/global.json + { + "sdk": { + "version": "$version", + "paths": [ + "${{ parameters.microBuildOutputFolder }}/.dotnet-microbuild" + ], + "errorMessage": "The .NET SDK version $version is required to install the MicroBuild signing plugin." + } + } + EOF + displayName: 'Add global.json to MicroBuild Installation path' + workingDirectory: ${{ parameters.microBuildOutputFolder }} condition: and(succeeded(), ne(variables['Agent.Os'], 'Windows_NT')) - script: | @@ -77,6 +95,7 @@ steps: signType: $(_SignType) zipSources: false feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json + workingDirectory: ${{ parameters.microBuildOutputFolder }} ${{ if eq(parameters.microbuildUseESRP, true) }}: ConnectedServiceName: 'MicroBuild Signing Task (DevDiv)' ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: diff --git a/eng/common/core-templates/steps/publish-logs.yml b/eng/common/core-templates/steps/publish-logs.yml index 10f825e270a..4eed0312b80 100644 --- a/eng/common/core-templates/steps/publish-logs.yml +++ b/eng/common/core-templates/steps/publish-logs.yml @@ -26,10 +26,11 @@ steps: # If the file exists - sensitive data for redaction will be sourced from it # (single entry per line, lines starting with '# ' are considered comments and skipped) arguments: -InputPath '$(System.DefaultWorkingDirectory)/PostBuildLogs' - -BinlogToolVersion ${{parameters.BinlogToolVersion}} + -BinlogToolVersion '${{parameters.BinlogToolVersion}}' -TokensFilePath '$(System.DefaultWorkingDirectory)/eng/BinlogSecretsRedactionFile.txt' + -runtimeSourceFeed https://ci.dot.net/internal + -runtimeSourceFeedKey '$(dotnetbuilds-internal-container-read-token-base64)' '$(publishing-dnceng-devdiv-code-r-build-re)' - '$(MaestroAccessToken)' '$(dn-bot-all-orgs-artifact-feeds-rw)' '$(akams-client-id)' '$(microsoft-symbol-server-pat)' @@ -49,13 +50,14 @@ steps: TargetFolder: '$(Build.ArtifactStagingDirectory)/PostBuildLogs' condition: always() -- template: /eng/common/core-templates/steps/publish-build-artifacts.yml +- template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml parameters: is1ESPipeline: ${{ parameters.is1ESPipeline }} args: displayName: Publish Logs - pathToPublish: '$(Build.ArtifactStagingDirectory)/PostBuildLogs' - publishLocation: Container - artifactName: PostBuildLogs + targetPath: '$(Build.ArtifactStagingDirectory)/PostBuildLogs' + artifactName: PostBuildLogs_${{ parameters.StageLabel }}_${{ parameters.JobLabel }}_Attempt$(System.JobAttempt) continueOnError: true condition: always() + retryCountOnTaskFailure: 10 # for any files being locked + isProduction: false # logs are non-production artifacts diff --git a/eng/common/core-templates/steps/source-build.yml b/eng/common/core-templates/steps/source-build.yml index acf16ed3496..09ae5cd73ae 100644 --- a/eng/common/core-templates/steps/source-build.yml +++ b/eng/common/core-templates/steps/source-build.yml @@ -24,7 +24,7 @@ steps: # in the default public locations. internalRuntimeDownloadArgs= if [ '$(dotnetbuilds-internal-container-read-token-base64)' != '$''(dotnetbuilds-internal-container-read-token-base64)' ]; then - internalRuntimeDownloadArgs='/p:DotNetRuntimeSourceFeed=https://ci.dot.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) --runtimesourcefeed https://ci.dot.net/internal --runtimesourcefeedkey $(dotnetbuilds-internal-container-read-token-base64)' + internalRuntimeDownloadArgs='/p:DotNetRuntimeSourceFeed=https://ci.dot.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) --runtimesourcefeed https://ci.dot.net/internal --runtimesourcefeedkey '$(dotnetbuilds-internal-container-read-token-base64)'' fi buildConfig=Release @@ -62,4 +62,4 @@ steps: artifactName: BuildLogs_SourceBuild_${{ parameters.platform.name }}_Attempt$(System.JobAttempt) continueOnError: true condition: succeededOrFailed() - sbomEnabled: false # we don't need SBOM for logs + isProduction: false # logs are non-production artifacts diff --git a/eng/common/darc-init.ps1 b/eng/common/darc-init.ps1 index e3374310563..a5be41db690 100644 --- a/eng/common/darc-init.ps1 +++ b/eng/common/darc-init.ps1 @@ -29,11 +29,11 @@ function InstallDarcCli ($darcVersion, $toolpath) { Write-Host "Installing Darc CLI version $darcVersion..." Write-Host 'You may need to restart your command window if this is the first dotnet tool you have installed.' if (-not $toolpath) { - Write-Host "'$dotnet' tool install $darcCliPackageName --version $darcVersion --add-source '$arcadeServicesSource' -v $verbosity -g" - & "$dotnet" tool install $darcCliPackageName --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity -g + Write-Host "'$dotnet' tool install $darcCliPackageName --version $darcVersion --source '$arcadeServicesSource' -v $verbosity -g" + & "$dotnet" tool install $darcCliPackageName --version $darcVersion --source "$arcadeServicesSource" -v $verbosity -g }else { - Write-Host "'$dotnet' tool install $darcCliPackageName --version $darcVersion --add-source '$arcadeServicesSource' -v $verbosity --tool-path '$toolpath'" - & "$dotnet" tool install $darcCliPackageName --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity --tool-path "$toolpath" + Write-Host "'$dotnet' tool install $darcCliPackageName --version $darcVersion --source '$arcadeServicesSource' -v $verbosity --tool-path '$toolpath'" + & "$dotnet" tool install $darcCliPackageName --version $darcVersion --source "$arcadeServicesSource" -v $verbosity --tool-path "$toolpath" } } diff --git a/eng/common/darc-init.sh b/eng/common/darc-init.sh index e889f439b8d..e6ba4ee28c1 100755 --- a/eng/common/darc-init.sh +++ b/eng/common/darc-init.sh @@ -73,9 +73,9 @@ function InstallDarcCli { echo "Installing Darc CLI version $darcVersion..." echo "You may need to restart your command shell if this is the first dotnet tool you have installed." if [ -z "$toolpath" ]; then - echo $($dotnet_root/dotnet tool install $darc_cli_package_name --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity -g) + echo $($dotnet_root/dotnet tool install $darc_cli_package_name --version $darcVersion --source "$arcadeServicesSource" -v $verbosity -g) else - echo $($dotnet_root/dotnet tool install $darc_cli_package_name --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity --tool-path "$toolpath") + echo $($dotnet_root/dotnet tool install $darc_cli_package_name --version $darcVersion --source "$arcadeServicesSource" -v $verbosity --tool-path "$toolpath") fi } diff --git a/eng/common/generate-sbom-prep.ps1 b/eng/common/generate-sbom-prep.ps1 deleted file mode 100644 index a0c7d792a76..00000000000 --- a/eng/common/generate-sbom-prep.ps1 +++ /dev/null @@ -1,29 +0,0 @@ -Param( - [Parameter(Mandatory=$true)][string] $ManifestDirPath # Manifest directory where sbom will be placed -) - -. $PSScriptRoot\pipeline-logging-functions.ps1 - -# Normally - we'd listen to the manifest path given, but 1ES templates will overwrite if this level gets uploaded directly -# with their own overwriting ours. So we create it as a sub directory of the requested manifest path. -$ArtifactName = "${env:SYSTEM_STAGENAME}_${env:AGENT_JOBNAME}_SBOM" -$SafeArtifactName = $ArtifactName -replace '["/:<>\\|?@*"() ]', '_' -$SbomGenerationDir = Join-Path $ManifestDirPath $SafeArtifactName - -Write-Host "Artifact name before : $ArtifactName" -Write-Host "Artifact name after : $SafeArtifactName" - -Write-Host "Creating dir $ManifestDirPath" - -# create directory for sbom manifest to be placed -if (!(Test-Path -path $SbomGenerationDir)) -{ - New-Item -ItemType Directory -path $SbomGenerationDir - Write-Host "Successfully created directory $SbomGenerationDir" -} -else{ - Write-PipelineTelemetryError -category 'Build' "Unable to create sbom folder." -} - -Write-Host "Updating artifact name" -Write-Host "##vso[task.setvariable variable=ARTIFACT_NAME]$SafeArtifactName" diff --git a/eng/common/generate-sbom-prep.sh b/eng/common/generate-sbom-prep.sh deleted file mode 100755 index b8ecca72bbf..00000000000 --- a/eng/common/generate-sbom-prep.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env bash - -source="${BASH_SOURCE[0]}" - -# resolve $SOURCE until the file is no longer a symlink -while [[ -h $source ]]; do - scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" - source="$(readlink "$source")" - - # if $source was a relative symlink, we need to resolve it relative to the path where the - # symlink file was located - [[ $source != /* ]] && source="$scriptroot/$source" -done -scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" -. $scriptroot/pipeline-logging-functions.sh - - -# replace all special characters with _, some builds use special characters like : in Agent.Jobname, that is not a permissible name while uploading artifacts. -artifact_name=$SYSTEM_STAGENAME"_"$AGENT_JOBNAME"_SBOM" -safe_artifact_name="${artifact_name//["/:<>\\|?@*$" ]/_}" -manifest_dir=$1 - -# Normally - we'd listen to the manifest path given, but 1ES templates will overwrite if this level gets uploaded directly -# with their own overwriting ours. So we create it as a sub directory of the requested manifest path. -sbom_generation_dir="$manifest_dir/$safe_artifact_name" - -if [ ! -d "$sbom_generation_dir" ] ; then - mkdir -p "$sbom_generation_dir" - echo "Sbom directory created." $sbom_generation_dir -else - Write-PipelineTelemetryError -category 'Build' "Unable to create sbom folder." -fi - -echo "Artifact name before : "$artifact_name -echo "Artifact name after : "$safe_artifact_name -export ARTIFACT_NAME=$safe_artifact_name -echo "##vso[task.setvariable variable=ARTIFACT_NAME]$safe_artifact_name" - -exit 0 diff --git a/eng/common/internal-feed-operations.ps1 b/eng/common/internal-feed-operations.ps1 index 92b77347d99..c282d3ae403 100644 --- a/eng/common/internal-feed-operations.ps1 +++ b/eng/common/internal-feed-operations.ps1 @@ -26,7 +26,7 @@ function SetupCredProvider { $url = 'https://raw.githubusercontent.com/microsoft/artifacts-credprovider/master/helpers/installcredprovider.ps1' Write-Host "Writing the contents of 'installcredprovider.ps1' locally..." - Invoke-WebRequest $url -OutFile installcredprovider.ps1 + Invoke-WebRequest $url -UseBasicParsing -OutFile installcredprovider.ps1 Write-Host 'Installing plugin...' .\installcredprovider.ps1 -Force diff --git a/eng/common/post-build/nuget-verification.ps1 b/eng/common/post-build/nuget-verification.ps1 index a365194a938..eea88e653c9 100644 --- a/eng/common/post-build/nuget-verification.ps1 +++ b/eng/common/post-build/nuget-verification.ps1 @@ -30,7 +30,7 @@ [CmdletBinding(PositionalBinding = $false)] param( [string]$NuGetExePath, - [string]$PackageSource = "https://api.nuget.org/v3/index.json", + [string]$PackageSource = "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public/nuget/v3/index.json", [string]$DownloadPath, [Parameter(ValueFromRemainingArguments = $true)] [string[]]$args @@ -65,7 +65,7 @@ if ($NuGetExePath) { Write-Host "Downloading nuget.exe from $nugetExeUrl..." $ProgressPreference = 'SilentlyContinue' try { - Invoke-WebRequest $nugetExeUrl -OutFile $downloadedNuGetExe + Invoke-WebRequest $nugetExeUrl -UseBasicParsing -OutFile $downloadedNuGetExe $ProgressPreference = 'Continue' } catch { $ProgressPreference = 'Continue' diff --git a/eng/common/post-build/publish-using-darc.ps1 b/eng/common/post-build/publish-using-darc.ps1 index 1eda208a3bb..48e55598bdd 100644 --- a/eng/common/post-build/publish-using-darc.ps1 +++ b/eng/common/post-build/publish-using-darc.ps1 @@ -7,7 +7,9 @@ param( [Parameter(Mandatory=$false)][string] $ArtifactsPublishingAdditionalParameters, [Parameter(Mandatory=$false)][string] $SymbolPublishingAdditionalParameters, [Parameter(Mandatory=$false)][string] $RequireDefaultChannels, - [Parameter(Mandatory=$false)][string] $SkipAssetsPublishing + [Parameter(Mandatory=$false)][string] $SkipAssetsPublishing, + [Parameter(Mandatory=$false)][string] $runtimeSourceFeed, + [Parameter(Mandatory=$false)][string] $runtimeSourceFeedKey ) try { diff --git a/eng/common/post-build/redact-logs.ps1 b/eng/common/post-build/redact-logs.ps1 index b7fc1959150..c1e4104b79a 100644 --- a/eng/common/post-build/redact-logs.ps1 +++ b/eng/common/post-build/redact-logs.ps1 @@ -7,8 +7,9 @@ param( # File with strings to redact - separated by newlines. # For comments start the line with '# ' - such lines are ignored [Parameter(Mandatory=$false)][string] $TokensFilePath, - [Parameter(ValueFromRemainingArguments=$true)][String[]]$TokensToRedact -) + [Parameter(ValueFromRemainingArguments=$true)][String[]]$TokensToRedact, + [Parameter(Mandatory=$false)][string] $runtimeSourceFeed, + [Parameter(Mandatory=$false)][string] $runtimeSourceFeedKey) try { $ErrorActionPreference = 'Stop' @@ -47,8 +48,8 @@ try { Write-Host "Installing Binlog redactor CLI..." Write-Host "'$dotnet' new tool-manifest" & "$dotnet" new tool-manifest - Write-Host "'$dotnet' tool install $packageName --local --add-source '$PackageFeed' -v $verbosity --version $BinlogToolVersion" - & "$dotnet" tool install $packageName --local --add-source "$PackageFeed" -v $verbosity --version $BinlogToolVersion + Write-Host "'$dotnet' tool install $packageName --local --source '$PackageFeed' -v $verbosity --version $BinlogToolVersion" + & "$dotnet" tool install $packageName --local --source "$PackageFeed" -v $verbosity --version $BinlogToolVersion if (Test-Path $TokensFilePath) { Write-Host "Adding additional sensitive data for redaction from file: " $TokensFilePath diff --git a/eng/common/sdk-task.ps1 b/eng/common/sdk-task.ps1 index b62e132d32a..b64b66a6275 100644 --- a/eng/common/sdk-task.ps1 +++ b/eng/common/sdk-task.ps1 @@ -9,6 +9,8 @@ Param( [switch][Alias('nobl')]$excludeCIBinaryLog, [switch]$noWarnAsError, [switch] $help, + [string] $runtimeSourceFeed = '', + [string] $runtimeSourceFeedKey = '', [Parameter(ValueFromRemainingArguments=$true)][String[]]$properties ) @@ -68,7 +70,7 @@ try { $GlobalJson.tools | Add-Member -Name "vs" -Value (ConvertFrom-Json "{ `"version`": `"16.5`" }") -MemberType NoteProperty } if( -not ($GlobalJson.tools.PSObject.Properties.Name -match "xcopy-msbuild" )) { - $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "17.13.0" -MemberType NoteProperty + $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "18.0.0" -MemberType NoteProperty } if ($GlobalJson.tools."xcopy-msbuild".Trim() -ine "none") { $xcopyMSBuildToolsFolder = InitializeXCopyMSBuild $GlobalJson.tools."xcopy-msbuild" -install $true diff --git a/eng/common/template-guidance.md b/eng/common/template-guidance.md index 4bf4cf41bd7..e2b07a865f1 100644 --- a/eng/common/template-guidance.md +++ b/eng/common/template-guidance.md @@ -82,7 +82,6 @@ eng\common\ publish-build-artifacts.yml (logic) publish-pipeline-artifacts.yml (logic) component-governance.yml (shim) - generate-sbom.yml (shim) publish-logs.yml (shim) retain-build.yml (shim) send-to-helix.yml (shim) @@ -107,7 +106,6 @@ eng\common\ setup-maestro-vars.yml (logic) steps\ component-governance.yml (logic) - generate-sbom.yml (logic) publish-build-artifacts.yml (redirect) publish-logs.yml (logic) publish-pipeline-artifacts.yml (redirect) diff --git a/eng/common/templates-official/job/job.yml b/eng/common/templates-official/job/job.yml index 92a0664f564..d68e9fbc265 100644 --- a/eng/common/templates-official/job/job.yml +++ b/eng/common/templates-official/job/job.yml @@ -1,24 +1,15 @@ parameters: -# Sbom related params - enableSbom: true runAsPublic: false - PackageVersion: 9.0.0 - BuildDropPath: '$(System.DefaultWorkingDirectory)/artifacts' +# Sbom related params, unused now and can eventually be removed + enableSbom: unused + PackageVersion: unused + BuildDropPath: unused jobs: - template: /eng/common/core-templates/job/job.yml parameters: is1ESPipeline: true - componentGovernanceSteps: - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.enableSbom, 'true')) }}: - - template: /eng/common/templates/steps/generate-sbom.yml - parameters: - PackageVersion: ${{ parameters.packageVersion }} - BuildDropPath: ${{ parameters.buildDropPath }} - ManifestDirPath: $(Build.ArtifactStagingDirectory)/sbom - publishArtifacts: false - # publish artifacts # for 1ES managed templates, use the templateContext.output to handle multiple outputs. templateContext: @@ -26,12 +17,19 @@ jobs: outputs: - ${{ if ne(parameters.artifacts.publish, '') }}: - ${{ if and(ne(parameters.artifacts.publish.artifacts, 'false'), ne(parameters.artifacts.publish.artifacts, '')) }}: - - output: buildArtifacts + - output: pipelineArtifact displayName: Publish pipeline artifacts - PathtoPublish: '$(Build.ArtifactStagingDirectory)/artifacts' - ArtifactName: ${{ coalesce(parameters.artifacts.publish.artifacts.name , 'Artifacts_$(Agent.Os)_$(_BuildConfig)') }} - condition: always() - retryCountOnTaskFailure: 10 # for any logs being locked + targetPath: '$(Build.ArtifactStagingDirectory)/artifacts' + artifactName: ${{ coalesce(parameters.artifacts.publish.artifacts.name , 'Artifacts_$(Agent.Os)_$(_BuildConfig)') }} + condition: succeeded() + retryCountOnTaskFailure: 10 # for any files being locked + continueOnError: true + - output: pipelineArtifact + displayName: Publish pipeline artifacts + targetPath: '$(Build.ArtifactStagingDirectory)/artifacts' + artifactName: ${{ coalesce(parameters.artifacts.publish.artifacts.name , 'Artifacts_$(Agent.Os)_$(_BuildConfig)') }}_Attempt$(System.JobAttempt) + condition: not(succeeded()) + retryCountOnTaskFailure: 10 # for any files being locked continueOnError: true - ${{ if and(ne(parameters.artifacts.publish.logs, 'false'), ne(parameters.artifacts.publish.logs, '')) }}: - output: pipelineArtifact @@ -40,18 +38,18 @@ jobs: displayName: 'Publish logs' continueOnError: true condition: always() - retryCountOnTaskFailure: 10 # for any logs being locked - sbomEnabled: false # we don't need SBOM for logs + retryCountOnTaskFailure: 10 # for any files being locked + isProduction: false # logs are non-production artifacts - ${{ if eq(parameters.enablePublishBuildArtifacts, true) }}: - - output: buildArtifacts + - output: pipelineArtifact displayName: Publish Logs - PathtoPublish: '$(Build.ArtifactStagingDirectory)/artifacts/log/$(_BuildConfig)' - publishLocation: Container - ArtifactName: ${{ coalesce(parameters.enablePublishBuildArtifacts.artifactName, '$(Agent.Os)_$(Agent.JobName)_Attempt$(System.JobAttempt)' ) }} + targetPath: '$(Build.ArtifactStagingDirectory)/artifacts/log/$(_BuildConfig)' + artifactName: ${{ coalesce(parameters.enablePublishBuildArtifacts.artifactName, '$(Agent.Os)_$(Agent.JobName)_Attempt$(System.JobAttempt)' ) }} continueOnError: true condition: always() - sbomEnabled: false # we don't need SBOM for logs + retryCountOnTaskFailure: 10 # for any files being locked + isProduction: false # logs are non-production artifacts - ${{ if eq(parameters.enableBuildRetry, 'true') }}: - output: pipelineArtifact @@ -59,14 +57,20 @@ jobs: artifactName: 'BuildConfiguration' displayName: 'Publish build retry configuration' continueOnError: true - sbomEnabled: false # we don't need SBOM for BuildConfiguration + retryCountOnTaskFailure: 10 # for any files being locked + isProduction: false # BuildConfiguration is a non-production artifact - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.enableSbom, 'true')) }}: + # V4 publishing: automatically publish staged artifacts as a pipeline artifact. + # The artifact name matches the SDK's FutureArtifactName ($(System.PhaseName)_Artifacts), + # which is encoded in the asset manifest for downstream publishing to discover. + # Jobs can opt in by setting enablePublishing: true. + - ${{ if and(eq(parameters.publishingVersion, 4), eq(parameters.enablePublishing, 'true')) }}: - output: pipelineArtifact - displayName: Publish SBOM manifest + displayName: 'Publish V4 pipeline artifacts' + targetPath: '$(Build.ArtifactStagingDirectory)/artifacts' + artifactName: '$(System.PhaseName)_Artifacts' continueOnError: true - targetPath: $(Build.ArtifactStagingDirectory)/sbom - artifactName: $(ARTIFACT_NAME) + retryCountOnTaskFailure: 10 # for any files being locked # add any outputs provided via root yaml - ${{ if ne(parameters.templateContext.outputs, '') }}: diff --git a/eng/common/templates-official/steps/component-governance.yml b/eng/common/templates-official/steps/component-governance.yml deleted file mode 100644 index 30bb3985ca2..00000000000 --- a/eng/common/templates-official/steps/component-governance.yml +++ /dev/null @@ -1,7 +0,0 @@ -steps: -- template: /eng/common/core-templates/steps/component-governance.yml - parameters: - is1ESPipeline: true - - ${{ each parameter in parameters }}: - ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates-official/steps/publish-pipeline-artifacts.yml b/eng/common/templates-official/steps/publish-pipeline-artifacts.yml index 172f9f0fdc9..9e5981365e5 100644 --- a/eng/common/templates-official/steps/publish-pipeline-artifacts.yml +++ b/eng/common/templates-official/steps/publish-pipeline-artifacts.yml @@ -24,5 +24,7 @@ steps: artifactName: ${{ parameters.args.artifactName }} ${{ if parameters.args.properties }}: properties: ${{ parameters.args.properties }} - ${{ if parameters.args.sbomEnabled }}: + ${{ if ne(parameters.args.sbomEnabled, '') }}: sbomEnabled: ${{ parameters.args.sbomEnabled }} + ${{ if ne(parameters.args.isProduction, '') }}: + isProduction: ${{ parameters.args.isProduction }} diff --git a/eng/common/templates-official/variables/pool-providers.yml b/eng/common/templates-official/variables/pool-providers.yml index 1f308b24efc..2cc3ae305d5 100644 --- a/eng/common/templates-official/variables/pool-providers.yml +++ b/eng/common/templates-official/variables/pool-providers.yml @@ -23,7 +23,7 @@ # # pool: # name: $(DncEngInternalBuildPool) -# image: 1es-windows-2022 +# image: windows.vs2026.amd64 variables: # Coalesce the target and source branches so we know when a PR targets a release branch diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml index 238fa0818f7..5e261f34db4 100644 --- a/eng/common/templates/job/job.yml +++ b/eng/common/templates/job/job.yml @@ -1,12 +1,12 @@ parameters: enablePublishBuildArtifacts: false - disableComponentGovernance: '' - componentGovernanceIgnoreDirectories: '' -# Sbom related params - enableSbom: true runAsPublic: false - PackageVersion: 9.0.0 - BuildDropPath: '$(System.DefaultWorkingDirectory)/artifacts' +# CG related params, unused now and can eventually be removed + disableComponentGovernance: unused +# Sbom related params, unused now and can eventually be removed + enableSbom: unused + PackageVersion: unused + BuildDropPath: unused jobs: - template: /eng/common/core-templates/job/job.yml @@ -21,32 +21,34 @@ jobs: - ${{ each step in parameters.steps }}: - ${{ step }} - componentGovernanceSteps: - - template: /eng/common/templates/steps/component-governance.yml - parameters: - ${{ if eq(parameters.disableComponentGovernance, '') }}: - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.runAsPublic, 'false'), or(startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/dotnet/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/microsoft/'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))) }}: - disableComponentGovernance: false - ${{ else }}: - disableComponentGovernance: true - ${{ else }}: - disableComponentGovernance: ${{ parameters.disableComponentGovernance }} - componentGovernanceIgnoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} + # we don't run CG in public + - ${{ if eq(variables['System.TeamProject'], 'public') }}: + - script: echo "##vso[task.setvariable variable=skipComponentGovernanceDetection]true" + displayName: Set skipComponentGovernanceDetection variable artifactPublishSteps: - ${{ if ne(parameters.artifacts.publish, '') }}: - ${{ if and(ne(parameters.artifacts.publish.artifacts, 'false'), ne(parameters.artifacts.publish.artifacts, '')) }}: - - template: /eng/common/core-templates/steps/publish-build-artifacts.yml + - template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml parameters: is1ESPipeline: false args: displayName: Publish pipeline artifacts - pathToPublish: '$(Build.ArtifactStagingDirectory)/artifacts' - publishLocation: Container + targetPath: '$(Build.ArtifactStagingDirectory)/artifacts' artifactName: ${{ coalesce(parameters.artifacts.publish.artifacts.name , 'Artifacts_$(Agent.Os)_$(_BuildConfig)') }} continueOnError: true - condition: always() - retryCountOnTaskFailure: 10 # for any logs being locked + condition: succeeded() + retryCountOnTaskFailure: 10 # for any files being locked + - template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml + parameters: + is1ESPipeline: false + args: + displayName: Publish pipeline artifacts + targetPath: '$(Build.ArtifactStagingDirectory)/artifacts' + artifactName: ${{ coalesce(parameters.artifacts.publish.artifacts.name , 'Artifacts_$(Agent.Os)_$(_BuildConfig)') }}_Attempt$(System.JobAttempt) + continueOnError: true + condition: not(succeeded()) + retryCountOnTaskFailure: 10 # for any files being locked - ${{ if and(ne(parameters.artifacts.publish.logs, 'false'), ne(parameters.artifacts.publish.logs, '')) }}: - template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml parameters: @@ -57,20 +59,19 @@ jobs: displayName: 'Publish logs' continueOnError: true condition: always() - retryCountOnTaskFailure: 10 # for any logs being locked - sbomEnabled: false # we don't need SBOM for logs + retryCountOnTaskFailure: 10 # for any files being locked - ${{ if ne(parameters.enablePublishBuildArtifacts, 'false') }}: - - template: /eng/common/core-templates/steps/publish-build-artifacts.yml + - template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml parameters: is1ESPipeline: false args: displayName: Publish Logs - pathToPublish: '$(Build.ArtifactStagingDirectory)/artifacts/log/$(_BuildConfig)' - publishLocation: Container + targetPath: '$(Build.ArtifactStagingDirectory)/artifacts/log/$(_BuildConfig)' artifactName: ${{ coalesce(parameters.enablePublishBuildArtifacts.artifactName, '$(Agent.Os)_$(Agent.JobName)_Attempt$(System.JobAttempt)' ) }} continueOnError: true condition: always() + retryCountOnTaskFailure: 10 # for any files being locked - ${{ if eq(parameters.enableBuildRetry, 'true') }}: - template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml @@ -81,4 +82,4 @@ jobs: artifactName: 'BuildConfiguration' displayName: 'Publish build retry configuration' continueOnError: true - sbomEnabled: false # we don't need SBOM for BuildConfiguration + retryCountOnTaskFailure: 10 # for any files being locked diff --git a/eng/common/templates/steps/component-governance.yml b/eng/common/templates/steps/component-governance.yml deleted file mode 100644 index c12a5f8d21d..00000000000 --- a/eng/common/templates/steps/component-governance.yml +++ /dev/null @@ -1,7 +0,0 @@ -steps: -- template: /eng/common/core-templates/steps/component-governance.yml - parameters: - is1ESPipeline: false - - ${{ each parameter in parameters }}: - ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates/steps/vmr-sync.yml b/eng/common/templates/steps/vmr-sync.yml index 599afb6186b..eb619c50268 100644 --- a/eng/common/templates/steps/vmr-sync.yml +++ b/eng/common/templates/steps/vmr-sync.yml @@ -38,27 +38,6 @@ steps: displayName: Label PR commit workingDirectory: $(Agent.BuildDirectory)/repo -- script: | - vmr_sha=$(grep -oP '(?<=Sha=")[^"]*' $(Agent.BuildDirectory)/repo/eng/Version.Details.xml) - echo "##vso[task.setvariable variable=vmr_sha]$vmr_sha" - displayName: Obtain the vmr sha from Version.Details.xml (Unix) - condition: ne(variables['Agent.OS'], 'Windows_NT') - workingDirectory: $(Agent.BuildDirectory)/repo - -- powershell: | - [xml]$xml = Get-Content -Path $(Agent.BuildDirectory)/repo/eng/Version.Details.xml - $vmr_sha = $xml.SelectSingleNode("//Source").Sha - Write-Output "##vso[task.setvariable variable=vmr_sha]$vmr_sha" - displayName: Obtain the vmr sha from Version.Details.xml (Windows) - condition: eq(variables['Agent.OS'], 'Windows_NT') - workingDirectory: $(Agent.BuildDirectory)/repo - -- script: | - git fetch --all - git checkout $(vmr_sha) - displayName: Checkout VMR at correct sha for repo flow - workingDirectory: ${{ parameters.vmrPath }} - - script: | git config --global user.name "dotnet-maestro[bot]" git config --global user.email "dotnet-maestro[bot]@users.noreply.github.com" diff --git a/eng/common/templates/variables/pool-providers.yml b/eng/common/templates/variables/pool-providers.yml index e0b19c14a07..587770f0add 100644 --- a/eng/common/templates/variables/pool-providers.yml +++ b/eng/common/templates/variables/pool-providers.yml @@ -23,7 +23,7 @@ # # pool: # name: $(DncEngInternalBuildPool) -# demands: ImageOverride -equals windows.vs2019.amd64 +# demands: ImageOverride -equals windows.vs2026.amd64 variables: - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - template: /eng/common/templates-official/variables/pool-providers.yml diff --git a/eng/common/templates/vmr-build-pr.yml b/eng/common/templates/vmr-build-pr.yml index ce3c29a62fa..2f3694fa132 100644 --- a/eng/common/templates/vmr-build-pr.yml +++ b/eng/common/templates/vmr-build-pr.yml @@ -34,6 +34,7 @@ resources: type: github name: dotnet/dotnet endpoint: dotnet + ref: refs/heads/main # Set to whatever VMR branch the PR build should insert into stages: - template: /eng/pipelines/templates/stages/vmr-build.yml@vmr diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index 06b44de7870..977a2d4b103 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -277,7 +277,7 @@ function GetDotNetInstallScript([string] $dotnetRoot) { Retry({ Write-Host "GET $uri" - Invoke-WebRequest $uri -OutFile $installScript + Invoke-WebRequest $uri -UseBasicParsing -OutFile $installScript }) } @@ -394,8 +394,8 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = # If the version of msbuild is going to be xcopied, # use this version. Version matches a package here: - # https://dev.azure.com/dnceng/public/_artifacts/feed/dotnet-eng/NuGet/Microsoft.DotNet.Arcade.MSBuild.Xcopy/versions/17.13.0 - $defaultXCopyMSBuildVersion = '17.13.0' + # https://dev.azure.com/dnceng/public/_artifacts/feed/dotnet-eng/NuGet/Microsoft.DotNet.Arcade.MSBuild.Xcopy/versions/18.0.0 + $defaultXCopyMSBuildVersion = '18.0.0' if (!$vsRequirements) { if (Get-Member -InputObject $GlobalJson.tools -Name 'vs') { @@ -510,7 +510,7 @@ function InitializeXCopyMSBuild([string]$packageVersion, [bool]$install) { Write-Host "Downloading $packageName $packageVersion" $ProgressPreference = 'SilentlyContinue' # Don't display the console progress UI - it's a huge perf hit Retry({ - Invoke-WebRequest "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/flat2/$packageName/$packageVersion/$packageName.$packageVersion.nupkg" -OutFile $packagePath + Invoke-WebRequest "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/flat2/$packageName/$packageVersion/$packageName.$packageVersion.nupkg" -UseBasicParsing -OutFile $packagePath }) if (!(Test-Path $packagePath)) { @@ -556,23 +556,30 @@ function LocateVisualStudio([object]$vsRequirements = $null){ Write-Host "Downloading vswhere $vswhereVersion" $ProgressPreference = 'SilentlyContinue' # Don't display the console progress UI - it's a huge perf hit Retry({ - Invoke-WebRequest "https://netcorenativeassets.blob.core.windows.net/resource-packages/external/windows/vswhere/$vswhereVersion/vswhere.exe" -OutFile $vswhereExe + Invoke-WebRequest "https://netcorenativeassets.blob.core.windows.net/resource-packages/external/windows/vswhere/$vswhereVersion/vswhere.exe" -UseBasicParsing -OutFile $vswhereExe }) } - if (!$vsRequirements) { $vsRequirements = $GlobalJson.tools.vs } + if (!$vsRequirements) { + if (Get-Member -InputObject $GlobalJson.tools -Name 'vs' -ErrorAction SilentlyContinue) { + $vsRequirements = $GlobalJson.tools.vs + } else { + $vsRequirements = $null + } + } + $args = @('-latest', '-format', 'json', '-requires', 'Microsoft.Component.MSBuild', '-products', '*') if (!$excludePrereleaseVS) { $args += '-prerelease' } - if (Get-Member -InputObject $vsRequirements -Name 'version') { + if ($vsRequirements -and (Get-Member -InputObject $vsRequirements -Name 'version' -ErrorAction SilentlyContinue)) { $args += '-version' $args += $vsRequirements.version } - if (Get-Member -InputObject $vsRequirements -Name 'components') { + if ($vsRequirements -and (Get-Member -InputObject $vsRequirements -Name 'components' -ErrorAction SilentlyContinue)) { foreach ($component in $vsRequirements.components) { $args += '-requires' $args += $component @@ -817,6 +824,11 @@ function MSBuild-Core() { $cmdArgs = "$($buildTool.Command) /m /nologo /clp:Summary /v:$verbosity /nr:$nodeReuse /p:ContinuousIntegrationBuild=$ci" + # Add -mt flag for MSBuild multithreaded mode if enabled via environment variable + if ($env:MSBUILD_MT_ENABLED -eq "1") { + $cmdArgs += ' -mt' + } + if ($warnAsError) { $cmdArgs += ' /warnaserror /p:TreatWarningsAsErrors=true' } diff --git a/eng/common/tools.sh b/eng/common/tools.sh index c1841c9dfd0..1b296f646c2 100755 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -526,7 +526,13 @@ function MSBuild-Core { } } - RunBuildTool "$_InitializeBuildToolCommand" /m /nologo /clp:Summary /v:$verbosity /nr:$node_reuse $warnaserror_switch /p:TreatWarningsAsErrors=$warn_as_error /p:ContinuousIntegrationBuild=$ci "$@" + # Add -mt flag for MSBuild multithreaded mode if enabled via environment variable + local mt_switch="" + if [[ "${MSBUILD_MT_ENABLED:-}" == "1" ]]; then + mt_switch="-mt" + fi + + RunBuildTool "$_InitializeBuildToolCommand" /m /nologo /clp:Summary /v:$verbosity /nr:$node_reuse $warnaserror_switch $mt_switch /p:TreatWarningsAsErrors=$warn_as_error /p:ContinuousIntegrationBuild=$ci "$@" } function GetDarc { diff --git a/eng/common/vmr-sync.ps1 b/eng/common/vmr-sync.ps1 index 97302f3205b..b37992d91cf 100644 --- a/eng/common/vmr-sync.ps1 +++ b/eng/common/vmr-sync.ps1 @@ -103,12 +103,20 @@ Set-StrictMode -Version Latest Highlight 'Installing .NET, preparing the tooling..' . .\eng\common\tools.ps1 $dotnetRoot = InitializeDotNetCli -install:$true +$env:DOTNET_ROOT = $dotnetRoot $darc = Get-Darc -$dotnet = "$dotnetRoot\dotnet.exe" Highlight "Starting the synchronization of VMR.." # Synchronize the VMR +$versionDetailsPath = Resolve-Path (Join-Path $PSScriptRoot '..\Version.Details.xml') | Select-Object -ExpandProperty Path +[xml]$versionDetails = Get-Content -Path $versionDetailsPath +$repoName = $versionDetails.SelectSingleNode('//Source').Mapping +if (-not $repoName) { + Fail "Failed to resolve repo mapping from $versionDetailsPath" + exit 1 +} + $darcArgs = ( "vmr", "forwardflow", "--tmp", $tmpDir, @@ -130,9 +138,27 @@ if ($LASTEXITCODE -eq 0) { Highlight "Synchronization succeeded" } else { - Fail "Synchronization of repo to VMR failed!" - Fail "'$vmrDir' is left in its last state (re-run of this script will reset it)." - Fail "Please inspect the logs which contain path to the failing patch file (use -debugOutput to get all the details)." - Fail "Once you make changes to the conflicting VMR patch, commit it locally and re-run this script." - exit 1 + Highlight "Failed to flow code into the local VMR. Falling back to resetting the VMR to match repo contents..." + git -C $vmrDir reset --hard + + $resetArgs = ( + "vmr", "reset", + "${repoName}:HEAD", + "--vmr", $vmrDir, + "--tmp", $tmpDir, + "--additional-remotes", "${repoName}:${repoRoot}" + ) + + & "$darc" $resetArgs + + if ($LASTEXITCODE -eq 0) { + Highlight "Successfully reset the VMR using 'darc vmr reset'" + } + else { + Fail "Synchronization of repo to VMR failed!" + Fail "'$vmrDir' is left in its last state (re-run of this script will reset it)." + Fail "Please inspect the logs which contain path to the failing patch file (use -debugOutput to get all the details)." + Fail "Once you make changes to the conflicting VMR patch, commit it locally and re-run this script." + exit 1 + } } diff --git a/eng/common/vmr-sync.sh b/eng/common/vmr-sync.sh index 44239e331c0..198caec59bd 100644 --- a/eng/common/vmr-sync.sh +++ b/eng/common/vmr-sync.sh @@ -186,6 +186,13 @@ fi # Synchronize the VMR +version_details_path=$(cd "$scriptroot/.."; pwd -P)/Version.Details.xml +repo_name=$(grep -m 1 '] type TTypeCacheKey = | TTypeCacheKey of TypeStructure * TypeStructure * CanCoerce - static member FromStrippedTypes(ty1, ty2, canCoerce) = - TTypeCacheKey(getTypeStructure ty1, getTypeStructure ty2, canCoerce) + static member TryGetFromStrippedTypes(ty1, ty2, canCoerce) = + let t1, t2 = getTypeStructure ty1, getTypeStructure ty2 + if t1.IsPossiblyInfinite || t2.IsPossiblyInfinite then + ValueNone + else + ValueSome (TTypeCacheKey(t1, t2, canCoerce)) let getTypeSubsumptionCache = let factory (g: TcGlobals) = @@ -157,8 +161,10 @@ let rec TypeFeasiblySubsumesType ndeep (g: TcGlobals) (amap: ImportMap) m (ty1: List.exists (TypeFeasiblySubsumesType (ndeep + 1) g amap m ty1 NoCoerce) interfaces if g.langVersion.SupportsFeature LanguageFeature.UseTypeSubsumptionCache then - let key = TTypeCacheKey.FromStrippedTypes(ty1, ty2, canCoerce) - (getTypeSubsumptionCache g).GetOrAdd(key, fun _ -> checkSubsumes ty1 ty2) + match TTypeCacheKey.TryGetFromStrippedTypes(ty1, ty2, canCoerce) with + | ValueSome key -> + (getTypeSubsumptionCache g).GetOrAdd(key, fun _ -> checkSubsumes ty1 ty2) + | _ -> checkSubsumes ty1 ty2 else checkSubsumes ty1 ty2 diff --git a/src/Compiler/Driver/fsc.fs b/src/Compiler/Driver/fsc.fs index 2bea066e427..89f196c810f 100644 --- a/src/Compiler/Driver/fsc.fs +++ b/src/Compiler/Driver/fsc.fs @@ -569,6 +569,7 @@ let main1 exiter.Exit 1 if tcConfig.showTimes then + StackGuardMetrics.CaptureStatsAndWriteToConsole() |> disposables.Register Caches.CacheMetrics.CaptureStatsAndWriteToConsole() |> disposables.Register Activity.Profiling.addConsoleListener () |> disposables.Register diff --git a/src/Compiler/Facilities/DiagnosticsLogger.fs b/src/Compiler/Facilities/DiagnosticsLogger.fs index eb96a5f6e0b..2a087283468 100644 --- a/src/Compiler/Facilities/DiagnosticsLogger.fs +++ b/src/Compiler/Facilities/DiagnosticsLogger.fs @@ -16,6 +16,7 @@ open System.Runtime.InteropServices open Internal.Utilities.Library open Internal.Utilities.Library.Extras open System.Threading.Tasks +open System.Collections.Concurrent /// Represents the style being used to format errors [] @@ -868,6 +869,66 @@ let internal languageFeatureNotSupportedInLibraryError (langFeature: LanguageFea let suggestedVersionStr = LanguageVersion.GetFeatureVersionString langFeature error (Error(FSComp.SR.chkFeatureNotSupportedInLibrary (featureStr, suggestedVersionStr), m)) +module StackGuardMetrics = + + let meter = FSharp.Compiler.Diagnostics.Metrics.Meter + + let jumpCounter = + meter.CreateCounter( + "stackguard-jumps", + description = "Tracks the number of times the stack guard has jumped to a new thread" + ) + + let countJump memberName location = + let tags = + let mutable tags = TagList() + tags.Add(Activity.Tags.callerMemberName, memberName) + tags.Add("source", location) + tags + + jumpCounter.Add(1L, &tags) + + // Used by the self-listener. + let jumpsByFunctionName = ConcurrentDictionary<_, int64 ref>() + + let Listen () = + let listener = new Metrics.MeterListener() + + listener.EnableMeasurementEvents jumpCounter + + listener.SetMeasurementEventCallback(fun _ v tags _ -> + let memberName = nonNull tags[0].Value :?> string + let source = nonNull tags[1].Value :?> string + let counter = jumpsByFunctionName.GetOrAdd((memberName, source), fun _ -> ref 0L) + Interlocked.Add(counter, v) |> ignore) + + listener.Start() + listener :> IDisposable + + let StatsToString () = + let headers = [ "caller"; "source"; "jumps" ] + + let data = + [ + for kvp in jumpsByFunctionName do + let (memberName, source) = kvp.Key + [ memberName; source; string kvp.Value.Value ] + ] + + if List.isEmpty data then + "" + else + $"StackGuard jumps:\n{Metrics.printTable headers data}" + + let CaptureStatsAndWriteToConsole () = + let listener = Listen() + + { new IDisposable with + member _.Dispose() = + listener.Dispose() + StatsToString() |> printfn "%s" + } + /// Guard against depth of expression nesting, by moving to new stack when a maximum depth is reached type StackGuard(maxDepth: int, name: string) = @@ -882,22 +943,15 @@ type StackGuard(maxDepth: int, name: string) = [] line: int ) = - Activity.addEventWithTags - "DiagnosticsLogger.StackGuard.Guard" - (seq { - Activity.Tags.stackGuardName, box name - Activity.Tags.stackGuardCurrentDepth, depth - Activity.Tags.stackGuardMaxDepth, maxDepth - Activity.Tags.callerMemberName, memberName - Activity.Tags.callerFilePath, path - Activity.Tags.callerLineNumber, line - }) - depth <- depth + 1 try if depth % maxDepth = 0 then + let fileName = System.IO.Path.GetFileName(path) + + StackGuardMetrics.countJump memberName $"{fileName}:{line}" + async { do! Async.SwitchToNewThread() Thread.CurrentThread.Name <- $"F# Extra Compilation Thread for {name} (depth {depth})" diff --git a/src/Compiler/Facilities/DiagnosticsLogger.fsi b/src/Compiler/Facilities/DiagnosticsLogger.fsi index 02471dd383b..7bef3d3a15a 100644 --- a/src/Compiler/Facilities/DiagnosticsLogger.fsi +++ b/src/Compiler/Facilities/DiagnosticsLogger.fsi @@ -459,6 +459,11 @@ val tryLanguageFeatureErrorOption: val languageFeatureNotSupportedInLibraryError: langFeature: LanguageFeature -> m: range -> 'T +module internal StackGuardMetrics = + val Listen: unit -> IDisposable + val StatsToString: unit -> string + val CaptureStatsAndWriteToConsole: unit -> IDisposable + type StackGuard = new: maxDepth: int * name: string -> StackGuard diff --git a/src/Compiler/Service/ServiceParseTreeWalk.fs b/src/Compiler/Service/ServiceParseTreeWalk.fs index 133df625a8a..f280df1b237 100644 --- a/src/Compiler/Service/ServiceParseTreeWalk.fs +++ b/src/Compiler/Service/ServiceParseTreeWalk.fs @@ -528,7 +528,17 @@ module SyntaxTraversal = for SynExprRecordField(fieldName = (field, _); expr = e; blockSeparator = sepOpt) in fields do yield dive (path, copyOpt, Some field) field.Range (fun r -> - if rangeContainsPos field.Range pos then + // Treat the caret placed right after the field name (before '=' or a value) as "inside" the field, + // but only if the field does not yet have a value. + // + // Examples (the '$' marks the caret): + // { r with Field1$ } + // { r with + // Field1$ + // } + let isCaretAfterFieldNameWithoutValue = (e.IsNone && posEq pos field.Range.End) + + if rangeContainsPos field.Range pos || isCaretAfterFieldNameWithoutValue then visitor.VisitRecordField r else None) diff --git a/src/Compiler/Utilities/Activity.fs b/src/Compiler/Utilities/Activity.fs index 818d22e6f81..1995e94ad6b 100644 --- a/src/Compiler/Utilities/Activity.fs +++ b/src/Compiler/Utilities/Activity.fs @@ -18,6 +18,54 @@ module ActivityNames = let AllRelevantNames = [| FscSourceName; ProfiledSourceName |] +module Metrics = + let Meter = new Metrics.Meter(ActivityNames.FscSourceName) + + let formatTable headers rows = + let columnWidths = + headers :: rows + |> List.transpose + |> List.map (List.map String.length >> List.max) + + let center width (cell: string) = + String.replicate ((width - cell.Length) / 2) " " + cell |> _.PadRight(width) + + let headers = (columnWidths, headers) ||> List.map2 center + + let printRow (row: string list) = + row + |> List.mapi (fun i (cell: string) -> + if i = 0 then + cell.PadRight(columnWidths[i]) + else + cell.PadLeft(columnWidths[i])) + |> String.concat " | " + |> sprintf "| %s |" + + let headerRow = printRow headers + + let divider = headerRow |> String.map (fun c -> if c = '|' then c else '-') + let hl = String.replicate divider.Length "-" + + use sw = new StringWriter() + + sw.WriteLine hl + sw.WriteLine headerRow + sw.WriteLine divider + + for row in rows do + sw.WriteLine(printRow row) + + sw.WriteLine hl + + string sw + + let printTable headers rows = + try + formatTable headers rows + with exn -> + $"Error formatting table: {exn}" + [] module internal Activity = diff --git a/src/Compiler/Utilities/Activity.fsi b/src/Compiler/Utilities/Activity.fsi index 83d4b2772ec..8ff0a4c3494 100644 --- a/src/Compiler/Utilities/Activity.fsi +++ b/src/Compiler/Utilities/Activity.fsi @@ -2,7 +2,7 @@ namespace FSharp.Compiler.Diagnostics open System -open Internal.Utilities.Library +open System.Diagnostics.Metrics /// For activities following the dotnet distributed tracing concept /// https://learn.microsoft.com/dotnet/core/diagnostics/distributed-tracing-concepts?source=recommendations @@ -16,6 +16,11 @@ module ActivityNames = val AllRelevantNames: string[] +module internal Metrics = + val Meter: Meter + + val printTable: headers: string list -> rows: string list list -> string + /// For activities following the dotnet distributed tracing concept /// https://learn.microsoft.com/dotnet/core/diagnostics/distributed-tracing-concepts?source=recommendations [] diff --git a/src/Compiler/Utilities/Caches.fs b/src/Compiler/Utilities/Caches.fs index aedc7f7b8e4..cae35a93015 100644 --- a/src/Compiler/Utilities/Caches.fs +++ b/src/Compiler/Utilities/Caches.fs @@ -10,7 +10,7 @@ open System.Diagnostics.Metrics open System.IO module CacheMetrics = - let Meter = new Meter("FSharp.Compiler.Cache") + let Meter = FSharp.Compiler.Diagnostics.Metrics.Meter let adds = Meter.CreateCounter("adds", "count") let updates = Meter.CreateCounter("updates", "count") let hits = Meter.CreateCounter("hits", "count") @@ -96,43 +96,24 @@ module CacheMetrics = listener :> IDisposable let StatsToString () = - use sw = new StringWriter() - - let nameColumnWidth = - [ yield! statsByName.Keys; "Cache name" ] |> Seq.map String.length |> Seq.max - - let columns = allCounters |> List.map _.Name - let columnWidths = columns |> List.map String.length |> List.map (max 8) - - let header = - "| " - + String.concat - " | " - [ - "Cache name".PadRight nameColumnWidth - "hit-ratio" - for w, c in (columnWidths, columns) ||> List.zip do - $"{c.PadLeft w}" - ] - + " |" - - sw.WriteLine(String('-', header.Length)) - sw.WriteLine(header) - sw.WriteLine(header |> String.map (fun c -> if c = '|' then '|' else '-')) - - for kv in statsByName do - let name = kv.Key - let stats = kv.Value - let totals = stats.GetTotals() - sw.Write $"| {name.PadLeft nameColumnWidth} | {stats.Ratio, 9:P2} |" - - for w, c in (columnWidths, columns) ||> List.zip do - sw.Write $" {totals[c].ToString().PadLeft(w)} |" - - sw.WriteLine() - - sw.WriteLine(String('-', header.Length)) - string sw + let headers = [ "Cache name"; "hit-ratio" ] @ (allCounters |> List.map _.Name) + + let rows = + [ + for kv in statsByName do + let name = kv.Key + let stats = kv.Value + let totals = stats.GetTotals() + + [ + yield name + yield $"{stats.Ratio:P2}" + for c in allCounters do + yield $"{totals[c.Name]}" + ] + ] + + FSharp.Compiler.Diagnostics.Metrics.printTable headers rows let CaptureStatsAndWriteToConsole () = let listener = ListenToAll() diff --git a/src/Compiler/Utilities/TypeHashing.fs b/src/Compiler/Utilities/TypeHashing.fs index af536a9d2b6..8e3752d5d33 100644 --- a/src/Compiler/Utilities/TypeHashing.fs +++ b/src/Compiler/Utilities/TypeHashing.fs @@ -401,7 +401,9 @@ module StructuralUtilities = | MeasureRational of int * int | NeverEqual of never: NeverEqual - type TypeStructure = TypeStructure of ImmutableArray + type TypeStructure = + | TypeStructure of TypeToken[] + | PossiblyInfinite of never: NeverEqual let inline toNullnessToken (n: Nullness) = match n.TryEvaluate() with @@ -464,6 +466,15 @@ module StructuralUtilities = | TType_measure m -> yield! accumulateMeasure m } + // If the sequence got too long, just drop it, we could be dealing with an infinite type. + let private toTypeStructure tokens = + let tokens = tokens |> Seq.truncate 256 |> Array.ofSeq + + if tokens.Length = 256 then + PossiblyInfinite NeverEqual.Singleton + else + TypeStructure tokens + /// Get the full structure of a type as a sequence of tokens, suitable for equality let getTypeStructure = - Extras.WeakMap.getOrCreate (fun ty -> accumulateTType ty |> ImmutableArray.ofSeq |> TypeStructure) + Extras.WeakMap.getOrCreate (fun ty -> accumulateTType ty |> toTypeStructure) diff --git a/src/FSharp.Build/Microsoft.FSharp.NetSdk.props b/src/FSharp.Build/Microsoft.FSharp.NetSdk.props index e45f864ce22..e0d5f5bd81e 100644 --- a/src/FSharp.Build/Microsoft.FSharp.NetSdk.props +++ b/src/FSharp.Build/Microsoft.FSharp.NetSdk.props @@ -94,7 +94,7 @@ WARNING: DO NOT MODIFY this file unless you are knowledgeable about MSBuild and - 4.6.1 + 4.6.2 diff --git a/tests/FSharp.Compiler.ComponentTests/EmittedIL/TryCatch/StackOverflowRepro.fs b/tests/FSharp.Compiler.ComponentTests/EmittedIL/TryCatch/StackOverflowRepro.fs index 310a6615aba..22f501c4f85 100644 --- a/tests/FSharp.Compiler.ComponentTests/EmittedIL/TryCatch/StackOverflowRepro.fs +++ b/tests/FSharp.Compiler.ComponentTests/EmittedIL/TryCatch/StackOverflowRepro.fs @@ -31,4 +31,4 @@ let main (args:string[]) = with | ex -> printf "%s" (ex.GetType().ToString()) - 0 \ No newline at end of file + 0 diff --git a/tests/FSharp.Compiler.ComponentTests/EmittedIL/TryCatch/TryCatch.fs b/tests/FSharp.Compiler.ComponentTests/EmittedIL/TryCatch/TryCatch.fs index bc3efb3cae2..b13739fa603 100644 --- a/tests/FSharp.Compiler.ComponentTests/EmittedIL/TryCatch/TryCatch.fs +++ b/tests/FSharp.Compiler.ComponentTests/EmittedIL/TryCatch/TryCatch.fs @@ -63,8 +63,9 @@ module TryCatch = let fsharpCoreFile = typeof>.Assembly.Location File.Copy(fsharpCoreFile, Path.Combine(Path.GetDirectoryName(dllFile), Path.GetFileName(fsharpCoreFile)), true) let result = CompilerAssert.ExecuteAndReturnResult (dllFile, isFsx=false, deps = s.Dependencies, newProcess=true) + printfn "%A" result - Assert.True(result.StdErr.Contains "stack overflow" || result.StdErr.Contains "StackOverflow") + Assert.True(result.StdErr.Contains "stack overflow" || result.StdErr.Contains "StackOverflow", result.StdErr) | _ -> failwith (sprintf "%A" compilationResult) diff --git a/tests/FSharp.Compiler.Private.Scripting.UnitTests/DependencyManagerInteractiveTests.fs b/tests/FSharp.Compiler.Private.Scripting.UnitTests/DependencyManagerInteractiveTests.fs index 819fe92ec2c..a0163a3f637 100644 --- a/tests/FSharp.Compiler.Private.Scripting.UnitTests/DependencyManagerInteractiveTests.fs +++ b/tests/FSharp.Compiler.Private.Scripting.UnitTests/DependencyManagerInteractiveTests.fs @@ -39,7 +39,7 @@ type DependencyManagerInteractiveTests() = let getErrors ((_value: Result), (errors: FSharpDiagnostic[])) = errors - [] + [] member _.``SmokeTest - #r nuget``() = let text = """ #r @"nuget:Newtonsoft.Json, Version=9.0.1" @@ -50,7 +50,7 @@ type DependencyManagerInteractiveTests() = Assert.Equal(typeof, value.ReflectionType) Assert.Equal(0, value.ReflectionValue :?> int) - [] + [] member _.``SmokeTest - #r nuget package not found``() = let text = """ #r @"nuget:System.Collections.Immutable.DoesNotExist, version=1.5.0" @@ -259,7 +259,7 @@ TorchSharp.Tensor.LongTensor.From([| 0L .. 100L |]).Device () - [] + [] member _.``Use Dependency Manager to restore packages with native dependencies, build and run script that depends on the results``() = // Skip test on arm64, because there is not an arm64 native library if RuntimeInformation.ProcessArchitecture = Architecture.Arm64 then @@ -358,7 +358,7 @@ printfn ""%A"" result let value = opt.Value Assert.Equal(123, value.ReflectionValue :?> int32) - [] + [] member _.``Use NativeResolver to resolve native dlls.``() = // Skip test on arm64, because there is not an arm64 native library if RuntimeInformation.ProcessArchitecture = Architecture.Arm64 then @@ -442,7 +442,7 @@ printfn ""%A"" result let value = opt.Value Assert.Equal(123, value.ReflectionValue :?> int32) - [] + [] member _.``Use AssemblyResolver to resolve assemblies``() = // Skip test on arm64, because there is not an arm64 native library if RuntimeInformation.ProcessArchitecture = Architecture.Arm64 then diff --git a/tests/FSharp.Compiler.Private.Scripting.UnitTests/FSharpScriptTests.fs b/tests/FSharp.Compiler.Private.Scripting.UnitTests/FSharpScriptTests.fs index e53dc45f877..34faed6114d 100644 --- a/tests/FSharp.Compiler.Private.Scripting.UnitTests/FSharpScriptTests.fs +++ b/tests/FSharp.Compiler.Private.Scripting.UnitTests/FSharpScriptTests.fs @@ -168,7 +168,7 @@ stacktype.Name = "Stack" | Error(ex) -> Assert.IsAssignableFrom(typeof, ex) - [] + [] member _.``Script using System.Configuration succeeds``() = use script = new FSharpScript() let result, errors = script.Eval(""" @@ -246,7 +246,7 @@ System.Configuration.ConfigurationManager.AppSettings.Item "Environment" <- "LOC /// Native dll resolution is not implemented on desktop #if NETSTANDARD - [] + [] member _.``ML - use assembly with native dependencies``() = // Skip test on arm64, because there is not an arm64 native library if RuntimeInformation.ProcessArchitecture = Architecture.Arm64 then @@ -304,7 +304,7 @@ printfn "{@"%A"}" result | Ok(_) -> Assert.False(true, "expected a failure") | Error(ex) -> Assert.IsAssignableFrom(typeof, ex) - [] + [] member _.``Eval script with invalid PackageName should fail immediately``() = use capture = new TestConsole.ExecutionCapture() use script = new FSharpScript(additionalArgs=[| |]) @@ -316,7 +316,7 @@ printfn "{@"%A"}" result Assert.True( errors |> Seq.exists (fun error -> error.Message.Contains("error NU1101:")), "Expect to error containing 'error NU1101:'") Assert.True( errors |> Seq.exists (fun error -> error.Message.Contains("FSharp.Really.Not.A.Package")), "Expect to error containing 'FSharp.Really.Not.A.Package'") - [] + [] member _.``Eval script with invalid PackageName should fail immediately and resolve one time only``() = use capture = new TestConsole.ExecutionCapture() use script = new FSharpScript(additionalArgs=[| |]) @@ -330,7 +330,7 @@ printfn "{@"%A"}" result Assert.Equal(1, (errors |> Seq.filter (fun error -> error.Message.Contains("FSharp.Really.Not.A.Package")) |> Seq.length)) Assert.Equal(1, (errors |> Seq.filter (fun error -> error.Message.Contains("FSharp.Really.Not.Another.Package")) |> Seq.length)) - [] + [] member _.``FsharpPlus - report errors``() = let code = """ #i "nuget:https://api.nuget.org/v3/index.json" @@ -366,7 +366,7 @@ printTable [{|Age = 15; Weight = 88; Name = "Blahboolahboogaloo"|}] let value = opt.Value Assert.Equal(1, downcast value.ReflectionValue) - [] + [] member _.``ML - use assembly with ref dependencies``() = let code = """ #r "nuget:Microsoft.ML.OnnxTransformer,1.4.0" @@ -383,7 +383,7 @@ tInput.Length let value = opt.Value Assert.Equal(4L, downcast value.ReflectionValue) - [] // usessdkrefs is not a valid option for desktop compiler + [] // usessdkrefs is not a valid option for desktop compiler member _.``ML - use assembly with ref dependencies and without refing SMemory``() = let code = """ #r "nuget:Microsoft.ML.OnnxTransformer,1.4.0" @@ -399,7 +399,7 @@ tInput.Length let value = opt.Value Assert.Equal(4L, downcast value.ReflectionValue) - [] + [] member _.``System.Device.Gpio - Ensure we reference the runtime version of the assembly``() = let code = """ #r "nuget:System.Device.Gpio, 1.0.0" @@ -512,7 +512,7 @@ let x = Assert.False(foundInner) - [] + [] member _.``Script with nuget package that yields out of order dependencies works correctly``() = // regression test for: https://github.com/dotnet/fsharp/issues/9217 let code = """ @@ -535,7 +535,7 @@ test pfloat "1.234" let value = opt.Value Assert.True(true = downcast value.ReflectionValue) - [] + [] member _.``Nuget package with method duplicates differing only in generic arity``() = // regression test for: https://github.com/dotnet/fsharp/issues/17796 // Was an internal error @@ -549,7 +549,7 @@ let add (col:IServiceCollection) = let _value,diag = script.Eval(code) Assert.Empty(diag) - [] + [] [] [] [] diff --git a/tests/FSharp.Compiler.Service.Tests/EditorTests.fs b/tests/FSharp.Compiler.Service.Tests/EditorTests.fs index 6df9a1dc4a9..6ce135326fc 100644 --- a/tests/FSharp.Compiler.Service.Tests/EditorTests.fs +++ b/tests/FSharp.Compiler.Service.Tests/EditorTests.fs @@ -1937,6 +1937,15 @@ let hasRecordType (recordTypeName: string) (symbolUses: FSharpSymbolUse list) = | _ -> false ) |> fun exists -> Assert.True(exists, $"Record type {recordTypeName} not found.") + +let private assertItemsWithNames contains names (completionInfo: DeclarationListInfo) = + let itemNames = completionInfo.Items |> Array.map _.NameInCode |> set + + for name in names do + Assert.True(Set.contains name itemNames = contains) + +let assertHasItemWithNames names (completionInfo: DeclarationListInfo) = + assertItemsWithNames true names completionInfo [] let ``Record fields are completed via type name usage`` () = @@ -2113,10 +2122,9 @@ let rUpdate = { r1 with } hasRecordField "Field1" declarations hasRecordField "Field2" declarations -[] +[] let ``Record fields are completed in update record with partial field name`` () = - let parseResults, checkResults = - getParseAndCheckResults """ + let info = Checker.getCompletionInfo """ module Module type R1 = @@ -2124,23 +2132,53 @@ type R1 = let r1 = { Field1 = 1; Field2 = 2 } -let rUpdate = { r1 with Fi } +let rUpdate = { r1 with Fi{caret} } """ - let declarations = - checkResults.GetDeclarationListSymbols( - Some parseResults, - 9, - "let rUpdate = { r1 with Fi }", - { - EndColumn = 26 - LastDotPos = None - PartialIdent = "Fi" - QualifyingIdents = [] - }, - fun _ -> List.empty - ) - |> List.concat + assertHasItemWithNames ["Field1"; "Field2"] info - hasRecordField "Field1" declarations - hasRecordField "Field2" declarations +[] +let ``Multiline record fields are completed in update record with partial field name`` () = + let info = Checker.getCompletionInfo """ +module Module + +type R1 = + { Field1: int; Field2: int } + +let r1 = { Field1 = 1; Field2 = 2 } + +let rUpdate = + { r1 with + Fi{caret} + } +""" + + assertHasItemWithNames ["Field1"; "Field2"] info + +[] +let ``No record field completion after '=' with missing value in first binding (after =;)`` () = + let info = Checker.getCompletionInfo """ +module M + +type X = + val field1: int + val field2: string + +let x = new X() +let _ = { field1 =; {caret} } +""" + assertItemsWithNames false ["field1"; "field2"] info + +[] +let ``No record field completion after '=' with missing value in first binding (after =; f)`` () = + let info = Checker.getCompletionInfo """ +module M + +type X = + val field1: int + val field2: string + +let x = new X() +let _ = { field1 =; f{caret} } +""" + assertItemsWithNames false ["field1"; "field2"] info \ No newline at end of file diff --git a/tests/FSharp.Compiler.Service.Tests/ProjectAnalysisTests.fs b/tests/FSharp.Compiler.Service.Tests/ProjectAnalysisTests.fs index c4261eaeab3..789bd16d6d7 100644 --- a/tests/FSharp.Compiler.Service.Tests/ProjectAnalysisTests.fs +++ b/tests/FSharp.Compiler.Service.Tests/ProjectAnalysisTests.fs @@ -5807,7 +5807,7 @@ let checkContentAsScript content = [] module ScriptClosureCacheUse = - [] + [] let ``References from #r nuget are included in script project options`` () = let checkResults = checkContentAsScript """ #i "nuget:https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json" diff --git a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Control/AsyncType.fs b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Control/AsyncType.fs index ef75dff9e00..3ba6ab2a344 100644 --- a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Control/AsyncType.fs +++ b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Control/AsyncType.fs @@ -49,9 +49,10 @@ type AsyncType() = [] let mutable spinloop = true - let waitASec (t:Task) = - let result = t.Wait(TimeSpan(hours=0,minutes=0,seconds=1)) - Assert.True(result, "Task did not finish after waiting for a second.") + // Use a generous timeout to avoid flaky failures on loaded CI machines where the thread pool may be saturated. + let waitForCompletion (t: Task) = + let result = t.Wait(TimeSpan.FromSeconds(30.0)) + Assert.True(result, "Task did not finish after waiting for 30 seconds.") [] member _.AsyncRunSynchronouslyReusesThreadPoolThread() = @@ -153,34 +154,41 @@ type AsyncType() = member _.CreateTask () = let s = "Hello tasks!" let a = async { return s } - use t : Task = Async.StartAsTask a - waitASec t + let t : Task = Async.StartAsTask a + waitForCompletion t Assert.True (t.IsCompleted) Assert.AreEqual(s, t.Result) [] member _.StartAsTaskCancellation () = let cts = new CancellationTokenSource() + let asyncStarted = new ManualResetEventSlim(false) let doSpinloop () = while spinloop do () let a = async { + asyncStarted.Set() cts.CancelAfter (100) doSpinloop() } - use t : Task = Async.StartAsTask(a, cancellationToken = cts.Token) + let t : Task = Async.StartAsTask(a, cancellationToken = cts.Token) + + // Wait for the async body to actually start executing before checking timing. + Assert.True(asyncStarted.Wait(30_000), "Async body did not start within 30 seconds") + // Should not finish, we don't eagerly mark the task done just because it's been signaled to cancel. try - let result = t.Wait(300) + let result = t.Wait(1000) Assert.False (result) with :? AggregateException -> Assert.Fail "Task should not finish, yet" spinloop <- false try - waitASec t + let result = t.Wait(TimeSpan(hours=0,minutes=0,seconds=5)) + Assert.True(result, "Task did not finish after waiting for 5 seconds.") with :? AggregateException as a -> match a.InnerException with - | :? TaskCanceledException as t -> () + | :? TaskCanceledException -> () | _ -> reraise() Assert.True (t.IsCompleted, "Task is not completed") @@ -204,7 +212,7 @@ type AsyncType() = innerTcs.SetResult () try - waitASec tcs.Task + waitForCompletion tcs.Task with :? AggregateException as a -> match a.InnerException with | :? TaskCanceledException -> () @@ -247,14 +255,17 @@ type AsyncType() = [] member _.CancellationPropagatesToTask () = + let ewh = new ManualResetEvent(false) let a = async { + ewh.Set() |> Assert.True while true do () } - use t = Async.StartAsTask a + let t = Async.StartAsTask a + ewh.WaitOne() |> Assert.True Async.CancelDefaultToken () let mutable exceptionThrown = false try - waitASec t + waitForCompletion t with e -> exceptionThrown <- true Assert.True (exceptionThrown) Assert.True(t.IsCanceled) @@ -287,8 +298,8 @@ type AsyncType() = member _.CreateImmediateAsTask () = let s = "Hello tasks!" let a = async { return s } - use t : Task = Async.StartImmediateAsTask a - waitASec t + let t : Task = Async.StartImmediateAsTask a + waitForCompletion t Assert.True (t.IsCompleted) Assert.AreEqual(s, t.Result) @@ -296,8 +307,8 @@ type AsyncType() = member _.StartImmediateAsTask () = let s = "Hello tasks!" let a = async { return s } - use t = Async.StartImmediateAsTask a - waitASec t + let t = Async.StartImmediateAsTask a + waitForCompletion t Assert.True (t.IsCompleted) Assert.AreEqual(s, t.Result) diff --git a/tests/FSharp.Test.Utilities/FSharp.Test.Utilities.fsproj b/tests/FSharp.Test.Utilities/FSharp.Test.Utilities.fsproj index 484f8acfbb0..d899b551e30 100644 --- a/tests/FSharp.Test.Utilities/FSharp.Test.Utilities.fsproj +++ b/tests/FSharp.Test.Utilities/FSharp.Test.Utilities.fsproj @@ -5,7 +5,6 @@ $(FSharpNetCoreProductTargetFramework) win-x86;win-x64;linux-x64;osx-x64 $(AssetTargetFallback);portable-net45+win8+wp8+wpa81 - true Library true xunit @@ -33,6 +32,7 @@ + @@ -69,8 +69,22 @@ - - + + runtime; native + all + + + runtime; native + all + + + runtime; native + all + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + @@ -80,29 +94,22 @@ + - - - $(NoWarn);NU1510 + $(NoWarn);NU1510;44 + - - - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - - + + + - diff --git a/tests/FSharp.Test.Utilities/TestFramework.fs b/tests/FSharp.Test.Utilities/TestFramework.fs index bbb1b556746..72dbfa46f3d 100644 --- a/tests/FSharp.Test.Utilities/TestFramework.fs +++ b/tests/FSharp.Test.Utilities/TestFramework.fs @@ -6,6 +6,7 @@ open System open System.IO open System.Diagnostics open System.Reflection +open System.Xml.Linq open Scripting open Xunit open FSharp.Compiler.IO @@ -269,8 +270,31 @@ let requireFile dir path = | Some _ -> fullPathLower | None -> failwith (sprintf "Couldn't find \"%s\" on the following paths: \"%s\", \"%s\". Running 'build test' once might solve this issue" path fullPath fullPathLower) +let SCRIPT_ROOT = __SOURCE_DIRECTORY__ +let repoRoot = SCRIPT_ROOT ++ ".." ++ ".." + +let loadVersionsProps () = + let versionsPropsPath = repoRoot ++ "eng" ++ "Versions.props" + if not (File.Exists versionsPropsPath) then + failwithf "Versions.props file not found at %s" versionsPropsPath + XDocument.Load(versionsPropsPath) + +let getMsbuildPropValue (xdoc: XDocument) (propName: string) = + xdoc.Descendants "PropertyGroup" + |> Seq.collect (fun pg -> pg.Elements()) + |> Seq.tryFind (fun el -> el.Name.LocalName = propName) + |> function + | Some el -> el.Value + | None -> failwithf "Property '%s' not found in Versions.props" propName + +// Usage example: +let versionsPropsDoc = loadVersionsProps () +let cscVersion = getMsbuildPropValue versionsPropsDoc "MicrosoftNetCompilersVersion" +let ildasmVersion = getMsbuildPropValue versionsPropsDoc "MicrosoftNETCoreILDAsmVersion" +let ilasmVersion = getMsbuildPropValue versionsPropsDoc "MicrosoftNETCoreILAsmVersion" + let config configurationName envVars = - let SCRIPT_ROOT = __SOURCE_DIRECTORY__ + let fsharpCoreArchitecture = "netstandard2.0" let fsharpBuildArchitecture = "netstandard2.0" let fsharpCompilerInteractiveSettingsArchitecture = "netstandard2.0" @@ -284,10 +308,8 @@ let config configurationName envVars = let fsiArchitecture = dotnetArchitecture //let peverifyArchitecture = dotnetArchitecture #endif - let repoRoot = SCRIPT_ROOT ++ ".." ++ ".." let artifactsPath = repoRoot ++ "artifacts" let artifactsBinPath = artifactsPath ++ "bin" - let coreClrRuntimePackageVersion = "5.0.0-preview.7.20364.11" let csc_flags = "/nologo" let vbc_flags = "/nologo" let fsc_flags = "-r:System.Core.dll --nowarn:20 --define:COMPILED --preferreduilang:en-US" @@ -298,12 +320,12 @@ let config configurationName envVars = let packagesDir = getPackagesDir () let requirePackage = requireFile packagesDir let requireArtifact = requireFile artifactsBinPath - let CSC = requirePackage ("Microsoft.Net.Compilers" ++ "4.3.0-1.22220.8" ++ "tools" ++ "csc.exe") - let VBC = requirePackage ("Microsoft.Net.Compilers" ++ "4.3.0-1.22220.8" ++ "tools" ++ "vbc.exe") + let CSC = requirePackage ("Microsoft.Net.Compilers" ++ cscVersion ++ "tools" ++ "csc.exe") + let VBC = requirePackage ("Microsoft.Net.Compilers" ++ cscVersion ++ "tools" ++ "vbc.exe") let ILDASM_EXE = if operatingSystem = "win" then "ildasm.exe" else "ildasm" - let ILDASM = requirePackage (("runtime." + operatingSystem + "-" + architectureMoniker + ".Microsoft.NETCore.ILDAsm") ++ coreClrRuntimePackageVersion ++ "runtimes" ++ (operatingSystem + "-" + architectureMoniker) ++ "native" ++ ILDASM_EXE) + let ILDASM = requirePackage (("runtime." + operatingSystem + "-" + architectureMoniker + ".Microsoft.NETCore.ILDAsm") ++ ildasmVersion ++ "runtimes" ++ (operatingSystem + "-" + architectureMoniker) ++ "native" ++ ILDASM_EXE) let ILASM_EXE = if operatingSystem = "win" then "ilasm.exe" else "ilasm" - let ILASM = requirePackage (("runtime." + operatingSystem + "-" + architectureMoniker + ".Microsoft.NETCore.ILAsm") ++ coreClrRuntimePackageVersion ++ "runtimes" ++ (operatingSystem + "-" + architectureMoniker) ++ "native" ++ ILASM_EXE) + let ILASM = requirePackage (("runtime." + operatingSystem + "-" + architectureMoniker + ".Microsoft.NETCore.ILAsm") ++ ilasmVersion ++ "runtimes" ++ (operatingSystem + "-" + architectureMoniker) ++ "native" ++ ILASM_EXE) //let PEVERIFY_EXE = if operatingSystem = "win" then "PEVerify.exe" elif operatingSystem = "osx" then "PEVerify.dll" else "PEVerify" let PEVERIFY = "ilverify" //requireArtifact ("PEVerify" ++ configurationName ++ peverifyArchitecture ++ PEVERIFY_EXE) // let FSI_FOR_SCRIPTS = artifactsBinPath ++ "fsi" ++ configurationName ++ fsiArchitecture ++ "fsi.exe" diff --git a/tests/FSharp.Test.Utilities/Utilities.fs b/tests/FSharp.Test.Utilities/Utilities.fs index 92b8c1dffa9..3a8a7ae487e 100644 --- a/tests/FSharp.Test.Utilities/Utilities.fs +++ b/tests/FSharp.Test.Utilities/Utilities.fs @@ -39,6 +39,29 @@ type FactForDESKTOPAttribute() = do base.Skip <- "NETCOREAPP is not supported runtime for this kind of test, it is intended for DESKTOP only" #endif +module SignedBuildSkip = + let isSignedBuild = System.Environment.GetEnvironmentVariable("SIGNTYPE") = "Real" + let skipMessage = "Test skipped on signed builds due to NuGet package restore restrictions" + + let skipIfSigned (attr: #FactAttribute) = + if isSignedBuild then + attr.Skip <- skipMessage + +type FactSkipOnSignedBuildAttribute() as this = + inherit FactAttribute() + do SignedBuildSkip.skipIfSigned this + +type TheorySkipOnSignedBuildAttribute() as this = + inherit TheoryAttribute() + do SignedBuildSkip.skipIfSigned this + +type FactForNETCOREAPPSkipOnSignedBuildAttribute() as this = + inherit FactAttribute() + do SignedBuildSkip.skipIfSigned this + #if !NETCOREAPP + do base.Skip <- "Only NETCOREAPP is supported runtime for this kind of test." + #endif + // This file mimics how Roslyn handles their compilation references for compilation testing module Utilities = diff --git a/tests/FSharp.Test.Utilities/VSInstallDiscovery.fs b/tests/FSharp.Test.Utilities/VSInstallDiscovery.fs new file mode 100644 index 00000000000..e59eec8eed2 --- /dev/null +++ b/tests/FSharp.Test.Utilities/VSInstallDiscovery.fs @@ -0,0 +1,180 @@ +// Copyright (c) Microsoft Corporation. All Rights Reserved. See License.txt in the project root for license information. + +namespace FSharp.Test + +/// Test-only Visual Studio installation discovery infrastructure. +/// Provides a centralized, robust, and graceful discovery mechanism for Visual Studio installations +/// used by integration/editor/unit tests under vsintegration/tests. +module VSInstallDiscovery = + + open System + open System.IO + open System.Diagnostics + + /// Result of VS installation discovery + type VSInstallResult = + | Found of installPath: string * source: string + | NotFound of reason: string + + /// Attempts to find a Visual Studio installation using multiple fallback strategies + let tryFindVSInstallation () : VSInstallResult = + + /// Check if a path exists and looks like a valid VS installation + let validateVSPath path = + if String.IsNullOrEmpty(path) then false + else + try + let fullPath = Path.GetFullPath(path) + Directory.Exists(fullPath) && + Directory.Exists(Path.Combine(fullPath, "IDE")) && + (File.Exists(Path.Combine(fullPath, "IDE", "devenv.exe")) || + File.Exists(Path.Combine(fullPath, "IDE", "VSIXInstaller.exe"))) + with + | _ -> false + + /// Strategy 1: VSAPPIDDIR (derive parent of Common7/IDE) + let tryVSAppIdDir () = + let envVar = Environment.GetEnvironmentVariable("VSAPPIDDIR") + if not (String.IsNullOrEmpty(envVar)) then + try + let parentPath = Path.Combine(envVar, "..") + if validateVSPath parentPath then + Some (Found (Path.GetFullPath(parentPath), "VSAPPIDDIR environment variable")) + else None + with + | _ -> None + else None + + /// Strategy 2: Highest version among VS*COMNTOOLS environment variables + let tryVSCommonTools () = + let vsVersions = [ + ("VS180COMNTOOLS", 18) // Visual Studio 2026 + ("VS170COMNTOOLS", 17) // Visual Studio 2022 + ("VS160COMNTOOLS", 16) // Visual Studio 2019 + ("VS150COMNTOOLS", 15) // Visual Studio 2017 + ("VS140COMNTOOLS", 14) // Visual Studio 2015 + ("VS120COMNTOOLS", 12) // Visual Studio 2013 + ] + + vsVersions + |> List.tryPick (fun (envName, version) -> + let envVar = Environment.GetEnvironmentVariable(envName) + if not (String.IsNullOrEmpty(envVar)) then + try + let parentPath = Path.Combine(envVar, "..") + if validateVSPath parentPath then + Some (Found (Path.GetFullPath(parentPath), $"{envName} environment variable (VS version {version})")) + else None + with + | _ -> None + else None) + + /// Strategy 3: vswhere.exe (Visual Studio Installer) + let tryVSWhere () = + try + let programFiles = Environment.GetFolderPath(Environment.SpecialFolder.ProgramFilesX86) + let vswherePath = Path.Combine(programFiles, "Microsoft Visual Studio", "Installer", "vswhere.exe") + + if File.Exists(vswherePath) then + let startInfo = ProcessStartInfo( + FileName = vswherePath, + Arguments = "-latest -products * -requires Microsoft.Component.MSBuild -property installationPath", + UseShellExecute = false, + RedirectStandardOutput = true, + RedirectStandardError = true, + CreateNoWindow = true + ) + + use proc = Process.Start(startInfo) + proc.WaitForExit(5000) |> ignore // 5 second timeout + + if proc.ExitCode = 0 then + let output = proc.StandardOutput.ReadToEnd().Trim() + if validateVSPath output then + Some (Found (Path.GetFullPath(output), "vswhere.exe discovery")) + else None + else None + else None + with + | _ -> None + + // Try each strategy in order of precedence + match tryVSAppIdDir () with + | Some result -> result + | None -> + match tryVSCommonTools () with + | Some result -> result + | None -> + match tryVSWhere () with + | Some result -> result + | None -> NotFound "No Visual Studio installation found using any discovery method" + + /// Gets the VS installation directory, with graceful fallback behavior. + /// Returns None if no VS installation can be found, allowing callers to handle gracefully. + let tryGetVSInstallDir () : string option = + match tryFindVSInstallation () with + | Found (path, _) -> Some path + | NotFound _ -> None + + /// Gets the VS installation directory with detailed logging. + /// Useful for debugging installation discovery issues in tests. + let getVSInstallDirWithLogging (logAction: string -> unit) : string option = + match tryFindVSInstallation () with + | Found (path, source) -> + logAction $"Visual Studio installation found at: {path} (via {source})" + Some path + | NotFound reason -> + logAction $"Visual Studio installation not found: {reason}" + None + + /// Gets the VS installation directory or fails with a detailed error message. + /// This is the recommended method for test scenarios that require VS to be installed. + let getVSInstallDirOrFail () : string = + match tryFindVSInstallation () with + | Found (path, _) -> path + | NotFound reason -> + failwith $"Visual Studio installation not found: {reason}. Ensure VS is installed or environment variables (VSAPPIDDIR, VS*COMNTOOLS) are set." + +/// Assembly resolver for Visual Studio test infrastructure. +/// Provides centralized assembly resolution for VS integration tests. +module VSAssemblyResolver = + open System + open System.IO + open System.Reflection + open System.Globalization + + /// Adds an assembly resolver that probes Visual Studio installation directories. + /// This should be called early in test initialization to ensure VS assemblies can be loaded. + let addResolver () = + let vsInstallDir = VSInstallDiscovery.getVSInstallDirOrFail () + + let probingPaths = + [| + Path.Combine(vsInstallDir, @"IDE\CommonExtensions\Microsoft\Editor") + Path.Combine(vsInstallDir, @"IDE\PublicAssemblies") + Path.Combine(vsInstallDir, @"IDE\PrivateAssemblies") + Path.Combine(vsInstallDir, @"IDE\CommonExtensions\Microsoft\ManagedLanguages\VBCSharp\LanguageServices") + Path.Combine(vsInstallDir, @"IDE\Extensions\Microsoft\CodeSense\Framework") + Path.Combine(vsInstallDir, @"IDE") + |] + + AppDomain.CurrentDomain.add_AssemblyResolve(fun _ args -> + let found () = + probingPaths + |> Seq.tryPick (fun p -> + try + let name = AssemblyName(args.Name) + let codebase = Path.GetFullPath(Path.Combine(p, name.Name) + ".dll") + if File.Exists(codebase) then + name.CodeBase <- codebase + name.CultureInfo <- Unchecked.defaultof + name.Version <- Unchecked.defaultof + Some name + else + None + with _ -> + None) + + match found () with + | None -> Unchecked.defaultof + | Some name -> Assembly.Load(name)) diff --git a/tests/FSharp.Test.Utilities/XunitHelpers.fs b/tests/FSharp.Test.Utilities/XunitHelpers.fs index 5365a243946..562ae04ce9b 100644 --- a/tests/FSharp.Test.Utilities/XunitHelpers.fs +++ b/tests/FSharp.Test.Utilities/XunitHelpers.fs @@ -77,13 +77,23 @@ module TestCaseCustomizations = let oldTestClass = oldTestMethod.TestClass let oldTestCollection = oldTestMethod.TestClass.TestCollection + // Create a DETERMINISTIC collection ID based on the test case's unique ID + // This ensures the same test case always gets the same collection ID + let collectionId = + use sha = System.Security.Cryptography.SHA256.Create() + let bytes = System.Text.Encoding.UTF8.GetBytes(testCase.UniqueID) + let hash = sha.ComputeHash(bytes) + System.Guid(hash.[0..15]) // Take first 16 bytes for GUID + + let newDisplayName = $"{oldTestCollection.DisplayName}_{collectionId:N}" + // Create a new collection with a unique id for the test case. let newTestCollection = new TestCollection( oldTestCollection.TestAssembly, oldTestCollection.CollectionDefinition, - oldTestCollection.DisplayName, - Guid.NewGuid() + newDisplayName, + collectionId ) let newTestClass = new TestClass(newTestCollection, oldTestClass.Class) @@ -175,7 +185,7 @@ type OpenTelemetryExport(testRunName, enable) = // Configure OpenTelemetry metrics export. Metrics can be viewed in Prometheus or other compatible tools. OpenTelemetry.Sdk.CreateMeterProviderBuilder() - .AddMeter(CacheMetrics.Meter.Name) + .AddMeter(ActivityNames.FscSourceName) .AddMeter("System.Runtime") .ConfigureResource(fun r -> r.AddService(testRunName) |> ignore) .AddOtlpExporter(fun e m -> diff --git a/tests/fsharp/Compiler/Language/OptionalInteropTests.fs b/tests/fsharp/Compiler/Language/OptionalInteropTests.fs index 17f22bbe378..868d46de779 100644 --- a/tests/fsharp/Compiler/Language/OptionalInteropTests.fs +++ b/tests/fsharp/Compiler/Language/OptionalInteropTests.fs @@ -2,6 +2,7 @@ namespace FSharp.Compiler.UnitTests +open System open System.Collections.Immutable open Xunit open FSharp.Test diff --git a/tests/fsharp/FSharpSuite.Tests.fsproj b/tests/fsharp/FSharpSuite.Tests.fsproj index b4540de0b33..d1b45048e75 100644 --- a/tests/fsharp/FSharpSuite.Tests.fsproj +++ b/tests/fsharp/FSharpSuite.Tests.fsproj @@ -2,11 +2,10 @@ - net472;$(FSharpNetCoreProductTargetFramework) + net472;$(FSharpNetCoreProductTargetFramework) $(FSharpNetCoreProductTargetFramework) win-x86;win-x64 $(AssetTargetFallback);portable-net45+win8+wp8+wpa81 - true Library true false @@ -112,27 +111,9 @@ - + false - - - - - - - - - - - - - - - - - - diff --git a/tests/fsharp/tests.fs b/tests/fsharp/tests.fs index c8cbfdab257..9147932627e 100644 --- a/tests/fsharp/tests.fs +++ b/tests/fsharp/tests.fs @@ -627,46 +627,46 @@ module CoreTests = | "" -> () | diffs -> failwithf "'%s' and '%s' differ; %A" diffFileErr expectedFileErr diffs - [] + [] let ``printing`` () = runPrintingTest "--multiemit- --debug+" "output" // F# 5.0 changed some things printing output - [] + [] let ``printing-langversion47`` () = runPrintingTest "--langversion:4.7" "output.47" // Output should not change with optimization off - [] + [] let ``printing-optimizeoff`` () = runPrintingTest "--multiemit- --debug+ --optimize-" "output" // Legacy one-dynamic-assembly emit is the default for .NET Framework, which these tests are using // Turning that off enables multi-assembly-emit. The printing test is useful for testing multi-assembly-emit // as it feeds in many incremental fragments into stdin of the FSI process. - [] + [] let ``printing-multiemit`` () = runPrintingTest "--multiemit+ --debug+" "output.multiemit" // Multi-assembly-emit establishes some slightly different rules regarding internals, and this // needs to be tested with optimizations off. The output should not change. - [] + [] let ``printing-multiemit-optimizeoff`` () = runPrintingTest "--multiemit+ --debug+ --optimize-" "output.multiemit" - [] + [] let ``printing-width-1000`` () = runPrintingTest "--use:preludePrintSize1000.fsx" "output.1000" - [] + [] let ``printing-width-200`` () = runPrintingTest "--use:preludePrintSize200.fsx" "output.200" - [] + [] let ``printing-off`` () = runPrintingTest "--use:preludeShowDeclarationValuesFalse.fsx" "output.off" - [] + [] let ``printing-quiet`` () = runPrintingTest "--quiet" "output.quiet" @@ -1645,7 +1645,7 @@ module RegressionTests = [] let ``12383-FSC_OPTIMIZED`` () = singleTestBuildAndRun "regression/12383" FSC_OPTIMIZED - [] + [] let ``13219-bug-FSI`` () = singleTestBuildAndRun "regression/13219" FSI [] @@ -1653,69 +1653,69 @@ module RegressionTests = let cfg = testConfig "regression/4715" fsc cfg "%s -o:test.exe --optimize+" cfg.fsc_flags ["date.fs"; "env.fs"; "main.fs"] - [] + [] let ``multi-package-type-provider-test-FSI`` () = singleTestBuildAndRun "regression/13710" FSI #if NETCOREAPP - [] + [] let ``Large inputs 12322 fsc.dll 64-bit fsc.dll .NET SDK generating optimized code`` () = let cfg = testConfig "regression/12322" let cfg = { cfg with fsc_flags = cfg.fsc_flags + " --debug:portable --define:PORTABLE_PDB" } singleTestBuildAndRunAux cfg (FSC_BUILDONLY true) - [] + [] let ``Large inputs 12322 fsc.dll 64-bit .NET SDK generating debug code`` () = let cfg = testConfig "regression/12322" let cfg = { cfg with fsc_flags = cfg.fsc_flags + " --debug:portable --define:PORTABLE_PDB" } singleTestBuildAndRunAux cfg (FSC_BUILDONLY false) #else - [] + [] let ``Large inputs 12322 fsc.exe 32-bit .NET Framework generating optimized code, portable PDB`` () = let cfg = testConfig "regression/12322" let cfg = { cfg with fsc_flags = cfg.fsc_flags + " --debug:portable --define:PORTABLE_PDB" } singleTestBuildAndRunAux cfg (FSC_BUILDONLY true) - [] + [] let ``Large inputs 12322 fsc.exe 32-bit .NET Framework generating optimized code, full PDB`` () = let cfg = testConfig "regression/12322" let cfg = { cfg with fsc_flags = cfg.fsc_flags + " --debug:full" } singleTestBuildAndRunAux cfg (FSC_BUILDONLY true) - [] + [] let ``Large inputs 12322 fsc.exe 32-bit .NET Framework generating debug code portable PDB`` () = let cfg = testConfig "regression/12322" let cfg = { cfg with fsc_flags = cfg.fsc_flags + " --debug:portable --define:PORTABLE_PDB" } singleTestBuildAndRunAux cfg (FSC_BUILDONLY false) - [] + [] let ``Large inputs 12322 fsc.exe 32-bit .NET Framework generating debug code, full PDB`` () = let cfg = testConfig "regression/12322" let cfg = { cfg with fsc_flags = cfg.fsc_flags + " --debug:full" } singleTestBuildAndRunAux cfg (FSC_BUILDONLY false) - [] + [] let ``Large inputs 12322 fscAnyCpu.exe 64-bit .NET Framework generating optimized code, portable PDB`` () = let cfg = testConfig "regression/12322" let cfg = { cfg with FSC = cfg.FSCANYCPU } let cfg = { cfg with fsc_flags = cfg.fsc_flags + " --debug:portable --define:PORTABLE_PDB" } singleTestBuildAndRunAux cfg (FSC_BUILDONLY true) - [] + [] let ``Large inputs 12322 fscAnyCpu.exe 64-bit .NET Framework generating optimized code, full PDB`` () = let cfg = testConfig "regression/12322" let cfg = { cfg with FSC = cfg.FSCANYCPU } let cfg = { cfg with fsc_flags = cfg.fsc_flags + " --debug:full " } singleTestBuildAndRunAux cfg (FSC_BUILDONLY true) - [] + [] let ``12322 fscAnyCpu.exe 64-bit .NET Framework generating debug code, portable PDB`` () = let cfg = testConfig "regression/12322" let cfg = { cfg with FSC = cfg.FSCANYCPU } let cfg = { cfg with fsc_flags = cfg.fsc_flags + " --debug:portable --define:PORTABLE_PDB" } singleTestBuildAndRunAux cfg (FSC_BUILDONLY false) - [] + [] let ``12322 fscAnyCpu.exe 64-bit .NET Framework generating debug code, full PDB`` () = let cfg = testConfig "regression/12322" let cfg = { cfg with FSC = cfg.FSCANYCPU } diff --git a/vsintegration/Vsix/VisualFSharpFull/VisualFSharpDebug.csproj b/vsintegration/Vsix/VisualFSharpFull/VisualFSharpDebug.csproj index 96fd6dc836d..5a6671c4d24 100644 --- a/vsintegration/Vsix/VisualFSharpFull/VisualFSharpDebug.csproj +++ b/vsintegration/Vsix/VisualFSharpFull/VisualFSharpDebug.csproj @@ -8,7 +8,9 @@ netcoreapp1.0 true net472 - true + true + false + true diff --git a/vsintegration/Vsix/VisualFSharpFull/VisualFSharpFull.csproj b/vsintegration/Vsix/VisualFSharpFull/VisualFSharpFull.csproj index 10c18b8530e..49f1cc33ab6 100644 --- a/vsintegration/Vsix/VisualFSharpFull/VisualFSharpFull.csproj +++ b/vsintegration/Vsix/VisualFSharpFull/VisualFSharpFull.csproj @@ -9,7 +9,9 @@ true net472 - false + false + false + true diff --git a/vsintegration/src/FSharp.Editor/Common/DebugHelpers.fs b/vsintegration/src/FSharp.Editor/Common/DebugHelpers.fs index 51cdd83f8fa..fa8303b7e60 100644 --- a/vsintegration/src/FSharp.Editor/Common/DebugHelpers.fs +++ b/vsintegration/src/FSharp.Editor/Common/DebugHelpers.fs @@ -120,13 +120,15 @@ module FSharpServiceTelemetry = ActivitySource.AddActivityListener(listener) - let periodicallyDisplayCacheStats = + let periodicallyDisplayMetrics = cancellableTask { use _ = CacheMetrics.ListenToAll() + use _ = FSharp.Compiler.DiagnosticsLogger.StackGuardMetrics.Listen() while true do do! Task.Delay(TimeSpan.FromSeconds 10.0) FSharpOutputPane.logMsg (CacheMetrics.StatsToString()) + FSharpOutputPane.logMsg (FSharp.Compiler.DiagnosticsLogger.StackGuardMetrics.StatsToString()) } #if DEBUG diff --git a/vsintegration/src/FSharp.Editor/FSharp.Editor.fsproj b/vsintegration/src/FSharp.Editor/FSharp.Editor.fsproj index 963332abc3c..26621fac1e6 100644 --- a/vsintegration/src/FSharp.Editor/FSharp.Editor.fsproj +++ b/vsintegration/src/FSharp.Editor/FSharp.Editor.fsproj @@ -170,36 +170,17 @@ + - - - - + - - - - - - - - - + - - - - - - - - - - + diff --git a/vsintegration/src/FSharp.Editor/LanguageService/LanguageService.fs b/vsintegration/src/FSharp.Editor/LanguageService/LanguageService.fs index 92b6979883a..cfb90fc9c37 100644 --- a/vsintegration/src/FSharp.Editor/LanguageService/LanguageService.fs +++ b/vsintegration/src/FSharp.Editor/LanguageService/LanguageService.fs @@ -26,6 +26,7 @@ open Microsoft.CodeAnalysis.Host.Mef open Microsoft.VisualStudio.FSharp.Editor.Telemetry open CancellableTasks open FSharp.Compiler.Text +open Microsoft.VisualStudio.Editor #nowarn "9" // NativePtr.toNativeInt #nowarn "57" // Experimental stuff @@ -417,7 +418,7 @@ type internal FSharpPackage() as this = false, fun _ _ -> task { - DebugHelpers.FSharpServiceTelemetry.periodicallyDisplayCacheStats + DebugHelpers.FSharpServiceTelemetry.periodicallyDisplayMetrics |> CancellableTask.start this.DisposalToken |> ignore } @@ -461,11 +462,13 @@ type internal FSharpLanguageService(package: FSharpPackage) = let outliningManagerService = this.Package.ComponentModel.GetService() - let wpfTextView = this.EditorAdaptersFactoryService.GetWpfTextView(textView) + let wpfTextView = + this.Package.ComponentModel.GetService().GetWpfTextView(textView) + let outliningManager = outliningManagerService.GetOutliningManager(wpfTextView) if not (isNull outliningManager) then - let settings = this.Workspace.Services.GetService() + let settings = this.Workspace.Value.Services.GetService() outliningManager.Enabled <- settings.Advanced.IsOutliningEnabled [] diff --git a/vsintegration/src/FSharp.LanguageService.Base/FSharp.LanguageService.Base.csproj b/vsintegration/src/FSharp.LanguageService.Base/FSharp.LanguageService.Base.csproj index fead6828567..fbf420a0741 100644 --- a/vsintegration/src/FSharp.LanguageService.Base/FSharp.LanguageService.Base.csproj +++ b/vsintegration/src/FSharp.LanguageService.Base/FSharp.LanguageService.Base.csproj @@ -46,16 +46,9 @@ - - - - - - - diff --git a/vsintegration/src/FSharp.LanguageService/FSharp.LanguageService.fsproj b/vsintegration/src/FSharp.LanguageService/FSharp.LanguageService.fsproj index 9968c121f7c..0d948cc586b 100644 --- a/vsintegration/src/FSharp.LanguageService/FSharp.LanguageService.fsproj +++ b/vsintegration/src/FSharp.LanguageService/FSharp.LanguageService.fsproj @@ -56,24 +56,15 @@ - - - - - - - - - diff --git a/vsintegration/src/FSharp.LanguageService/ProjectSitesAndFiles.fs b/vsintegration/src/FSharp.LanguageService/ProjectSitesAndFiles.fs index b2a437dc99f..39fe59fc933 100644 --- a/vsintegration/src/FSharp.LanguageService/ProjectSitesAndFiles.fs +++ b/vsintegration/src/FSharp.LanguageService/ProjectSitesAndFiles.fs @@ -34,7 +34,6 @@ module internal rec Microsoft.VisualStudio.FSharp.LanguageService.SiteProvider open System open System.Collections.Concurrent -open System.ComponentModel.Composition open System.IO open System.Diagnostics open Microsoft.VisualStudio diff --git a/vsintegration/src/FSharp.ProjectSystem.Base/FSharp.ProjectSystem.Base.csproj b/vsintegration/src/FSharp.ProjectSystem.Base/FSharp.ProjectSystem.Base.csproj index c5608f1a6b6..be6eb82d080 100644 --- a/vsintegration/src/FSharp.ProjectSystem.Base/FSharp.ProjectSystem.Base.csproj +++ b/vsintegration/src/FSharp.ProjectSystem.Base/FSharp.ProjectSystem.Base.csproj @@ -40,22 +40,10 @@ - - - - - - - - - - - - diff --git a/vsintegration/src/FSharp.ProjectSystem.FSharp/FSharp.ProjectSystem.FSharp.fsproj b/vsintegration/src/FSharp.ProjectSystem.FSharp/FSharp.ProjectSystem.FSharp.fsproj index 576dc2aa464..da59e918292 100644 --- a/vsintegration/src/FSharp.ProjectSystem.FSharp/FSharp.ProjectSystem.FSharp.fsproj +++ b/vsintegration/src/FSharp.ProjectSystem.FSharp/FSharp.ProjectSystem.FSharp.fsproj @@ -104,23 +104,12 @@ - - - - - - - - - - - - + diff --git a/vsintegration/src/FSharp.ProjectSystem.PropertyPages/FSharp.ProjectSystem.PropertyPages.vbproj b/vsintegration/src/FSharp.ProjectSystem.PropertyPages/FSharp.ProjectSystem.PropertyPages.vbproj index 91418865d89..e964555f55f 100644 --- a/vsintegration/src/FSharp.ProjectSystem.PropertyPages/FSharp.ProjectSystem.PropertyPages.vbproj +++ b/vsintegration/src/FSharp.ProjectSystem.PropertyPages/FSharp.ProjectSystem.PropertyPages.vbproj @@ -46,15 +46,9 @@ - - - - - - diff --git a/vsintegration/src/FSharp.VS.FSI/FSharp.VS.FSI.fsproj b/vsintegration/src/FSharp.VS.FSI/FSharp.VS.FSI.fsproj index 0171cdf9b26..2478700de2d 100644 --- a/vsintegration/src/FSharp.VS.FSI/FSharp.VS.FSI.fsproj +++ b/vsintegration/src/FSharp.VS.FSI/FSharp.VS.FSI.fsproj @@ -19,7 +19,7 @@ - + @@ -57,18 +57,9 @@ - - - - - - - - - diff --git a/vsintegration/tests/FSharp.Editor.Tests/FSharp.Editor.Tests.fsproj b/vsintegration/tests/FSharp.Editor.Tests/FSharp.Editor.Tests.fsproj index c084fc6f06a..1625be60b9a 100644 --- a/vsintegration/tests/FSharp.Editor.Tests/FSharp.Editor.Tests.fsproj +++ b/vsintegration/tests/FSharp.Editor.Tests/FSharp.Editor.Tests.fsproj @@ -95,12 +95,12 @@ - - - + + + + - diff --git a/vsintegration/tests/FSharp.Editor.Tests/Helpers/AssemblyResolver.fs b/vsintegration/tests/FSharp.Editor.Tests/Helpers/AssemblyResolver.fs index f41f10fa138..ef8c01ca908 100644 --- a/vsintegration/tests/FSharp.Editor.Tests/Helpers/AssemblyResolver.fs +++ b/vsintegration/tests/FSharp.Editor.Tests/Helpers/AssemblyResolver.fs @@ -2,57 +2,9 @@ namespace FSharp.Editor.Tests.Helpers -open System -open System.IO -open System.Reflection - module AssemblyResolver = - open System.Globalization - - let vsInstallDir = - // use the environment variable to find the VS installdir - let vsvar = - let var = Environment.GetEnvironmentVariable("VS170COMNTOOLS") - - if String.IsNullOrEmpty var then - Environment.GetEnvironmentVariable("VSAPPIDDIR") - else - var - - if String.IsNullOrEmpty vsvar then - failwith "VS170COMNTOOLS and VSAPPIDDIR environment variables not found." - - Path.Combine(vsvar, "..") - - let probingPaths = - [| - Path.Combine(vsInstallDir, @"IDE\CommonExtensions\Microsoft\Editor") - Path.Combine(vsInstallDir, @"IDE\PublicAssemblies") - Path.Combine(vsInstallDir, @"IDE\PrivateAssemblies") - Path.Combine(vsInstallDir, @"IDE\CommonExtensions\Microsoft\ManagedLanguages\VBCSharp\LanguageServices") - Path.Combine(vsInstallDir, @"IDE\Extensions\Microsoft\CodeSense\Framework") - Path.Combine(vsInstallDir, @"IDE") - |] - - let addResolver () = - AppDomain.CurrentDomain.add_AssemblyResolve (fun h args -> - let found () = - (probingPaths) - |> Seq.tryPick (fun p -> - try - let name = AssemblyName(args.Name) - let codebase = Path.GetFullPath(Path.Combine(p, name.Name) + ".dll") - - if File.Exists(codebase) then - name.CodeBase <- codebase - name.CultureInfo <- Unchecked.defaultof - name.Version <- Unchecked.defaultof - Some(name) - else - None - with _ -> - None) + open FSharp.Test.VSAssemblyResolver - match found () with - | None -> Unchecked.defaultof - | Some name -> Assembly.Load(name)) + /// Adds an assembly resolver that probes Visual Studio installation directories. + /// This is a compatibility shim that delegates to the centralized implementation. + let addResolver = addResolver diff --git a/vsintegration/tests/Salsa/VisualFSharp.Salsa.fsproj b/vsintegration/tests/Salsa/VisualFSharp.Salsa.fsproj index e1e657a651a..d41b116c21e 100644 --- a/vsintegration/tests/Salsa/VisualFSharp.Salsa.fsproj +++ b/vsintegration/tests/Salsa/VisualFSharp.Salsa.fsproj @@ -24,6 +24,9 @@ UnitTests.TestLib.Utils.fs + + VSInstallDiscovery.fs + @@ -51,22 +54,11 @@ - - - - - - - - - - - diff --git a/vsintegration/tests/Salsa/VsMocks.fs b/vsintegration/tests/Salsa/VsMocks.fs index 5189b7faec5..54fe8b906de 100644 --- a/vsintegration/tests/Salsa/VsMocks.fs +++ b/vsintegration/tests/Salsa/VsMocks.fs @@ -1642,6 +1642,7 @@ module internal VsActual = open System.ComponentModel.Composition.Primitives open Microsoft.VisualStudio.Text open Microsoft.VisualStudio.Threading + open FSharp.Test.VSInstallDiscovery type TestExportJoinableTaskContext () = @@ -1650,16 +1651,7 @@ module internal VsActual = [)>] member public _.JoinableTaskContext : JoinableTaskContext = jtc - let vsInstallDir = - // use the environment variable to find the VS installdir - let vsvar = - let var = Environment.GetEnvironmentVariable("VS170COMNTOOLS") - if String.IsNullOrEmpty var then - Environment.GetEnvironmentVariable("VSAPPIDDIR") - else - var - if String.IsNullOrEmpty vsvar then failwith "VS170COMNTOOLS and VSAPPIDDIR environment variables not found." - Path.Combine(vsvar, "..") + let vsInstallDir = getVSInstallDirOrFail () let CreateEditorCatalog() = let thisAssembly = Assembly.GetExecutingAssembly().Location diff --git a/vsintegration/tests/UnitTests/AssemblyResolver.fs b/vsintegration/tests/UnitTests/AssemblyResolver.fs index aab95cc46fc..cf36b723e40 100644 --- a/vsintegration/tests/UnitTests/AssemblyResolver.fs +++ b/vsintegration/tests/UnitTests/AssemblyResolver.fs @@ -1,47 +1,8 @@ namespace Microsoft.VisualStudio.FSharp -open System -open System.IO -open System.Reflection - module AssemblyResolver = - open System.Globalization - - let vsInstallDir = - // use the environment variable to find the VS installdir - let vsvar = - let var = Environment.GetEnvironmentVariable("VS170COMNTOOLS") - if String.IsNullOrEmpty var then - Environment.GetEnvironmentVariable("VSAPPIDDIR") - else - var - if String.IsNullOrEmpty vsvar then failwith "VS170COMNTOOLS and VSAPPIDDIR environment variables not found." - Path.Combine(vsvar, "..") - - let probingPaths = [| - Path.Combine(vsInstallDir, @"IDE\CommonExtensions\Microsoft\Editor") - Path.Combine(vsInstallDir, @"IDE\PublicAssemblies") - Path.Combine(vsInstallDir, @"IDE\PrivateAssemblies") - Path.Combine(vsInstallDir, @"IDE\CommonExtensions\Microsoft\ManagedLanguages\VBCSharp\LanguageServices") - Path.Combine(vsInstallDir, @"IDE\Extensions\Microsoft\CodeSense\Framework") - Path.Combine(vsInstallDir, @"IDE") - |] + open FSharp.Test.VSAssemblyResolver - let addResolver () = - AppDomain.CurrentDomain.add_AssemblyResolve(fun h args -> - let found () = - (probingPaths ) |> Seq.tryPick(fun p -> - try - let name = AssemblyName(args.Name) - let codebase = Path.GetFullPath(Path.Combine(p, name.Name) + ".dll") - if File.Exists(codebase) then - name.CodeBase <- codebase - name.CultureInfo <- Unchecked.defaultof - name.Version <- Unchecked.defaultof - Some (name) - else None - with | _ -> None - ) - match found() with - | None -> Unchecked.defaultof - | Some name -> Assembly.Load(name) ) + /// Adds an assembly resolver that probes Visual Studio installation directories. + /// This is a compatibility shim that delegates to the centralized implementation. + let addResolver = addResolver diff --git a/vsintegration/tests/UnitTests/VisualFSharp.UnitTests.fsproj b/vsintegration/tests/UnitTests/VisualFSharp.UnitTests.fsproj index 96fde123dcc..d5167d28820 100644 --- a/vsintegration/tests/UnitTests/VisualFSharp.UnitTests.fsproj +++ b/vsintegration/tests/UnitTests/VisualFSharp.UnitTests.fsproj @@ -120,34 +120,14 @@ - - - - - - - - - - - - - + + + + - - - - - - - - - - -