From 9487d36e776a5c7315f04a6ddf2887f92ab60d49 Mon Sep 17 00:00:00 2001 From: Edgar Gonzalez Date: Wed, 20 May 2026 05:58:20 -0500 Subject: [PATCH 1/5] Fix #17904 and #19020 (#19738) * Rotate [] binding attrs into the arity-info return position In mkSynBinding, move any attribute written with the explicit 'return:' target from the binding's prefix attributes into SynValData.SynValInfo.retInfo. This makes the syntactic placement (which the parser puts on the binding) match the semantic intent (the attribute targets the method's return value). Fixes: - #19020: [] silently dropped on class members - #17904: false-positive AllowMultiple=false when [] and [] appear on the same member --- .../.FSharp.Compiler.Service/11.0.100.md | 1 + .../Checking/Expressions/CheckExpressions.fs | 36 ++---- src/Compiler/Checking/NameResolution.fs | 15 ++- src/Compiler/SyntaxTree/SyntaxTreeOps.fs | 44 +++++++ .../Language/AttributeCheckingTests.fs | 109 +++++++++++++++++- .../FSharp.Compiler.Service.Tests/Symbols.fs | 37 ++++++ 6 files changed, 212 insertions(+), 30 deletions(-) diff --git a/docs/release-notes/.FSharp.Compiler.Service/11.0.100.md b/docs/release-notes/.FSharp.Compiler.Service/11.0.100.md index 4a6110cf514..7e9cd230a4c 100644 --- a/docs/release-notes/.FSharp.Compiler.Service/11.0.100.md +++ b/docs/release-notes/.FSharp.Compiler.Service/11.0.100.md @@ -1,5 +1,6 @@ ### Fixed +* Fix `[]` prefix attributes being silently dropped on class members, and fix false-positive `AllowMultiple=false` errors when `[]` and `[]` are applied to the same binding. ([Issue #17904](https://github.com/dotnet/fsharp/issues/17904), [Issue #19020](https://github.com/dotnet/fsharp/issues/19020), [PR #19738](https://github.com/dotnet/fsharp/pull/19738)) * Fix attributes on return type of unparenthesized tuple methods being silently dropped from IL. ([Issue #462](https://github.com/dotnet/fsharp/issues/462), [PR #19714](https://github.com/dotnet/fsharp/pull/19714)) * Fix internal error FS0073 "Undefined or unsolved type variable" in IlxGen when nested inline SRTP functions with multiple overloads leave unsolved typars in the non-witness codegen path. ([Issue #19709](https://github.com/dotnet/fsharp/issues/19709), [PR #19710](https://github.com/dotnet/fsharp/pull/19710)) * Fix NRE when calling virtual Object methods on value types through inline SRTP functions. ([Issue #8098](https://github.com/dotnet/fsharp/issues/8098), [PR #19511](https://github.com/dotnet/fsharp/pull/19511)) diff --git a/src/Compiler/Checking/Expressions/CheckExpressions.fs b/src/Compiler/Checking/Expressions/CheckExpressions.fs index 30d49e776c8..6636901e460 100644 --- a/src/Compiler/Checking/Expressions/CheckExpressions.fs +++ b/src/Compiler/Checking/Expressions/CheckExpressions.fs @@ -11169,7 +11169,7 @@ and TcNormalizedBinding declKind (cenv: cenv) env tpenv overallTy safeThisValOpt let envinner = AddDeclaredTypars NoCheckForDuplicateTypars (enclosingDeclaredTypars@declaredTypars) env match bind with - | NormalizedBinding(vis, kind, isInline, isMutable, attrs, xmlDoc, _, valSynData, pat, NormalizedBindingRhs(spatsL, rtyOpt, rhsExpr), _, debugPoint) -> + | NormalizedBinding(vis, kind, isInline, isMutable, attrs, xmlDoc, _, valSynData, pat, NormalizedBindingRhs(spatsL, _, rhsExpr), _, debugPoint) -> let (SynValData(memberFlags = memberFlagsOpt)) = valSynData let mBinding = pat.Range @@ -11223,29 +11223,17 @@ and TcNormalizedBinding declKind (cenv: cenv) env tpenv overallTy safeThisValOpt errorR(Error(FSComp.SR.tcAttributesAreNotPermittedOnLetBindings(), attr.Range)) attrs - // Rotate [] from binding to return value - // Also patch the syntactic representation - let retAttribs, valAttribs, valSynData = - let attribs = TcAttrs attrTgt false attrs - let rotRetSynAttrs, rotRetAttribs, valAttribs = - // Do not rotate if some attrs fail to typecheck... - if attribs.Length <> attrs.Length then [], [], attribs - else attribs - |> List.zip attrs - |> List.partition(function | _, Attrib(_, _, _, _, _, Some ts, _) -> ts &&& AttributeTargets.ReturnValue <> enum 0 | _ -> false) - |> fun (r, v) -> (List.map fst r, List.map snd r, List.map snd v) - let retAttribs = - match rtyOpt with - | Some (SynBindingReturnInfo(attributes = Attributes retAttrs)) -> - rotRetAttribs @ TcAttrs AttributeTargets.ReturnValue true retAttrs - | None -> rotRetAttribs - let valSynData = - match rotRetSynAttrs with - | [] -> valSynData - | {Range=mHead} :: _ -> - let (SynValData(valMf, SynValInfo(args, SynArgInfo(attrs, opt, retId)), valId)) = valSynData - SynValData(valMf, SynValInfo(args, SynArgInfo({Attributes=rotRetSynAttrs; Range=mHead} :: attrs, opt, retId)), valId) - retAttribs, valAttribs, valSynData + // [] attributes are moved out of the binding's prefix and into + // SynValData.SynValInfo.retInfo by SynInfo.RotateReturnAttributes in mkSynBinding, + // alongside any attributes on the return type annotation populated by InferSynReturnData. + // Use that as the single source of truth. + let valAttribs = TcAttrs attrTgt false attrs + + let retAttribs = + let (SynValData(_, SynValInfo(_, SynArgInfo(retAttrs, _, _)), _)) = valSynData + retAttrs + |> List.collect (fun a -> a.Attributes) + |> TcAttrs AttributeTargets.ReturnValue true let valAttribFlags = computeValWellKnownFlags g valAttribs diff --git a/src/Compiler/Checking/NameResolution.fs b/src/Compiler/Checking/NameResolution.fs index 54290c21bd0..c2ff7d12589 100644 --- a/src/Compiler/Checking/NameResolution.fs +++ b/src/Compiler/Checking/NameResolution.fs @@ -94,11 +94,16 @@ let ActivePatternElemsOfValRef g (vref: ValRef) = ActivePatternReturnKind.RefTypeWrapper else let _, apReturnTy = stripFunTy g vref.TauType - let hasStructAttribute() = - vref.Attribs - |> List.exists (function - | Attrib(targetsOpt = Some(System.AttributeTargets.ReturnValue)) as a -> hasFlag (classifyValAttrib g a) WellKnownValAttributes.StructAttribute - | _ -> false) + let hasStructAttribute() = + // After SynInfo.RotateReturnAttributes, [] lives in + // ValReprInfo's result ArgReprInfo rather than vref.Attribs. The flag bits + // in ArgReprInfo.Attribs are computed lazily, so classify the underlying + // attribute list directly. + match vref.ValReprInfo with + | Some(ValReprInfo(_, _, retInfo)) -> + let flags = computeValWellKnownFlags g (retInfo.Attribs.AsList()) + hasFlag flags WellKnownValAttributes.StructAttribute + | None -> false if isValueOptionTy g apReturnTy || hasStructAttribute() then ActivePatternReturnKind.StructTypeWrapper elif isBoolTy g apReturnTy then ActivePatternReturnKind.Boolean else ActivePatternReturnKind.RefTypeWrapper diff --git a/src/Compiler/SyntaxTree/SyntaxTreeOps.fs b/src/Compiler/SyntaxTree/SyntaxTreeOps.fs index 9ce91852896..da070557119 100644 --- a/src/Compiler/SyntaxTree/SyntaxTreeOps.fs +++ b/src/Compiler/SyntaxTree/SyntaxTreeOps.fs @@ -740,6 +740,48 @@ module SynInfo = let argInfos = infosForObjArgs @ infosForArgs SynValData(Some memFlags, SynValInfo(argInfos, retInfo), None) + let private isReturnTargetedAttribute (a: SynAttribute) = + match a.Target with + | Some id -> id.idText = "return" + | None -> false + + /// Rotate any `[]` attributes from a binding's prefix attribute list into the + /// arity-info return position (`SynValInfo.retInfo`). Without this, downstream code that + /// reads `Val.Attribs` would incorrectly see them alongside method-targeted attributes + /// (see issues #17904 and #19020). + let RotateReturnAttributes (attrs: SynAttributes) (valSynData: SynValData) : SynAttributes * SynValData = + // Fast path: avoid all allocation when there's nothing to rotate (the common case). + let hasReturn = + attrs + |> List.exists (fun lst -> lst.Attributes |> List.exists isReturnTargetedAttribute) + + if not hasReturn then + attrs, valSynData + else + let mutable returnTargeted = [] + + let newAttrs = + attrs + |> List.choose (fun lst -> + let ret, kept = lst.Attributes |> List.partition isReturnTargetedAttribute + returnTargeted <- returnTargeted @ ret + + if List.isEmpty kept then + None + else + Some { lst with Attributes = kept }) + + let (SynValData(memFlags, SynValInfo(args, SynArgInfo(retAttrs, opt, retId)), thisIdOpt)) = + valSynData + + let retList: SynAttributeList = + { + Attributes = returnTargeted + Range = (List.head returnTargeted).Range + } + + newAttrs, SynValData(memFlags, SynValInfo(args, SynArgInfo(retList :: retAttrs, opt, retId)), thisIdOpt) + let mkSynBindingRhs staticOptimizations rhsExpr mRhs retInfo = let rhsExpr = List.foldBack (fun (c, e1) e2 -> SynExpr.LibraryOnlyStaticOptimization(c, e1, e2, mRhs)) staticOptimizations rhsExpr @@ -759,6 +801,8 @@ let mkSynBinding let info = SynInfo.InferSynValData(memberFlagsOpt, Some headPat, Option.map snd retInfo, origRhsExpr) + let attrs, info = SynInfo.RotateReturnAttributes attrs info + let rhsExpr, retTyOpt = mkSynBindingRhs staticOptimizations origRhsExpr mRhs retInfo let mBind = unionRangeWithXmlDoc xmlDoc mBind SynBinding(vis, SynBindingKind.Normal, isInline, isMutable, attrs, xmlDoc, info, headPat, retTyOpt, rhsExpr, mBind, spBind, trivia) diff --git a/tests/FSharp.Compiler.ComponentTests/Language/AttributeCheckingTests.fs b/tests/FSharp.Compiler.ComponentTests/Language/AttributeCheckingTests.fs index 8262651bec0..7464dec733b 100644 --- a/tests/FSharp.Compiler.ComponentTests/Language/AttributeCheckingTests.fs +++ b/tests/FSharp.Compiler.ComponentTests/Language/AttributeCheckingTests.fs @@ -43,7 +43,114 @@ type C() = |> ignoreWarnings |> compile |> shouldSucceed - + + // Regression tests for #17904 and #19020. Both have a common root cause: + // [] prefix attributes must route to the return-value metadata slot, + // not stay on the binding alongside method-targeted attributes. + + [] + let ``Issue 19020 - [] is emitted on the return parameter of class members`` () = + FSharp """ +module M +open System +[] +type DescAttribute() = inherit Attribute() + +type T() = + [] + member _.Inst a = a + 1 + [] + static member Stat a = a + 1 + +[] +let main _ = + let inst = typeof.GetMethod("Inst").ReturnParameter.GetCustomAttributes(typeof, false).Length + let stat = typeof.GetMethod("Stat").ReturnParameter.GetCustomAttributes(typeof, false).Length + if inst <> 1 then failwithf "instance member: expected 1, got %d" inst + if stat <> 1 then failwithf "static member: expected 1, got %d" stat + 0 + """ + |> asExe + |> compile + |> shouldSucceed + |> run + |> shouldSucceed + + [] + let ``Issue 17904 - [] on method and [] on return value are not duplicates`` () = + Fsx """ +open System +[] +type AttrAttribute() = inherit Attribute() + +type T() = + [] + [] + member _.Foo() = () + """ + |> ignoreWarnings + |> compile + |> shouldSucceed + + [] + let ``Issue 17904 - two [] on the same return value remain duplicates`` () = + Fsx """ +open System +[] +type AttrAttribute() = inherit Attribute() + +type T() = + [] + [] + member _.Foo() = () + """ + |> ignoreWarnings + |> compile + |> shouldFail + |> withSingleDiagnostic (Error 429, Line 8, Col 7, Line 8, Col 19, "The attribute type 'AttrAttribute' has 'AllowMultiple=false'. Multiple instances of this attribute cannot be attached to a single language element.") + + [] + let ``Issue 17904 - [] and [] on a method remain duplicates`` () = + Fsx """ +open System +[] +type AttrAttribute() = inherit Attribute() + +type T() = + [] + [] + member _.Foo() = () + """ + |> ignoreWarnings + |> compile + |> shouldFail + |> withSingleDiagnostic (Error 429, Line 8, Col 7, Line 8, Col 19, "The attribute type 'AttrAttribute' has 'AllowMultiple=false'. Multiple instances of this attribute cannot be attached to a single language element.") + + [] + let ``CompilationRepresentation(Instance) on a union-type member is not rotated to the return value`` () = + FSharp """ +module M +type MyOption<'T> = + | MySome of 'T + | MyNone + [] + member this.Value = match this with | MySome v -> v | MyNone -> failwith "MyNone" + +[] +let main _ = + let m = typeof>.GetMethod("get_Value") + if isNull m then failwith "get_Value not found — CompilationRepresentation(Instance) was likely rotated away from the member" + if m.IsStatic then failwith "get_Value should be an instance method" + if (MySome 42).Value <> 42 then failwith "Value did not return the carried payload" + 0 + """ + |> asExe + |> compile + |> shouldSucceed + |> run + |> shouldSucceed + + [] let ``Regression: typechecker does not fail when attribute is on type variable (https://github.com/dotnet/fsharp/issues/13525)`` () = let csharpBaseClass = diff --git a/tests/FSharp.Compiler.Service.Tests/Symbols.fs b/tests/FSharp.Compiler.Service.Tests/Symbols.fs index 69bfb70519a..25cc0cd4b6e 100644 --- a/tests/FSharp.Compiler.Service.Tests/Symbols.fs +++ b/tests/FSharp.Compiler.Service.Tests/Symbols.fs @@ -256,6 +256,43 @@ let x = 123 |> Option.map (fun su -> su.Symbol :?> FSharpMemberOrFunctionOrValue) |> Option.iter (fun symbol -> symbol.Attributes.Count |> shouldEqual 1) + [] + let ``FCS - [] surfaces on the method's Attributes only`` () = + let source = """ +open System.ComponentModel +type Calculator() = + [] + member _.Compu{caret}te () = 1 +""" + let mfv = (Checker.getSymbolUse source).Symbol :?> FSharpMemberOrFunctionOrValue + mfv.HasAttribute() |> shouldEqual true + mfv.ReturnParameter.HasAttribute() |> shouldEqual false + + [] + let ``FCS - [] surfaces on ReturnParameter.Attributes only`` () = + let source = """ +open System.ComponentModel +type Calculator() = + [] + member _.Compu{caret}te () = 1 +""" + let mfv = (Checker.getSymbolUse source).Symbol :?> FSharpMemberOrFunctionOrValue + mfv.HasAttribute() |> shouldEqual false + mfv.ReturnParameter.HasAttribute() |> shouldEqual true + + [] + let ``FCS - [] and [] surface independently on the same member`` () = + let source = """ +open System.ComponentModel +type Calculator() = + [] + [] + member _.Compu{caret}te () = 1 +""" + let mfv = (Checker.getSymbolUse source).Symbol :?> FSharpMemberOrFunctionOrValue + mfv.HasAttribute() |> shouldEqual true + mfv.ReturnParameter.HasAttribute() |> shouldEqual true + module Types = [] let ``FSharpType.Print parent namespace qualifiers`` () = From d98bdadd5e0217ec65b4faf9eb5f330220f97a4f Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Wed, 20 May 2026 13:33:21 +0200 Subject: [PATCH 2/5] Update dependencies from https://github.com/dotnet/msbuild build 20260519.11 (#19770) On relative base path root Microsoft.Build , Microsoft.Build.Framework , Microsoft.Build.Tasks.Core , Microsoft.Build.Utilities.Core From Version 18.8.0-preview-26265-07 -> To Version 18.8.0-preview-26269-11 Co-authored-by: dotnet-maestro[bot] --- eng/Version.Details.props | 8 ++++---- eng/Version.Details.xml | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/eng/Version.Details.props b/eng/Version.Details.props index 2f01a912e8d..f170696310e 100644 --- a/eng/Version.Details.props +++ b/eng/Version.Details.props @@ -8,10 +8,10 @@ This file should be imported by eng/Versions.props 10.0.0-beta.26257.4 - 18.8.0-preview-26265-07 - 18.8.0-preview-26265-07 - 18.8.0-preview-26265-07 - 18.8.0-preview-26265-07 + 18.8.0-preview-26269-11 + 18.8.0-preview-26269-11 + 18.8.0-preview-26269-11 + 18.8.0-preview-26269-11 1.0.0-prerelease.26180.1 1.0.0-prerelease.26180.1 diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index a9b09287cee..8d95fbeeaf7 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -2,21 +2,21 @@ - + https://github.com/dotnet/msbuild - d7f2e704dfd7caebd9a34ab81e7257c9b18e3a36 + 911bea0b57d3613eb9c29f49ff9858d03884c397 - + https://github.com/dotnet/msbuild - d7f2e704dfd7caebd9a34ab81e7257c9b18e3a36 + 911bea0b57d3613eb9c29f49ff9858d03884c397 - + https://github.com/dotnet/msbuild - d7f2e704dfd7caebd9a34ab81e7257c9b18e3a36 + 911bea0b57d3613eb9c29f49ff9858d03884c397 - + https://github.com/dotnet/msbuild - d7f2e704dfd7caebd9a34ab81e7257c9b18e3a36 + 911bea0b57d3613eb9c29f49ff9858d03884c397 https://github.com/dotnet/roslyn From 993ffba422b0572d15796c3268699c3d89a519b1 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Wed, 20 May 2026 13:33:26 +0200 Subject: [PATCH 3/5] Update dependencies from https://github.com/dotnet/arcade build 20260519.2 (#19769) On relative base path root Microsoft.DotNet.Arcade.Sdk From Version 10.0.0-beta.26257.4 -> To Version 10.0.0-beta.26269.2 Co-authored-by: dotnet-maestro[bot] --- eng/Version.Details.props | 2 +- eng/Version.Details.xml | 4 ++-- eng/common/darc-init.ps1 | 8 ++++---- eng/common/darc-init.sh | 4 ++-- eng/common/post-build/redact-logs.ps1 | 4 ++-- global.json | 2 +- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/eng/Version.Details.props b/eng/Version.Details.props index f170696310e..7ca679c8807 100644 --- a/eng/Version.Details.props +++ b/eng/Version.Details.props @@ -6,7 +6,7 @@ This file should be imported by eng/Versions.props - 10.0.0-beta.26257.4 + 10.0.0-beta.26269.2 18.8.0-preview-26269-11 18.8.0-preview-26269-11 diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 8d95fbeeaf7..2b53302b0ea 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -76,9 +76,9 @@ - + https://github.com/dotnet/arcade - 3454b2fe822e52373f2604856417b0e6bce71d70 + 96a58d029003171c2302b951688fc87464b197f5 https://dev.azure.com/dnceng/internal/_git/dotnet-optimization diff --git a/eng/common/darc-init.ps1 b/eng/common/darc-init.ps1 index e3374310563..a5be41db690 100644 --- a/eng/common/darc-init.ps1 +++ b/eng/common/darc-init.ps1 @@ -29,11 +29,11 @@ function InstallDarcCli ($darcVersion, $toolpath) { Write-Host "Installing Darc CLI version $darcVersion..." Write-Host 'You may need to restart your command window if this is the first dotnet tool you have installed.' if (-not $toolpath) { - Write-Host "'$dotnet' tool install $darcCliPackageName --version $darcVersion --add-source '$arcadeServicesSource' -v $verbosity -g" - & "$dotnet" tool install $darcCliPackageName --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity -g + Write-Host "'$dotnet' tool install $darcCliPackageName --version $darcVersion --source '$arcadeServicesSource' -v $verbosity -g" + & "$dotnet" tool install $darcCliPackageName --version $darcVersion --source "$arcadeServicesSource" -v $verbosity -g }else { - Write-Host "'$dotnet' tool install $darcCliPackageName --version $darcVersion --add-source '$arcadeServicesSource' -v $verbosity --tool-path '$toolpath'" - & "$dotnet" tool install $darcCliPackageName --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity --tool-path "$toolpath" + Write-Host "'$dotnet' tool install $darcCliPackageName --version $darcVersion --source '$arcadeServicesSource' -v $verbosity --tool-path '$toolpath'" + & "$dotnet" tool install $darcCliPackageName --version $darcVersion --source "$arcadeServicesSource" -v $verbosity --tool-path "$toolpath" } } diff --git a/eng/common/darc-init.sh b/eng/common/darc-init.sh index e889f439b8d..e6ba4ee28c1 100755 --- a/eng/common/darc-init.sh +++ b/eng/common/darc-init.sh @@ -73,9 +73,9 @@ function InstallDarcCli { echo "Installing Darc CLI version $darcVersion..." echo "You may need to restart your command shell if this is the first dotnet tool you have installed." if [ -z "$toolpath" ]; then - echo $($dotnet_root/dotnet tool install $darc_cli_package_name --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity -g) + echo $($dotnet_root/dotnet tool install $darc_cli_package_name --version $darcVersion --source "$arcadeServicesSource" -v $verbosity -g) else - echo $($dotnet_root/dotnet tool install $darc_cli_package_name --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity --tool-path "$toolpath") + echo $($dotnet_root/dotnet tool install $darc_cli_package_name --version $darcVersion --source "$arcadeServicesSource" -v $verbosity --tool-path "$toolpath") fi } diff --git a/eng/common/post-build/redact-logs.ps1 b/eng/common/post-build/redact-logs.ps1 index 472d5bb562c..c1e4104b79a 100644 --- a/eng/common/post-build/redact-logs.ps1 +++ b/eng/common/post-build/redact-logs.ps1 @@ -48,8 +48,8 @@ try { Write-Host "Installing Binlog redactor CLI..." Write-Host "'$dotnet' new tool-manifest" & "$dotnet" new tool-manifest - Write-Host "'$dotnet' tool install $packageName --local --add-source '$PackageFeed' -v $verbosity --version $BinlogToolVersion" - & "$dotnet" tool install $packageName --local --add-source "$PackageFeed" -v $verbosity --version $BinlogToolVersion + Write-Host "'$dotnet' tool install $packageName --local --source '$PackageFeed' -v $verbosity --version $BinlogToolVersion" + & "$dotnet" tool install $packageName --local --source "$PackageFeed" -v $verbosity --version $BinlogToolVersion if (Test-Path $TokensFilePath) { Write-Host "Adding additional sensitive data for redaction from file: " $TokensFilePath diff --git a/global.json b/global.json index cfa33eae0d5..fee65ff38ab 100644 --- a/global.json +++ b/global.json @@ -22,7 +22,7 @@ "xcopy-msbuild": "18.0.0" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "10.0.0-beta.26257.4", + "Microsoft.DotNet.Arcade.Sdk": "10.0.0-beta.26269.2", "Microsoft.DotNet.Helix.Sdk": "8.0.0-beta.23255.2" } } From 725045ecd4c2d3afaff27a261e5ac7265e47f763 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Wed, 20 May 2026 13:33:31 +0200 Subject: [PATCH 4/5] Update dependencies from https://github.com/dotnet/roslyn build 20260518.6 (#19766) On relative base path root Microsoft.CodeAnalysis , Microsoft.CodeAnalysis.Compilers , Microsoft.CodeAnalysis.CSharp , Microsoft.CodeAnalysis.EditorFeatures , Microsoft.CodeAnalysis.EditorFeatures.Text , Microsoft.CodeAnalysis.ExternalAccess.FSharp , Microsoft.CodeAnalysis.Features , Microsoft.VisualStudio.LanguageServices From Version 5.8.0-1.26267.4 -> To Version 5.8.0-1.26268.6 Co-authored-by: dotnet-maestro[bot] --- eng/Version.Details.props | 16 ++++++++-------- eng/Version.Details.xml | 32 ++++++++++++++++---------------- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/eng/Version.Details.props b/eng/Version.Details.props index 7ca679c8807..7f6cdf3d02c 100644 --- a/eng/Version.Details.props +++ b/eng/Version.Details.props @@ -19,14 +19,14 @@ This file should be imported by eng/Versions.props 1.0.0-prerelease.26180.1 1.0.0-prerelease.26180.1 - 5.8.0-1.26267.4 - 5.8.0-1.26267.4 - 5.8.0-1.26267.4 - 5.8.0-1.26267.4 - 5.8.0-1.26267.4 - 5.8.0-1.26267.4 - 5.8.0-1.26267.4 - 5.8.0-1.26267.4 + 5.8.0-1.26268.6 + 5.8.0-1.26268.6 + 5.8.0-1.26268.6 + 5.8.0-1.26268.6 + 5.8.0-1.26268.6 + 5.8.0-1.26268.6 + 5.8.0-1.26268.6 + 5.8.0-1.26268.6 10.0.2 10.0.2 diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 2b53302b0ea..d1e5b5c3bfe 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -18,37 +18,37 @@ https://github.com/dotnet/msbuild 911bea0b57d3613eb9c29f49ff9858d03884c397 - + https://github.com/dotnet/roslyn - deb193d8ca4ec49bf33c52e6cb5de6f490a07543 + 256895f3327e7e2f3f7e94248f227a47c698c1e5 - + https://github.com/dotnet/roslyn - deb193d8ca4ec49bf33c52e6cb5de6f490a07543 + 256895f3327e7e2f3f7e94248f227a47c698c1e5 - + https://github.com/dotnet/roslyn - deb193d8ca4ec49bf33c52e6cb5de6f490a07543 + 256895f3327e7e2f3f7e94248f227a47c698c1e5 - + https://github.com/dotnet/roslyn - deb193d8ca4ec49bf33c52e6cb5de6f490a07543 + 256895f3327e7e2f3f7e94248f227a47c698c1e5 - + https://github.com/dotnet/roslyn - deb193d8ca4ec49bf33c52e6cb5de6f490a07543 + 256895f3327e7e2f3f7e94248f227a47c698c1e5 - + https://github.com/dotnet/roslyn - deb193d8ca4ec49bf33c52e6cb5de6f490a07543 + 256895f3327e7e2f3f7e94248f227a47c698c1e5 - + https://github.com/dotnet/roslyn - deb193d8ca4ec49bf33c52e6cb5de6f490a07543 + 256895f3327e7e2f3f7e94248f227a47c698c1e5 - + https://github.com/dotnet/roslyn - deb193d8ca4ec49bf33c52e6cb5de6f490a07543 + 256895f3327e7e2f3f7e94248f227a47c698c1e5 From 92aaed259588fea3a465e0e5a056d547889639c8 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 20 May 2026 11:34:27 +0000 Subject: [PATCH 5/5] Reset files to feature/net11-scouting Reset patterns: - global.json - eng/Version.Details.xml - eng/Version.Details.props - eng/Versions.props - eng/common/** - eng/TargetFrameworks.props --- eng/TargetFrameworks.props | 2 +- eng/Version.Details.props | 26 +- eng/Version.Details.xml | 54 +- eng/Versions.props | 9 +- eng/common/AGENTS.md | 5 + eng/common/SetupNugetSources.ps1 | 17 +- eng/common/SetupNugetSources.sh | 17 +- eng/common/build.ps1 | 2 + eng/common/build.sh | 8 +- eng/common/core-templates/job/job.yml | 8 + eng/common/core-templates/job/onelocbuild.yml | 3 + .../job/publish-build-assets.yml | 12 +- eng/common/core-templates/job/renovate.yml | 196 +++++++ .../job/source-index-stage1.yml | 6 +- .../post-build/common-variables.yml | 2 - .../core-templates/post-build/post-build.yml | 528 ++++++++---------- eng/common/core-templates/stages/renovate.yml | 111 ++++ .../steps/install-microbuild-impl.yml | 34 ++ .../steps/install-microbuild.yml | 64 ++- .../core-templates/steps/publish-logs.yml | 3 +- .../core-templates/steps/source-build.yml | 2 +- .../steps/source-index-stage1-publish.yml | 8 +- eng/common/cross/build-rootfs.sh | 188 ++++--- eng/common/cross/install-debs.py | 136 +++-- eng/common/cross/toolchain.cmake | 27 +- eng/common/darc-init.sh | 2 +- eng/common/dotnet-install.ps1 | 6 +- eng/common/dotnet-install.sh | 8 +- eng/common/dotnet.sh | 2 +- eng/common/internal-feed-operations.sh | 2 +- eng/common/native/init-distro-rid.sh | 2 + eng/common/native/install-dependencies.sh | 11 +- eng/common/post-build/redact-logs.ps1 | 3 +- eng/common/renovate.env | 42 ++ eng/common/sdk-task.ps1 | 17 +- eng/common/sdk-task.sh | 2 +- eng/common/template-guidance.md | 3 - eng/common/tools.ps1 | 222 ++++---- eng/common/tools.sh | 129 ++++- global.json | 6 +- 40 files changed, 1244 insertions(+), 681 deletions(-) create mode 100644 eng/common/AGENTS.md create mode 100644 eng/common/core-templates/job/renovate.yml create mode 100644 eng/common/core-templates/stages/renovate.yml create mode 100644 eng/common/core-templates/steps/install-microbuild-impl.yml create mode 100644 eng/common/renovate.env diff --git a/eng/TargetFrameworks.props b/eng/TargetFrameworks.props index d384e5fbcaa..e3938d0f73a 100644 --- a/eng/TargetFrameworks.props +++ b/eng/TargetFrameworks.props @@ -11,7 +11,7 @@ - net10.0 + net11.0 $([System.Text.RegularExpressions.Regex]::Replace('$(FSharpNetCoreProductTargetFramework)', '^net(\d+)\.0$', '$1')) diff --git a/eng/Version.Details.props b/eng/Version.Details.props index 7f6cdf3d02c..f704480c2d7 100644 --- a/eng/Version.Details.props +++ b/eng/Version.Details.props @@ -6,12 +6,12 @@ This file should be imported by eng/Versions.props - 10.0.0-beta.26269.2 + 11.0.0-beta.26265.4 - 18.8.0-preview-26269-11 - 18.8.0-preview-26269-11 - 18.8.0-preview-26269-11 - 18.8.0-preview-26269-11 + 18.8.0-preview-26265-07 + 18.8.0-preview-26265-07 + 18.8.0-preview-26265-07 + 18.8.0-preview-26265-07 1.0.0-prerelease.26180.1 1.0.0-prerelease.26180.1 @@ -19,14 +19,14 @@ This file should be imported by eng/Versions.props 1.0.0-prerelease.26180.1 1.0.0-prerelease.26180.1 - 5.8.0-1.26268.6 - 5.8.0-1.26268.6 - 5.8.0-1.26268.6 - 5.8.0-1.26268.6 - 5.8.0-1.26268.6 - 5.8.0-1.26268.6 - 5.8.0-1.26268.6 - 5.8.0-1.26268.6 + 5.8.0-1.26267.4 + 5.8.0-1.26267.4 + 5.8.0-1.26267.4 + 5.8.0-1.26267.4 + 5.8.0-1.26267.4 + 5.8.0-1.26267.4 + 5.8.0-1.26267.4 + 5.8.0-1.26267.4 10.0.2 10.0.2 diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index d1e5b5c3bfe..d2b2954d109 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,54 +1,54 @@ - + - + https://github.com/dotnet/msbuild - 911bea0b57d3613eb9c29f49ff9858d03884c397 + d7f2e704dfd7caebd9a34ab81e7257c9b18e3a36 - + https://github.com/dotnet/msbuild - 911bea0b57d3613eb9c29f49ff9858d03884c397 + d7f2e704dfd7caebd9a34ab81e7257c9b18e3a36 - + https://github.com/dotnet/msbuild - 911bea0b57d3613eb9c29f49ff9858d03884c397 + d7f2e704dfd7caebd9a34ab81e7257c9b18e3a36 - + https://github.com/dotnet/msbuild - 911bea0b57d3613eb9c29f49ff9858d03884c397 + d7f2e704dfd7caebd9a34ab81e7257c9b18e3a36 - + https://github.com/dotnet/roslyn - 256895f3327e7e2f3f7e94248f227a47c698c1e5 + deb193d8ca4ec49bf33c52e6cb5de6f490a07543 - + https://github.com/dotnet/roslyn - 256895f3327e7e2f3f7e94248f227a47c698c1e5 + deb193d8ca4ec49bf33c52e6cb5de6f490a07543 - + https://github.com/dotnet/roslyn - 256895f3327e7e2f3f7e94248f227a47c698c1e5 + deb193d8ca4ec49bf33c52e6cb5de6f490a07543 - + https://github.com/dotnet/roslyn - 256895f3327e7e2f3f7e94248f227a47c698c1e5 + deb193d8ca4ec49bf33c52e6cb5de6f490a07543 - + https://github.com/dotnet/roslyn - 256895f3327e7e2f3f7e94248f227a47c698c1e5 + deb193d8ca4ec49bf33c52e6cb5de6f490a07543 - + https://github.com/dotnet/roslyn - 256895f3327e7e2f3f7e94248f227a47c698c1e5 + deb193d8ca4ec49bf33c52e6cb5de6f490a07543 - + https://github.com/dotnet/roslyn - 256895f3327e7e2f3f7e94248f227a47c698c1e5 + deb193d8ca4ec49bf33c52e6cb5de6f490a07543 - + https://github.com/dotnet/roslyn - 256895f3327e7e2f3f7e94248f227a47c698c1e5 + deb193d8ca4ec49bf33c52e6cb5de6f490a07543 @@ -76,9 +76,9 @@ - + https://github.com/dotnet/arcade - 96a58d029003171c2302b951688fc87464b197f5 + 60e594d8a0e742ae93f96384508dcd690b2fbf01 https://dev.azure.com/dnceng/internal/_git/dotnet-optimization diff --git a/eng/Versions.props b/eng/Versions.props index 64e545f6118..14548944aa1 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -89,7 +89,12 @@ 4.6.1 4.6.3 6.1.2 - + + 10.0.2 + $(SystemPackagesVersion) + $(SystemPackagesVersion) + $(SystemPackagesVersion) + $(SystemPackagesVersion) @@ -156,7 +161,7 @@ 5.0.0-preview.7.20364.11 5.0.0-preview.7.20364.11 - 17.14.1 + 18.0.1 2.0.2 13.0.4 3.2.2 diff --git a/eng/common/AGENTS.md b/eng/common/AGENTS.md new file mode 100644 index 00000000000..a5ed8f72926 --- /dev/null +++ b/eng/common/AGENTS.md @@ -0,0 +1,5 @@ +# `eng/common` + +Files under `eng/common` come from [Arcade](https://github.com/dotnet/arcade). +Edits in `eng/common` will be overwritten by automation unless the changes are made directly in the Arcade repository. +For more information, see the [Arcade documentation](https://github.com/dotnet/arcade/tree/main/Documentation). diff --git a/eng/common/SetupNugetSources.ps1 b/eng/common/SetupNugetSources.ps1 index 65ed3a8adef..fc8d618014e 100644 --- a/eng/common/SetupNugetSources.ps1 +++ b/eng/common/SetupNugetSources.ps1 @@ -1,7 +1,6 @@ # This script adds internal feeds required to build commits that depend on internal package sources. For instance, -# dotnet6-internal would be added automatically if dotnet6 was found in the nuget.config file. Similarly, -# dotnet-eng-internal and dotnet-tools-internal are added if dotnet-eng and dotnet-tools are present. -# In addition, this script also enables disabled internal Maestro (darc-int*) feeds. +# dotnet6-internal would be added automatically if dotnet6 was found in the nuget.config file. In addition also enables +# disabled internal Maestro (darc-int*) feeds. # # Optionally, this script also adds a credential entry for each of the internal feeds if supplied. # @@ -174,16 +173,4 @@ foreach ($dotnetVersion in $dotnetVersions) { } } -# Check for dotnet-eng and add dotnet-eng-internal if present -$dotnetEngSource = $sources.SelectSingleNode("add[@key='dotnet-eng']") -if ($dotnetEngSource -ne $null) { - AddOrEnablePackageSource -Sources $sources -DisabledPackageSources $disabledSources -SourceName "dotnet-eng-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet-eng-internal/nuget/$feedSuffix" -Creds $creds -Username $userName -pwd $Password -} - -# Check for dotnet-tools and add dotnet-tools-internal if present -$dotnetToolsSource = $sources.SelectSingleNode("add[@key='dotnet-tools']") -if ($dotnetToolsSource -ne $null) { - AddOrEnablePackageSource -Sources $sources -DisabledPackageSources $disabledSources -SourceName "dotnet-tools-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet-tools-internal/nuget/$feedSuffix" -Creds $creds -Username $userName -pwd $Password -} - $doc.Save($filename) diff --git a/eng/common/SetupNugetSources.sh b/eng/common/SetupNugetSources.sh index b2163abbe71..b97cc536379 100755 --- a/eng/common/SetupNugetSources.sh +++ b/eng/common/SetupNugetSources.sh @@ -1,9 +1,8 @@ #!/usr/bin/env bash # This script adds internal feeds required to build commits that depend on internal package sources. For instance, -# dotnet6-internal would be added automatically if dotnet6 was found in the nuget.config file. Similarly, -# dotnet-eng-internal and dotnet-tools-internal are added if dotnet-eng and dotnet-tools are present. -# In addition, this script also enables disabled internal Maestro (darc-int*) feeds. +# dotnet6-internal would be added automatically if dotnet6 was found in the nuget.config file. In addition also enables +# disabled internal Maestro (darc-int*) feeds. # # Optionally, this script also adds a credential entry for each of the internal feeds if supplied. # @@ -174,18 +173,6 @@ for DotNetVersion in ${DotNetVersions[@]} ; do fi done -# Check for dotnet-eng and add dotnet-eng-internal if present -grep -i " /dev/null -if [ "$?" == "0" ]; then - AddOrEnablePackageSource "dotnet-eng-internal" "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet-eng-internal/nuget/$FeedSuffix" -fi - -# Check for dotnet-tools and add dotnet-tools-internal if present -grep -i " /dev/null -if [ "$?" == "0" ]; then - AddOrEnablePackageSource "dotnet-tools-internal" "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet-tools-internal/nuget/$FeedSuffix" -fi - # I want things split line by line PrevIFS=$IFS IFS=$'\n' diff --git a/eng/common/build.ps1 b/eng/common/build.ps1 index 8cfee107e7a..18397a60eb8 100644 --- a/eng/common/build.ps1 +++ b/eng/common/build.ps1 @@ -6,6 +6,7 @@ Param( [string][Alias('v')]$verbosity = "minimal", [string] $msbuildEngine = $null, [bool] $warnAsError = $true, + [string] $warnNotAsError = '', [bool] $nodeReuse = $true, [switch] $buildCheck = $false, [switch][Alias('r')]$restore, @@ -70,6 +71,7 @@ function Print-Usage() { Write-Host " -excludeCIBinarylog Don't output binary log (short: -nobl)" Write-Host " -prepareMachine Prepare machine for CI run, clean up processes after build" Write-Host " -warnAsError Sets warnaserror msbuild parameter ('true' or 'false')" + Write-Host " -warnNotAsError Sets a semi-colon delimited list of warning codes that should not be treated as errors" Write-Host " -msbuildEngine Msbuild engine to use to run build ('dotnet', 'vs', or unspecified)." Write-Host " -excludePrereleaseVS Set to exclude build engines in prerelease versions of Visual Studio" Write-Host " -nativeToolsOnMachine Sets the native tools on machine environment variable (indicating that the script should use native tools on machine)" diff --git a/eng/common/build.sh b/eng/common/build.sh index 9767bb411a4..5883e53bcfb 100755 --- a/eng/common/build.sh +++ b/eng/common/build.sh @@ -42,6 +42,7 @@ usage() echo " --prepareMachine Prepare machine for CI run, clean up processes after build" echo " --nodeReuse Sets nodereuse msbuild parameter ('true' or 'false')" echo " --warnAsError Sets warnaserror msbuild parameter ('true' or 'false')" + echo " --warnNotAsError Sets a semi-colon delimited list of warning codes that should not be treated as errors" echo " --buildCheck Sets /check msbuild parameter" echo " --fromVMR Set when building from within the VMR" echo "" @@ -78,6 +79,7 @@ ci=false clean=false warn_as_error=true +warn_not_as_error='' node_reuse=true build_check=false binary_log=false @@ -92,7 +94,7 @@ runtime_source_feed='' runtime_source_feed_key='' properties=() -while [[ $# > 0 ]]; do +while [[ $# -gt 0 ]]; do opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")" case "$opt" in -help|-h) @@ -176,6 +178,10 @@ while [[ $# > 0 ]]; do warn_as_error=$2 shift ;; + -warnnotaserror) + warn_not_as_error=$2 + shift + ;; -nodereuse) node_reuse=$2 shift diff --git a/eng/common/core-templates/job/job.yml b/eng/common/core-templates/job/job.yml index eaed6d87e65..66c7988f222 100644 --- a/eng/common/core-templates/job/job.yml +++ b/eng/common/core-templates/job/job.yml @@ -19,6 +19,8 @@ parameters: # publishing defaults artifacts: '' enableMicrobuild: false + enablePreviewMicrobuild: false + microbuildPluginVersion: 'latest' enableMicrobuildForMacAndLinux: false microbuildUseESRP: true enablePublishBuildArtifacts: false @@ -71,6 +73,8 @@ jobs: templateContext: ${{ parameters.templateContext }} variables: + - name: AllowPtrToDetectTestRunRetryFiles + value: true - ${{ if ne(parameters.enableTelemetry, 'false') }}: - name: DOTNET_CLI_TELEMETRY_PROFILE value: '$(Build.Repository.Uri)' @@ -128,6 +132,8 @@ jobs: - template: /eng/common/core-templates/steps/install-microbuild.yml parameters: enableMicrobuild: ${{ parameters.enableMicrobuild }} + enablePreviewMicrobuild: ${{ parameters.enablePreviewMicrobuild }} + microbuildPluginVersion: ${{ parameters.microbuildPluginVersion }} enableMicrobuildForMacAndLinux: ${{ parameters.enableMicrobuildForMacAndLinux }} microbuildUseESRP: ${{ parameters.microbuildUseESRP }} continueOnError: ${{ parameters.continueOnError }} @@ -150,6 +156,8 @@ jobs: - template: /eng/common/core-templates/steps/cleanup-microbuild.yml parameters: enableMicrobuild: ${{ parameters.enableMicrobuild }} + enablePreviewMicrobuild: ${{ parameters.enablePreviewMicrobuild }} + microbuildPluginVersion: ${{ parameters.microbuildPluginVersion }} enableMicrobuildForMacAndLinux: ${{ parameters.enableMicrobuildForMacAndLinux }} continueOnError: ${{ parameters.continueOnError }} diff --git a/eng/common/core-templates/job/onelocbuild.yml b/eng/common/core-templates/job/onelocbuild.yml index eefed3b667a..86ea9f63504 100644 --- a/eng/common/core-templates/job/onelocbuild.yml +++ b/eng/common/core-templates/job/onelocbuild.yml @@ -22,6 +22,7 @@ parameters: GitHubOrg: dotnet MirrorRepo: '' MirrorBranch: main + xLocCustomPowerShellScript: '' condition: '' JobNameSuffix: '' is1ESPipeline: '' @@ -97,6 +98,8 @@ jobs: gitHubOrganization: ${{ parameters.GitHubOrg }} mirrorRepo: ${{ parameters.MirrorRepo }} mirrorBranch: ${{ parameters.MirrorBranch }} + ${{ if ne(parameters.xLocCustomPowerShellScript, '') }}: + xLocCustomPowerShellScript: ${{ parameters.xLocCustomPowerShellScript }} condition: ${{ parameters.condition }} # Copy the locProject.json to the root of the Loc directory, then publish a pipeline artifact diff --git a/eng/common/core-templates/job/publish-build-assets.yml b/eng/common/core-templates/job/publish-build-assets.yml index 06f2eed0323..700f7711465 100644 --- a/eng/common/core-templates/job/publish-build-assets.yml +++ b/eng/common/core-templates/job/publish-build-assets.yml @@ -91,8 +91,8 @@ jobs: fetchDepth: 3 clean: true - - ${{ if eq(parameters.isAssetlessBuild, 'false') }}: - - ${{ if eq(parameters.publishingVersion, 3) }}: + - ${{ if eq(parameters.isAssetlessBuild, 'false') }}: + - ${{ if eq(parameters.publishingVersion, 3) }}: - task: DownloadPipelineArtifact@2 displayName: Download Asset Manifests inputs: @@ -117,7 +117,7 @@ jobs: flattenFolders: true condition: ${{ parameters.condition }} continueOnError: ${{ parameters.continueOnError }} - + - task: NuGetAuthenticate@1 # Populate internal runtime variables. @@ -125,7 +125,7 @@ jobs: ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: parameters: legacyCredential: $(dn-bot-dnceng-artifact-feeds-rw) - + - template: /eng/common/templates/steps/enable-internal-runtimes.yml - task: AzureCLI@2 @@ -145,7 +145,7 @@ jobs: condition: ${{ parameters.condition }} continueOnError: ${{ parameters.continueOnError }} - + - task: powershell@2 displayName: Create ReleaseConfigs Artifact inputs: @@ -191,7 +191,7 @@ jobs: BARBuildId: ${{ parameters.BARBuildId }} PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} is1ESPipeline: ${{ parameters.is1ESPipeline }} - + # Darc is targeting 8.0, so make sure it's installed - task: UseDotNet@2 inputs: diff --git a/eng/common/core-templates/job/renovate.yml b/eng/common/core-templates/job/renovate.yml new file mode 100644 index 00000000000..ff86c80b468 --- /dev/null +++ b/eng/common/core-templates/job/renovate.yml @@ -0,0 +1,196 @@ +# -------------------------------------------------------------------------------------- +# Renovate Bot Job Template +# -------------------------------------------------------------------------------------- +# This Azure DevOps pipeline job template runs Renovate (https://docs.renovatebot.com/) +# to automatically update dependencies in a GitHub repository. +# +# Renovate scans the repository for dependency files and creates pull requests to update +# outdated dependencies based on the configuration specified in the renovateConfigPath +# parameter. +# +# Usage: +# For each product repo wanting to make use of Renovate, this template is called from +# an internal Azure DevOps pipeline, typically with a schedule trigger, to check for +# and propose dependency updates. +# +# For more info, see https://github.com/dotnet/arcade/blob/main/Documentation/Renovate.md +# -------------------------------------------------------------------------------------- + +parameters: + +# Path to the Renovate configuration file within the repository. +- name: renovateConfigPath + type: string + default: 'eng/renovate.json' + +# GitHub repository to run Renovate against, in the format 'owner/repo'. +# This could technically be any repo but convention is to target the same +# repo that contains the calling pipeline. The Renovate config file would +# be co-located with the pipeline's repo and, in most cases, the config +# file is specific to the repo being targeted. +- name: gitHubRepo + type: string + +# List of base branches to target for Renovate PRs. +# NOTE: The Renovate configuration file is always read from the branch where the +# pipeline is run, NOT from the target branches specified here. If you need different +# configurations for different branches, run the pipeline from each branch separately. +- name: baseBranches + type: object + default: + - main + +# When true, Renovate will run in dry run mode, which previews changes without creating PRs. +# See the 'Run Renovate' step log output for details of what would have been changed. +- name: dryRun + type: boolean + default: false + +# By default, Renovate will not recreate a PR for a given dependency/version pair that was +# previously closed. This allows opting in to always recreating PRs even if they were +# previously closed. +- name: forceRecreatePR + type: boolean + default: false + +# Name of the arcade repository resource in the pipeline. +# This allows repos which haven't been onboarded to Arcade to still use this +# template by checking out the repo as a resource with a custom name and pointing +# this parameter to it. +- name: arcadeRepoResource + type: string + default: self + +# Directory name for the self repo under $(Build.SourcesDirectory) in multi-checkout. +# In multi-checkout (when arcadeRepoResource != 'self'), Azure DevOps checks out the +# self repo to $(Build.SourcesDirectory)/. Set this to match the auto-generated +# directory name. Using the auto-generated name is necessary rather than explicitly +# defining a checkout path because container jobs expect repos to live under the agent's +# workspace ($(Pipeline.Workspace)). On some self-hosted setups the host path +# (e.g., /mnt/vss/_work) differs from the container path (e.g., /__w), and a custom checkout +# path can fail validation. Using the default checkout location keeps the paths consistent +# and avoids this issue. +- name: selfRepoName + type: string + default: '' +- name: arcadeRepoName + type: string + default: '' + +# Pool configuration for the job. +- name: pool + type: object + default: + name: NetCore1ESPool-Internal + image: build.azurelinux.3.amd64 + os: linux + +jobs: +- job: Renovate + displayName: Run Renovate + container: RenovateContainer + variables: + - group: dotnet-renovate-bot + # The Renovate version is automatically updated by https://github.com/dotnet/arcade/blob/main/azure-pipelines-renovate.yml. + # Changing the variable name here would require updating the name in https://github.com/dotnet/arcade/blob/main/eng/renovate.json as well. + - name: renovateVersion + value: '42' + readonly: true + - name: renovateLogFilePath + value: '$(Build.ArtifactStagingDirectory)/renovate.json' + readonly: true + - name: dryRunArg + readonly: true + ${{ if eq(parameters.dryRun, true) }}: + value: 'full' + ${{ else }}: + value: '' + - name: recreateWhenArg + readonly: true + ${{ if eq(parameters.forceRecreatePR, true) }}: + value: 'always' + ${{ else }}: + value: '' + # In multi-checkout (without custom paths), Azure DevOps places each repo under + # $(Build.SourcesDirectory)/. selfRepoName must be provided in that case. + - name: selfRepoPath + readonly: true + ${{ if eq(parameters.arcadeRepoResource, 'self') }}: + value: '$(Build.SourcesDirectory)' + ${{ else }}: + value: '$(Build.SourcesDirectory)/${{ parameters.selfRepoName }}' + - name: arcadeRepoPath + readonly: true + ${{ if eq(parameters.arcadeRepoResource, 'self') }}: + value: '$(Build.SourcesDirectory)' + ${{ else }}: + value: '$(Build.SourcesDirectory)/${{ parameters.arcadeRepoName }}' + pool: ${{ parameters.pool }} + + templateContext: + outputParentDirectory: $(Build.ArtifactStagingDirectory) + outputs: + - output: pipelineArtifact + displayName: Publish Renovate Log + condition: succeededOrFailed() + targetPath: $(Build.ArtifactStagingDirectory) + artifactName: $(Agent.JobName)_Logs_Attempt$(System.JobAttempt) + isProduction: false # logs are non-production artifacts + + steps: + - checkout: self + fetchDepth: 1 + + - ${{ if ne(parameters.arcadeRepoResource, 'self') }}: + - checkout: ${{ parameters.arcadeRepoResource }} + fetchDepth: 1 + + - script: | + renovate-config-validator $(selfRepoPath)/${{parameters.renovateConfigPath}} 2>&1 | tee /tmp/renovate-config-validator.out + validatorExit=${PIPESTATUS[0]} + if grep -q '^ WARN:' /tmp/renovate-config-validator.out; then + echo "##vso[task.logissue type=warning]Renovate config validator produced warnings." + echo "##vso[task.complete result=SucceededWithIssues]" + fi + exit $validatorExit + displayName: Validate Renovate config + env: + LOG_LEVEL: info + LOG_FILE_LEVEL: debug + LOG_FILE: $(Build.ArtifactStagingDirectory)/renovate-config-validator.json + + - script: | + . $(arcadeRepoPath)/eng/common/renovate.env + renovate 2>&1 | tee /tmp/renovate.out + renovateExit=${PIPESTATUS[0]} + if grep -q '^ WARN:' /tmp/renovate.out; then + echo "##vso[task.logissue type=warning]Renovate produced warnings." + echo "##vso[task.complete result=SucceededWithIssues]" + fi + exit $renovateExit + displayName: Run Renovate + env: + RENOVATE_FORK_TOKEN: $(BotAccount-dotnet-renovate-bot-PAT) + RENOVATE_TOKEN: $(BotAccount-dotnet-renovate-bot-PAT) + RENOVATE_REPOSITORIES: ${{parameters.gitHubRepo}} + RENOVATE_BASE_BRANCHES: ${{ convertToJson(parameters.baseBranches) }} + RENOVATE_DRY_RUN: $(dryRunArg) + RENOVATE_RECREATE_WHEN: $(recreateWhenArg) + LOG_LEVEL: info + LOG_FILE_LEVEL: debug + LOG_FILE: $(renovateLogFilePath) + RENOVATE_CONFIG_FILE: $(selfRepoPath)/${{parameters.renovateConfigPath}} + + - script: | + echo "PRs created by Renovate:" + if [ -s "$(renovateLogFilePath)" ]; then + if ! jq -r 'select(.msg == "PR created" and .pr != null) | "https://github.com/\(.repository)/pull/\(.pr)"' "$(renovateLogFilePath)" | sort -u; then + echo "##vso[task.logissue type=warning]Failed to parse Renovate log file with jq." + echo "##vso[task.complete result=SucceededWithIssues]" + fi + else + echo "##vso[task.logissue type=warning]No Renovate log file found or file is empty." + echo "##vso[task.complete result=SucceededWithIssues]" + fi + displayName: List created PRs + condition: and(succeededOrFailed(), eq('${{ parameters.dryRun }}', false)) diff --git a/eng/common/core-templates/job/source-index-stage1.yml b/eng/common/core-templates/job/source-index-stage1.yml index 76baf5c2725..bac6ac5faac 100644 --- a/eng/common/core-templates/job/source-index-stage1.yml +++ b/eng/common/core-templates/job/source-index-stage1.yml @@ -15,6 +15,8 @@ jobs: variables: - name: BinlogPath value: ${{ parameters.binlogPath }} + - name: skipComponentGovernanceDetection + value: true - template: /eng/common/core-templates/variables/pool-providers.yml parameters: is1ESPipeline: ${{ parameters.is1ESPipeline }} @@ -25,10 +27,10 @@ jobs: pool: ${{ if eq(variables['System.TeamProject'], 'public') }}: name: $(DncEngPublicBuildPool) - image: windows.vs2026preview.scout.amd64.open + image: windows.vs2026.amd64.open ${{ if eq(variables['System.TeamProject'], 'internal') }}: name: $(DncEngInternalBuildPool) - image: windows.vs2026preview.scout.amd64 + image: windows.vs2026.amd64 steps: - ${{ if eq(parameters.is1ESPipeline, '') }}: diff --git a/eng/common/core-templates/post-build/common-variables.yml b/eng/common/core-templates/post-build/common-variables.yml index d5627a994ae..db298ae16ba 100644 --- a/eng/common/core-templates/post-build/common-variables.yml +++ b/eng/common/core-templates/post-build/common-variables.yml @@ -11,8 +11,6 @@ variables: - name: MaestroApiVersion value: "2020-02-20" - - name: SourceLinkCLIVersion - value: 3.0.0 - name: SymbolToolVersion value: 1.0.1 - name: BinlogToolVersion diff --git a/eng/common/core-templates/post-build/post-build.yml b/eng/common/core-templates/post-build/post-build.yml index 905a6315e2d..8aa86e30491 100644 --- a/eng/common/core-templates/post-build/post-build.yml +++ b/eng/common/core-templates/post-build/post-build.yml @@ -1,118 +1,108 @@ parameters: - # Which publishing infra should be used. THIS SHOULD MATCH THE VERSION ON THE BUILD MANIFEST. - # Publishing V1 is no longer supported - # Publishing V2 is no longer supported - # Publishing V3 is the default - - name: publishingInfraVersion - displayName: Which version of publishing should be used to promote the build definition? - type: number - default: 3 - values: - - 3 - - 4 - - - name: BARBuildId - displayName: BAR Build Id - type: number - default: 0 - - - name: PromoteToChannelIds - displayName: Channel to promote BARBuildId to - type: string - default: '' - - - name: enableSourceLinkValidation - displayName: Enable SourceLink validation - type: boolean - default: false - - - name: enableSigningValidation - displayName: Enable signing validation - type: boolean - default: true - - - name: enableSymbolValidation - displayName: Enable symbol validation - type: boolean - default: false - - - name: enableNugetValidation - displayName: Enable NuGet validation - type: boolean - default: true - - - name: publishInstallersAndChecksums - displayName: Publish installers and checksums - type: boolean - default: true - - - name: requireDefaultChannels - displayName: Fail the build if there are no default channel(s) registrations for the current build - type: boolean - default: false - - - name: SDLValidationParameters - type: object - default: - enable: false - publishGdn: false - continueOnError: false - params: '' - artifactNames: '' - downloadArtifacts: true - - - name: isAssetlessBuild - type: boolean - displayName: Is Assetless Build - default: false - - # These parameters let the user customize the call to sdk-task.ps1 for publishing - # symbols & general artifacts as well as for signing validation - - name: symbolPublishingAdditionalParameters - displayName: Symbol publishing additional parameters - type: string - default: '' - - - name: artifactsPublishingAdditionalParameters - displayName: Artifact publishing additional parameters - type: string - default: '' - - - name: signingValidationAdditionalParameters - displayName: Signing validation additional parameters - type: string - default: '' - - # Which stages should finish execution before post-build stages start - - name: validateDependsOn - type: object - default: - - build - - - name: publishDependsOn - type: object - default: - - Validate - - # Optional: Call asset publishing rather than running in a separate stage - - name: publishAssetsImmediately - type: boolean - default: false - - - name: is1ESPipeline - type: boolean - default: false +# Which publishing infra should be used. THIS SHOULD MATCH THE VERSION ON THE BUILD MANIFEST. +# Publishing V1 is no longer supported +# Publishing V2 is no longer supported +# Publishing V3 is the default +- name: publishingInfraVersion + displayName: Which version of publishing should be used to promote the build definition? + type: number + default: 3 + values: + - 3 + - 4 + +- name: BARBuildId + displayName: BAR Build Id + type: number + default: 0 + +- name: PromoteToChannelIds + displayName: Channel to promote BARBuildId to + type: string + default: '' + +- name: enableSourceLinkValidation + displayName: Enable SourceLink validation + type: boolean + default: false + +- name: enableSigningValidation + displayName: Enable signing validation + type: boolean + default: true + +- name: enableSymbolValidation + displayName: Enable symbol validation + type: boolean + default: false + +- name: enableNugetValidation + displayName: Enable NuGet validation + type: boolean + default: true + +- name: publishInstallersAndChecksums + displayName: Publish installers and checksums + type: boolean + default: true + +- name: requireDefaultChannels + displayName: Fail the build if there are no default channel(s) registrations for the current build + type: boolean + default: false + +- name: isAssetlessBuild + type: boolean + displayName: Is Assetless Build + default: false + +# These parameters let the user customize the call to sdk-task.ps1 for publishing +# symbols & general artifacts as well as for signing validation +- name: symbolPublishingAdditionalParameters + displayName: Symbol publishing additional parameters + type: string + default: '' + +- name: artifactsPublishingAdditionalParameters + displayName: Artifact publishing additional parameters + type: string + default: '' + +- name: signingValidationAdditionalParameters + displayName: Signing validation additional parameters + type: string + default: '' + +# Which stages should finish execution before post-build stages start +- name: validateDependsOn + type: object + default: + - build + +- name: publishDependsOn + type: object + default: + - Validate + +# Optional: Call asset publishing rather than running in a separate stage +- name: publishAssetsImmediately + type: boolean + default: false + +- name: is1ESPipeline + type: boolean + default: false stages: -- ${{ if or(eq( parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}: +- ${{ if or(eq( parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true')) }}: - stage: Validate dependsOn: ${{ parameters.validateDependsOn }} displayName: Validate Build Assets variables: - - template: /eng/common/core-templates/post-build/common-variables.yml - - template: /eng/common/core-templates/variables/pool-providers.yml - parameters: - is1ESPipeline: ${{ parameters.is1ESPipeline }} + - template: /eng/common/core-templates/post-build/common-variables.yml + - template: /eng/common/core-templates/variables/pool-providers.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} jobs: - job: displayName: NuGet Validation @@ -128,49 +118,49 @@ stages: ${{ else }}: ${{ if eq(parameters.is1ESPipeline, true) }}: name: $(DncEngInternalBuildPool) - image: windows.vs2026preview.scout.amd64 + image: windows.vs2026.amd64 os: windows ${{ else }}: name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals windows.vs2026preview.scout.amd64 + demands: ImageOverride -equals windows.vs2026.amd64 steps: - - template: /eng/common/core-templates/post-build/setup-maestro-vars.yml - parameters: - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - is1ESPipeline: ${{ parameters.is1ESPipeline }} - - - ${{ if ne(parameters.publishingInfraVersion, 4) }}: - - task: DownloadBuildArtifacts@0 - displayName: Download Package Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: PackageArtifacts - checkDownloadedFiles: true - - ${{ if eq(parameters.publishingInfraVersion, 4) }}: - - task: DownloadPipelineArtifact@2 - displayName: Download Pipeline Artifacts (V4) - inputs: - itemPattern: '*/packages/**/*.nupkg' - targetPath: '$(Build.ArtifactStagingDirectory)/PipelineArtifactsDownload' - - task: CopyFiles@2 - displayName: Flatten packages to PackageArtifacts - inputs: - SourceFolder: '$(Build.ArtifactStagingDirectory)/PipelineArtifactsDownload' - Contents: '**/*.nupkg' - TargetFolder: '$(Build.ArtifactStagingDirectory)/PackageArtifacts' - flattenFolders: true - - - task: PowerShell@2 - displayName: Validate + - template: /eng/common/core-templates/post-build/setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + is1ESPipeline: ${{ parameters.is1ESPipeline }} + + - ${{ if ne(parameters.publishingInfraVersion, 4) }}: + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: PackageArtifacts + checkDownloadedFiles: true + - ${{ if eq(parameters.publishingInfraVersion, 4) }}: + - task: DownloadPipelineArtifact@2 + displayName: Download Pipeline Artifacts (V4) + inputs: + itemPattern: '*/packages/**/*.nupkg' + targetPath: '$(Build.ArtifactStagingDirectory)/PipelineArtifactsDownload' + - task: CopyFiles@2 + displayName: Flatten packages to PackageArtifacts inputs: - filePath: $(System.DefaultWorkingDirectory)/eng/common/post-build/nuget-validation.ps1 - arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ + SourceFolder: '$(Build.ArtifactStagingDirectory)/PipelineArtifactsDownload' + Contents: '**/*.nupkg' + TargetFolder: '$(Build.ArtifactStagingDirectory)/PackageArtifacts' + flattenFolders: true + + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: $(System.DefaultWorkingDirectory)/eng/common/post-build/nuget-validation.ps1 + arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ - job: displayName: Signing Validation @@ -184,143 +174,96 @@ stages: os: windows # If it's not devdiv, it's dnceng ${{ else }}: - ${{ if eq(parameters.is1ESPipeline, true) }}: + ${{ if eq(parameters.is1ESPipeline, true) }}: name: $(DncEngInternalBuildPool) image: windows.vs2026.amd64 os: windows ${{ else }}: name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals windows.vs2026preview.scout.amd64 + demands: ImageOverride -equals windows.vs2026.amd64 steps: - - template: /eng/common/core-templates/post-build/setup-maestro-vars.yml - parameters: - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - is1ESPipeline: ${{ parameters.is1ESPipeline }} - - - ${{ if ne(parameters.publishingInfraVersion, 4) }}: - - task: DownloadBuildArtifacts@0 - displayName: Download Package Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: PackageArtifacts - checkDownloadedFiles: true - - ${{ if eq(parameters.publishingInfraVersion, 4) }}: - - task: DownloadPipelineArtifact@2 - displayName: Download Pipeline Artifacts (V4) - inputs: - itemPattern: '*/packages/**/*.nupkg' - targetPath: '$(Build.ArtifactStagingDirectory)/PipelineArtifactsDownload' - - task: CopyFiles@2 - displayName: Flatten packages to PackageArtifacts - inputs: - SourceFolder: '$(Build.ArtifactStagingDirectory)/PipelineArtifactsDownload' - Contents: '**/*.nupkg' - TargetFolder: '$(Build.ArtifactStagingDirectory)/PackageArtifacts' - flattenFolders: true - - # This is necessary whenever we want to publish/restore to an AzDO private feed - # Since sdk-task.ps1 tries to restore packages we need to do this authentication here - # otherwise it'll complain about accessing a private feed. - - task: NuGetAuthenticate@1 - displayName: 'Authenticate to AzDO Feeds' - - # Signing validation will optionally work with the buildmanifest file which is downloaded from - # Azure DevOps above. - - task: PowerShell@2 - displayName: Validate - inputs: - filePath: eng\common\sdk-task.ps1 - arguments: -task SigningValidation -restore -msbuildEngine vs - /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts' - /p:SignCheckExclusionsFile='$(System.DefaultWorkingDirectory)/eng/SignCheckExclusionsFile.txt' - ${{ parameters.signingValidationAdditionalParameters }} - - - template: /eng/common/core-templates/steps/publish-logs.yml - parameters: - is1ESPipeline: ${{ parameters.is1ESPipeline }} - StageLabel: 'Validation' - JobLabel: 'Signing' - BinlogToolVersion: $(BinlogToolVersion) + - template: /eng/common/core-templates/post-build/setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + is1ESPipeline: ${{ parameters.is1ESPipeline }} - - job: - displayName: SourceLink Validation - condition: eq( ${{ parameters.enableSourceLinkValidation }}, 'true') - pool: - # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) - ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: - name: AzurePipelines-EO - image: 1ESPT-Windows2025 - demands: Cmd - os: windows - # If it's not devdiv, it's dnceng - ${{ else }}: - ${{ if eq(parameters.is1ESPipeline, true) }}: - name: $(DncEngInternalBuildPool) - image: windows.vs2026.amd64 - os: windows - ${{ else }}: - name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals windows.vs2026preview.scout.amd64 - steps: - - template: /eng/common/core-templates/post-build/setup-maestro-vars.yml - parameters: - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - is1ESPipeline: ${{ parameters.is1ESPipeline }} - - - ${{ if ne(parameters.publishingInfraVersion, 4) }}: - - task: DownloadBuildArtifacts@0 - displayName: Download Blob Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: BlobArtifacts - checkDownloadedFiles: true - - ${{ if eq(parameters.publishingInfraVersion, 4) }}: - - task: DownloadPipelineArtifact@2 - displayName: Download Pipeline Artifacts (V4) - inputs: - itemPattern: '*/assets/**' - targetPath: '$(Build.ArtifactStagingDirectory)/PipelineArtifactsDownload' - - task: CopyFiles@2 - displayName: Flatten assets to BlobArtifacts - inputs: - SourceFolder: '$(Build.ArtifactStagingDirectory)/PipelineArtifactsDownload' - Contents: '**/*' - TargetFolder: '$(Build.ArtifactStagingDirectory)/BlobArtifacts' - flattenFolders: true - - - task: PowerShell@2 - displayName: Validate + - ${{ if ne(parameters.publishingInfraVersion, 4) }}: + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: PackageArtifacts + checkDownloadedFiles: true + - ${{ if eq(parameters.publishingInfraVersion, 4) }}: + - task: DownloadPipelineArtifact@2 + displayName: Download Pipeline Artifacts (V4) + inputs: + itemPattern: '*/packages/**/*.nupkg' + targetPath: '$(Build.ArtifactStagingDirectory)/PipelineArtifactsDownload' + - task: CopyFiles@2 + displayName: Flatten packages to PackageArtifacts + inputs: + SourceFolder: '$(Build.ArtifactStagingDirectory)/PipelineArtifactsDownload' + Contents: '**/*.nupkg' + TargetFolder: '$(Build.ArtifactStagingDirectory)/PackageArtifacts' + flattenFolders: true + + # This is necessary whenever we want to publish/restore to an AzDO private feed + # Since sdk-task.ps1 tries to restore packages we need to do this authentication here + # otherwise it'll complain about accessing a private feed. + - task: NuGetAuthenticate@1 + displayName: 'Authenticate to AzDO Feeds' + + # Signing validation will optionally work with the buildmanifest file which is downloaded from + # Azure DevOps above. + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task SigningValidation -restore -msbuildEngine dotnet + /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts' + /p:SignCheckExclusionsFile='$(System.DefaultWorkingDirectory)/eng/SignCheckExclusionsFile.txt' + ${{ parameters.signingValidationAdditionalParameters }} + + - template: /eng/common/core-templates/steps/publish-logs.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + StageLabel: 'Validation' + JobLabel: 'Signing' + BinlogToolVersion: $(BinlogToolVersion) + + # SourceLink validation has been removed — the underlying CLI tool + # (targeting netcoreapp2.1) has not functioned for years. + # The enableSourceLinkValidation parameter is kept but ignored so + # existing pipelines that pass it are not broken. + # See https://github.com/dotnet/arcade/issues/16647 + - ${{ if eq(parameters.enableSourceLinkValidation, 'true') }}: + - job: + displayName: 'SourceLink Validation Removed - please remove enableSourceLinkValidation from your pipeline' + pool: server + steps: + - task: Delay@1 + displayName: 'Warning: SourceLink validation removed (see https://github.com/dotnet/arcade/issues/16647)' inputs: - filePath: $(System.DefaultWorkingDirectory)/eng/common/post-build/sourcelink-validation.ps1 - arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/ - -ExtractPath $(Agent.BuildDirectory)/Extract/ - -GHRepoName $(Build.Repository.Name) - -GHCommit $(Build.SourceVersion) - -SourcelinkCliVersion $(SourceLinkCLIVersion) - continueOnError: true + delayForMinutes: '0' - ${{ if ne(parameters.publishAssetsImmediately, 'true') }}: - stage: publish_using_darc - ${{ if or(eq(parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}: + ${{ if or(eq(parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true')) }}: dependsOn: ${{ parameters.publishDependsOn }} ${{ else }}: dependsOn: ${{ parameters.validateDependsOn }} displayName: Publish using Darc variables: - - template: /eng/common/core-templates/post-build/common-variables.yml - - template: /eng/common/core-templates/variables/pool-providers.yml - parameters: - is1ESPipeline: ${{ parameters.is1ESPipeline }} + - template: /eng/common/core-templates/post-build/common-variables.yml + - template: /eng/common/core-templates/variables/pool-providers.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} jobs: - job: displayName: Publish Using Darc @@ -334,7 +277,7 @@ stages: os: windows # If it's not devdiv, it's dnceng ${{ else }}: - ${{ if eq(parameters.is1ESPipeline, true) }}: + ${{ if eq(parameters.is1ESPipeline, true) }}: name: NetCore1ESPool-Publishing-Internal image: windows.vs2026.amd64 os: windows @@ -342,34 +285,33 @@ stages: name: NetCore1ESPool-Publishing-Internal demands: ImageOverride -equals windows.vs2026.amd64 steps: - - template: /eng/common/core-templates/post-build/setup-maestro-vars.yml - parameters: - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - is1ESPipeline: ${{ parameters.is1ESPipeline }} - - - task: NuGetAuthenticate@1 - - # Populate internal runtime variables. - - template: /eng/common/templates/steps/enable-internal-sources.yml - parameters: - legacyCredential: $(dn-bot-dnceng-artifact-feeds-rw) - - - template: /eng/common/templates/steps/enable-internal-runtimes.yml + - template: /eng/common/core-templates/post-build/setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + is1ESPipeline: ${{ parameters.is1ESPipeline }} - # Darc is targeting 8.0, so make sure it's installed - - task: UseDotNet@2 - inputs: - version: 8.0.x + - task: NuGetAuthenticate@1 - - task: AzureCLI@2 - displayName: Publish Using Darc - inputs: - azureSubscription: "Darc: Maestro Production" - scriptType: ps - scriptLocation: scriptPath - scriptPath: $(System.DefaultWorkingDirectory)/eng/common/post-build/publish-using-darc.ps1 - arguments: > + # Populate internal runtime variables. + - template: /eng/common/templates/steps/enable-internal-sources.yml + parameters: + legacyCredential: $(dn-bot-dnceng-artifact-feeds-rw) + + - template: /eng/common/templates/steps/enable-internal-runtimes.yml + + - task: UseDotNet@2 + inputs: + version: 8.0.x + + - task: AzureCLI@2 + displayName: Publish Using Darc + inputs: + azureSubscription: "Darc: Maestro Production" + scriptType: ps + scriptLocation: scriptPath + scriptPath: $(System.DefaultWorkingDirectory)/eng/common/post-build/publish-using-darc.ps1 + arguments: > -BuildId $(BARBuildId) -PublishingInfraVersion 3 -AzdoToken '$(System.AccessToken)' diff --git a/eng/common/core-templates/stages/renovate.yml b/eng/common/core-templates/stages/renovate.yml new file mode 100644 index 00000000000..edab2818258 --- /dev/null +++ b/eng/common/core-templates/stages/renovate.yml @@ -0,0 +1,111 @@ +# -------------------------------------------------------------------------------------- +# Renovate Pipeline Template +# -------------------------------------------------------------------------------------- +# This template provides a complete reusable pipeline definition for running Renovate +# in a 1ES Official pipeline. Pipelines can extend from this template and only need +# to pass the Renovate job parameters. +# +# For more info, see https://github.com/dotnet/arcade/blob/main/Documentation/Renovate.md +# -------------------------------------------------------------------------------------- + +parameters: + +# Path to the Renovate configuration file within the repository. +- name: renovateConfigPath + type: string + default: 'eng/renovate.json' + +# GitHub repository to run Renovate against, in the format 'owner/repo'. +- name: gitHubRepo + type: string + +# List of base branches to target for Renovate PRs. +- name: baseBranches + type: object + default: + - main + +# When true, Renovate will run in dry run mode. +- name: dryRun + type: boolean + default: false + +# When true, Renovate will recreate PRs even if they were previously closed. +- name: forceRecreatePR + type: boolean + default: false + +# Name of the arcade repository resource in the pipeline. +# This allows repos which haven't been onboarded to Arcade to still use this +# template by checking out the repo as a resource with a custom name and pointing +# this parameter to it. +- name: arcadeRepoResource + type: string + default: 'self' + +- name: selfRepoName + type: string + default: '' +- name: arcadeRepoName + type: string + default: '' + +# Pool configuration for the pipeline. +- name: pool + type: object + default: + name: NetCore1ESPool-Internal + image: build.azurelinux.3.amd64 + os: linux + +# Renovate version used in the container image tag. +- name: renovateVersion + default: 43 + type: number + +# Pool configuration for SDL analysis. +- name: sdlPool + type: object + default: + name: NetCore1ESPool-Internal + image: windows.vs2026.amd64 + os: windows + +resources: + repositories: + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release + +extends: + template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates + parameters: + pool: ${{ parameters.pool }} + sdl: + sourceAnalysisPool: ${{ parameters.sdlPool }} + # When repos that aren't onboarded to Arcade use this template, they set the + # arcadeRepoResource parameter to point to their Arcade repo resource. In that case, + # Aracde will be excluded from SDL analysis. + ${{ if ne(parameters.arcadeRepoResource, 'self') }}: + sourceRepositoriesToScan: + exclude: + - repository: ${{ parameters.arcadeRepoResource }} + containers: + RenovateContainer: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:azurelinux-3.0-renovate-${{ parameters.renovateVersion }}-amd64 + stages: + - stage: Renovate + displayName: Run Renovate + jobs: + - template: /eng/common/core-templates/job/renovate.yml@${{ parameters.arcadeRepoResource }} + parameters: + renovateConfigPath: ${{ parameters.renovateConfigPath }} + gitHubRepo: ${{ parameters.gitHubRepo }} + baseBranches: ${{ parameters.baseBranches }} + dryRun: ${{ parameters.dryRun }} + forceRecreatePR: ${{ parameters.forceRecreatePR }} + pool: ${{ parameters.pool }} + arcadeRepoResource: ${{ parameters.arcadeRepoResource }} + selfRepoName: ${{ parameters.selfRepoName }} + arcadeRepoName: ${{ parameters.arcadeRepoName }} diff --git a/eng/common/core-templates/steps/install-microbuild-impl.yml b/eng/common/core-templates/steps/install-microbuild-impl.yml new file mode 100644 index 00000000000..da22beb3f60 --- /dev/null +++ b/eng/common/core-templates/steps/install-microbuild-impl.yml @@ -0,0 +1,34 @@ +parameters: + - name: microbuildTaskInputs + type: object + default: {} + + - name: microbuildEnv + type: object + default: {} + + - name: enablePreviewMicrobuild + type: boolean + default: false + + - name: condition + type: string + + - name: continueOnError + type: boolean + +steps: +- ${{ if eq(parameters.enablePreviewMicrobuild, true) }}: + - task: MicroBuildSigningPluginPreview@4 + displayName: Install Preview MicroBuild plugin + inputs: ${{ parameters.microbuildTaskInputs }} + env: ${{ parameters.microbuildEnv }} + continueOnError: ${{ parameters.continueOnError }} + condition: ${{ parameters.condition }} +- ${{ else }}: + - task: MicroBuildSigningPlugin@4 + displayName: Install MicroBuild plugin + inputs: ${{ parameters.microbuildTaskInputs }} + env: ${{ parameters.microbuildEnv }} + continueOnError: ${{ parameters.continueOnError }} + condition: ${{ parameters.condition }} diff --git a/eng/common/core-templates/steps/install-microbuild.yml b/eng/common/core-templates/steps/install-microbuild.yml index 553fce66b94..76a54e157fd 100644 --- a/eng/common/core-templates/steps/install-microbuild.yml +++ b/eng/common/core-templates/steps/install-microbuild.yml @@ -4,6 +4,8 @@ parameters: # Enable install tasks for MicroBuild on Mac and Linux # Will be ignored if 'enableMicrobuild' is false or 'Agent.Os' is 'Windows_NT' enableMicrobuildForMacAndLinux: false + # Enable preview version of MB signing plugin + enablePreviewMicrobuild: false # Determines whether the ESRP service connection information should be passed to the signing plugin. # This overlaps with _SignType to some degree. We only need the service connection for real signing. # It's important that the service connection not be passed to the MicroBuildSigningPlugin task in this place. @@ -13,6 +15,8 @@ parameters: microbuildUseESRP: true # Microbuild installation directory microBuildOutputFolder: $(Agent.TempDirectory)/MicroBuild + # Microbuild version + microbuildPluginVersion: 'latest' continueOnError: false @@ -69,42 +73,46 @@ steps: # YAML expansion, and Windows vs. Linux/Mac uses different service connections. However, # we can avoid including the MB install step if not enabled at all. This avoids a bunch of # extra pipeline authorizations, since most pipelines do not sign on non-Windows. - - task: MicroBuildSigningPlugin@4 - displayName: Install MicroBuild plugin (Windows) - inputs: - signType: $(_SignType) - zipSources: false - feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json - ${{ if eq(parameters.microbuildUseESRP, true) }}: - ConnectedServiceName: 'MicroBuild Signing Task (DevDiv)' - ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: - ConnectedPMEServiceName: 6cc74545-d7b9-4050-9dfa-ebefcc8961ea - ${{ else }}: - ConnectedPMEServiceName: 248d384a-b39b-46e3-8ad5-c2c210d5e7ca - env: - TeamName: $(_TeamName) - MicroBuildOutputFolderOverride: ${{ parameters.microBuildOutputFolder }} - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - continueOnError: ${{ parameters.continueOnError }} - condition: and(succeeded(), eq(variables['Agent.Os'], 'Windows_NT'), in(variables['_SignType'], 'real', 'test')) - - - ${{ if eq(parameters.enableMicrobuildForMacAndLinux, true) }}: - - task: MicroBuildSigningPlugin@4 - displayName: Install MicroBuild plugin (non-Windows) - inputs: + - template: /eng/common/core-templates/steps/install-microbuild-impl.yml + parameters: + enablePreviewMicrobuild: ${{ parameters.enablePreviewMicrobuild }} + microbuildTaskInputs: signType: $(_SignType) zipSources: false feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json - workingDirectory: ${{ parameters.microBuildOutputFolder }} + version: ${{ parameters.microbuildPluginVersion }} ${{ if eq(parameters.microbuildUseESRP, true) }}: ConnectedServiceName: 'MicroBuild Signing Task (DevDiv)' ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: - ConnectedPMEServiceName: beb8cb23-b303-4c95-ab26-9e44bc958d39 + ConnectedPMEServiceName: 6cc74545-d7b9-4050-9dfa-ebefcc8961ea ${{ else }}: - ConnectedPMEServiceName: c24de2a5-cc7a-493d-95e4-8e5ff5cad2bc - env: + ConnectedPMEServiceName: 248d384a-b39b-46e3-8ad5-c2c210d5e7ca + microbuildEnv: TeamName: $(_TeamName) MicroBuildOutputFolderOverride: ${{ parameters.microBuildOutputFolder }} SYSTEM_ACCESSTOKEN: $(System.AccessToken) continueOnError: ${{ parameters.continueOnError }} - condition: and(succeeded(), ne(variables['Agent.Os'], 'Windows_NT'), eq(variables['_SignType'], 'real')) + condition: and(succeeded(), eq(variables['Agent.Os'], 'Windows_NT'), in(variables['_SignType'], 'real', 'test')) + + - ${{ if eq(parameters.enableMicrobuildForMacAndLinux, true) }}: + - template: /eng/common/core-templates/steps/install-microbuild-impl.yml + parameters: + enablePreviewMicrobuild: ${{ parameters.enablePreviewMicrobuild }} + microbuildTaskInputs: + signType: $(_SignType) + zipSources: false + feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json + version: ${{ parameters.microbuildPluginVersion }} + workingDirectory: ${{ parameters.microBuildOutputFolder }} + ${{ if eq(parameters.microbuildUseESRP, true) }}: + ConnectedServiceName: 'MicroBuild Signing Task (DevDiv)' + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + ConnectedPMEServiceName: beb8cb23-b303-4c95-ab26-9e44bc958d39 + ${{ else }}: + ConnectedPMEServiceName: c24de2a5-cc7a-493d-95e4-8e5ff5cad2bc + microbuildEnv: + TeamName: $(_TeamName) + MicroBuildOutputFolderOverride: ${{ parameters.microBuildOutputFolder }} + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + continueOnError: ${{ parameters.continueOnError }} + condition: and(succeeded(), ne(variables['Agent.Os'], 'Windows_NT'), eq(variables['_SignType'], 'real')) diff --git a/eng/common/core-templates/steps/publish-logs.yml b/eng/common/core-templates/steps/publish-logs.yml index 4eed0312b80..5fc099a1143 100644 --- a/eng/common/core-templates/steps/publish-logs.yml +++ b/eng/common/core-templates/steps/publish-logs.yml @@ -33,8 +33,6 @@ steps: '$(publishing-dnceng-devdiv-code-r-build-re)' '$(dn-bot-all-orgs-artifact-feeds-rw)' '$(akams-client-id)' - '$(microsoft-symbol-server-pat)' - '$(symweb-symbol-server-pat)' '$(dnceng-symbol-server-pat)' '$(dn-bot-all-orgs-build-rw-code-rw)' '$(System.AccessToken)' @@ -61,3 +59,4 @@ steps: condition: always() retryCountOnTaskFailure: 10 # for any files being locked isProduction: false # logs are non-production artifacts + diff --git a/eng/common/core-templates/steps/source-build.yml b/eng/common/core-templates/steps/source-build.yml index 09ae5cd73ae..b75f59c428d 100644 --- a/eng/common/core-templates/steps/source-build.yml +++ b/eng/common/core-templates/steps/source-build.yml @@ -24,7 +24,7 @@ steps: # in the default public locations. internalRuntimeDownloadArgs= if [ '$(dotnetbuilds-internal-container-read-token-base64)' != '$''(dotnetbuilds-internal-container-read-token-base64)' ]; then - internalRuntimeDownloadArgs='/p:DotNetRuntimeSourceFeed=https://ci.dot.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) --runtimesourcefeed https://ci.dot.net/internal --runtimesourcefeedkey '$(dotnetbuilds-internal-container-read-token-base64)'' + internalRuntimeDownloadArgs='/p:DotNetRuntimeSourceFeed=https://ci.dot.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) --runtimesourcefeed https://ci.dot.net/internal --runtimesourcefeedkey $(dotnetbuilds-internal-container-read-token-base64)' fi buildConfig=Release diff --git a/eng/common/core-templates/steps/source-index-stage1-publish.yml b/eng/common/core-templates/steps/source-index-stage1-publish.yml index e9a694afa58..3ad83b8c307 100644 --- a/eng/common/core-templates/steps/source-index-stage1-publish.yml +++ b/eng/common/core-templates/steps/source-index-stage1-publish.yml @@ -1,6 +1,6 @@ parameters: - sourceIndexUploadPackageVersion: 2.0.0-20250818.1 - sourceIndexProcessBinlogPackageVersion: 1.0.1-20250818.1 + sourceIndexUploadPackageVersion: 2.0.0-20250906.1 + sourceIndexProcessBinlogPackageVersion: 1.0.1-20250906.1 sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json binlogPath: artifacts/log/Debug/Build.binlog @@ -14,8 +14,8 @@ steps: workingDirectory: $(Agent.TempDirectory) - script: | - $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version ${{parameters.sourceIndexProcessBinlogPackageVersion}} --add-source ${{parameters.SourceIndexPackageSource}} --tool-path $(Agent.TempDirectory)/.source-index/tools - $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version ${{parameters.sourceIndexUploadPackageVersion}} --add-source ${{parameters.SourceIndexPackageSource}} --tool-path $(Agent.TempDirectory)/.source-index/tools + $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version ${{parameters.sourceIndexProcessBinlogPackageVersion}} --source ${{parameters.sourceIndexPackageSource}} --tool-path $(Agent.TempDirectory)/.source-index/tools + $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version ${{parameters.sourceIndexUploadPackageVersion}} --source ${{parameters.sourceIndexPackageSource}} --tool-path $(Agent.TempDirectory)/.source-index/tools displayName: "Source Index: Download netsourceindex Tools" # Set working directory to temp directory so 'dotnet' doesn't try to use global.json and use the repo's sdk. workingDirectory: $(Agent.TempDirectory) diff --git a/eng/common/cross/build-rootfs.sh b/eng/common/cross/build-rootfs.sh index 8abfb71f727..cef5d2d6716 100755 --- a/eng/common/cross/build-rootfs.sh +++ b/eng/common/cross/build-rootfs.sh @@ -5,10 +5,11 @@ set -e usage() { echo "Usage: $0 [BuildArch] [CodeName] [lldbx.y] [llvmx[.y]] [--skipunmount] --rootfsdir ]" - echo "BuildArch can be: arm(default), arm64, armel, armv6, loongarch64, ppc64le, riscv64, s390x, x64, x86" + echo "BuildArch can be: arm(default), arm64, loongarch64, ppc64le, riscv64, s390x, x64, x86" echo "CodeName - optional, Code name for Linux, can be: xenial(default), zesty, bionic, alpine" echo " for alpine can be specified with version: alpineX.YY or alpineedge" echo " for FreeBSD can be: freebsd13, freebsd14" + echo " for OpenBSD can be: openbsd" echo " for illumos can be: illumos" echo " for Haiku can be: haiku." echo "lldbx.y - optional, LLDB version, can be: lldb3.9(default), lldb4.0, lldb5.0, lldb6.0 no-lldb. Ignored for alpine and FreeBSD" @@ -27,6 +28,8 @@ __BuildArch=arm __AlpineArch=armv7 __FreeBSDArch=arm __FreeBSDMachineArch=armv7 +__OpenBSDArch=arm +__OpenBSDMachineArch=armv7 __IllumosArch=arm7 __HaikuArch=arm __QEMUArch=arm @@ -72,7 +75,7 @@ __AlpinePackages+=" krb5-dev" __AlpinePackages+=" openssl-dev" __AlpinePackages+=" zlib-dev" -__FreeBSDBase="13.4-RELEASE" +__FreeBSDBase="13.5-RELEASE" __FreeBSDPkg="1.21.3" __FreeBSDABI="13" __FreeBSDPackages="libunwind" @@ -82,6 +85,12 @@ __FreeBSDPackages+=" openssl" __FreeBSDPackages+=" krb5" __FreeBSDPackages+=" terminfo-db" +__OpenBSDVersion="7.8" +__OpenBSDPackages="heimdal-libs" +__OpenBSDPackages+=" icu4c" +__OpenBSDPackages+=" inotify-tools" +__OpenBSDPackages+=" openssl" + __IllumosPackages="icu" __IllumosPackages+=" mit-krb5" __IllumosPackages+=" openssl" @@ -130,7 +139,6 @@ __AlpineKeys=' 616db30d:MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAnpUpyWDWjlUk3smlWeA0\nlIMW+oJ38t92CRLHH3IqRhyECBRW0d0aRGtq7TY8PmxjjvBZrxTNDpJT6KUk4LRm\na6A6IuAI7QnNK8SJqM0DLzlpygd7GJf8ZL9SoHSH+gFsYF67Cpooz/YDqWrlN7Vw\ntO00s0B+eXy+PCXYU7VSfuWFGK8TGEv6HfGMALLjhqMManyvfp8hz3ubN1rK3c8C\nUS/ilRh1qckdbtPvoDPhSbTDmfU1g/EfRSIEXBrIMLg9ka/XB9PvWRrekrppnQzP\nhP9YE3x/wbFc5QqQWiRCYyQl/rgIMOXvIxhkfe8H5n1Et4VAorkpEAXdsfN8KSVv\nLSMazVlLp9GYq5SUpqYX3KnxdWBgN7BJoZ4sltsTpHQ/34SXWfu3UmyUveWj7wp0\nx9hwsPirVI00EEea9AbP7NM2rAyu6ukcm4m6ATd2DZJIViq2es6m60AE6SMCmrQF\nwmk4H/kdQgeAELVfGOm2VyJ3z69fQuywz7xu27S6zTKi05Qlnohxol4wVb6OB7qG\nLPRtK9ObgzRo/OPumyXqlzAi/Yvyd1ZQk8labZps3e16bQp8+pVPiumWioMFJDWV\nGZjCmyMSU8V6MB6njbgLHoyg2LCukCAeSjbPGGGYhnKLm1AKSoJh3IpZuqcKCk5C\n8CM1S15HxV78s9dFntEqIokCAwEAAQ== 66ba20fe:MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAtfB12w4ZgqsXWZDfUAV/\n6Y4aHUKIu3q4SXrNZ7CXF9nXoAVYrS7NAxJdAodsY3vPCN0g5O8DFXR+390LdOuQ\n+HsGKCc1k5tX5ZXld37EZNTNSbR0k+NKhd9h6X3u6wqPOx7SIKxwAQR8qeeFq4pP\nrt9GAGlxtuYgzIIcKJPwE0dZlcBCg+GnptCUZXp/38BP1eYC+xTXSL6Muq1etYfg\nodXdb7Yl+2h1IHuOwo5rjgY5kpY7GcAs8AjGk3lDD/av60OTYccknH0NCVSmPoXK\nvrxDBOn0LQRNBLcAfnTKgHrzy0Q5h4TNkkyTgxkoQw5ObDk9nnabTxql732yy9BY\ns+hM9+dSFO1HKeVXreYSA2n1ndF18YAvAumzgyqzB7I4pMHXq1kC/8bONMJxwSkS\nYm6CoXKyavp7RqGMyeVpRC7tV+blkrrUml0BwNkxE+XnwDRB3xDV6hqgWe0XrifD\nYTfvd9ScZQP83ip0r4IKlq4GMv/R5shcCRJSkSZ6QSGshH40JYSoiwJf5FHbj9ND\n7do0UAqebWo4yNx63j/wb2ULorW3AClv0BCFSdPsIrCStiGdpgJDBR2P2NZOCob3\nG9uMj+wJD6JJg2nWqNJxkANXX37Qf8plgzssrhrgOvB0fjjS7GYhfkfmZTJ0wPOw\nA8+KzFseBh4UFGgue78KwgkCAwEAAQ== ' -__Keyring= __KeyringFile="/usr/share/keyrings/ubuntu-archive-keyring.gpg" __SkipSigCheck=0 __SkipEmulation=0 @@ -153,6 +161,10 @@ while :; do __AlpineArch=armv7 __QEMUArch=arm ;; + armel) + # this is only used for tizen-build-rootfs.sh + __BuildArch=armel + ;; arm64) __BuildArch=arm64 __UbuntuArch=arm64 @@ -160,31 +172,8 @@ while :; do __QEMUArch=aarch64 __FreeBSDArch=arm64 __FreeBSDMachineArch=aarch64 - ;; - armel) - __BuildArch=armel - __UbuntuArch=armel - __UbuntuRepo="http://archive.debian.org/debian/" - __CodeName=buster - __KeyringFile="/usr/share/keyrings/debian-archive-keyring.gpg" - __LLDB_Package="liblldb-6.0-dev" - __UbuntuPackages="${__UbuntuPackages// libomp-dev/}" - __UbuntuPackages="${__UbuntuPackages// libomp5/}" - __UbuntuSuites= - ;; - armv6) - __BuildArch=armv6 - __UbuntuArch=armhf - __QEMUArch=arm - __UbuntuRepo="http://raspbian.raspberrypi.org/raspbian/" - __CodeName=buster - __KeyringFile="/usr/share/keyrings/raspbian-archive-keyring.gpg" - __LLDB_Package="liblldb-6.0-dev" - __UbuntuSuites= - - if [[ -e "$__KeyringFile" ]]; then - __Keyring="--keyring $__KeyringFile" - fi + __OpenBSDArch=arm64 + __OpenBSDMachineArch=aarch64 ;; loongarch64) __BuildArch=loongarch64 @@ -193,10 +182,6 @@ while :; do __UbuntuArch=loong64 __UbuntuSuites=unreleased __LLDB_Package="liblldb-19-dev" - - if [[ "$__CodeName" == "sid" ]]; then - __UbuntuRepo="http://ftp.ports.debian.org/debian-ports/" - fi ;; riscv64) __BuildArch=riscv64 @@ -212,7 +197,7 @@ while :; do __AlpineArch=ppc64le __QEMUArch=ppc64le __UbuntuArch=ppc64el - __UbuntuRepo="http://ports.ubuntu.com/ubuntu-ports/" + __UbuntuRepo="https://ports.ubuntu.com/ubuntu-ports/" __UbuntuPackages="${__UbuntuPackages// libunwind8-dev/}" __UbuntuPackages="${__UbuntuPackages// libomp-dev/}" __UbuntuPackages="${__UbuntuPackages// libomp5/}" @@ -223,7 +208,7 @@ while :; do __AlpineArch=s390x __QEMUArch=s390x __UbuntuArch=s390x - __UbuntuRepo="http://ports.ubuntu.com/ubuntu-ports/" + __UbuntuRepo="https://ports.ubuntu.com/ubuntu-ports/" __UbuntuPackages="${__UbuntuPackages// libunwind8-dev/}" __UbuntuPackages="${__UbuntuPackages// libomp-dev/}" __UbuntuPackages="${__UbuntuPackages// libomp5/}" @@ -235,15 +220,17 @@ while :; do __UbuntuArch=amd64 __FreeBSDArch=amd64 __FreeBSDMachineArch=amd64 + __OpenBSDArch=amd64 + __OpenBSDMachineArch=amd64 __illumosArch=x86_64 __HaikuArch=x86_64 - __UbuntuRepo="http://archive.ubuntu.com/ubuntu/" + __UbuntuRepo="https://archive.ubuntu.com/ubuntu/" ;; x86) __BuildArch=x86 __UbuntuArch=i386 __AlpineArch=x86 - __UbuntuRepo="http://archive.ubuntu.com/ubuntu/" + __UbuntuRepo="https://archive.ubuntu.com/ubuntu/" ;; lldb*) version="$(echo "$lowerI" | tr -d '[:alpha:]-=')" @@ -295,9 +282,7 @@ while :; do ;; noble) # Ubuntu 24.04 __CodeName=noble - if [[ -z "$__LLDB_Package" ]]; then - __LLDB_Package="liblldb-19-dev" - fi + __LLDB_Package="liblldb-19-dev" ;; stretch) # Debian 9 __CodeName=stretch @@ -305,7 +290,7 @@ while :; do __KeyringFile="/usr/share/keyrings/debian-archive-keyring.gpg" if [[ -z "$__UbuntuRepo" ]]; then - __UbuntuRepo="http://ftp.debian.org/debian/" + __UbuntuRepo="https://archive.debian.org/debian/" fi ;; buster) # Debian 10 @@ -314,7 +299,7 @@ while :; do __KeyringFile="/usr/share/keyrings/debian-archive-keyring.gpg" if [[ -z "$__UbuntuRepo" ]]; then - __UbuntuRepo="http://archive.debian.org/debian/" + __UbuntuRepo="https://archive.debian.org/debian/" fi ;; bullseye) # Debian 11 @@ -322,7 +307,7 @@ while :; do __KeyringFile="/usr/share/keyrings/debian-archive-keyring.gpg" if [[ -z "$__UbuntuRepo" ]]; then - __UbuntuRepo="http://ftp.debian.org/debian/" + __UbuntuRepo="https://ftp.debian.org/debian/" fi ;; bookworm) # Debian 12 @@ -330,7 +315,7 @@ while :; do __KeyringFile="/usr/share/keyrings/debian-archive-keyring.gpg" if [[ -z "$__UbuntuRepo" ]]; then - __UbuntuRepo="http://ftp.debian.org/debian/" + __UbuntuRepo="https://ftp.debian.org/debian/" fi ;; sid) # Debian sid @@ -339,25 +324,21 @@ while :; do # Debian-Ports architectures need different values case "$__UbuntuArch" in - amd64|arm64|armel|armhf|i386|mips64el|ppc64el|riscv64|s390x) + amd64|arm64|armhf|i386|mips64el|ppc64el|riscv64|s390x) __KeyringFile="/usr/share/keyrings/debian-archive-keyring.gpg" if [[ -z "$__UbuntuRepo" ]]; then - __UbuntuRepo="http://ftp.debian.org/debian/" + __UbuntuRepo="https://ftp.debian.org/debian/" fi ;; *) __KeyringFile="/usr/share/keyrings/debian-ports-archive-keyring.gpg" if [[ -z "$__UbuntuRepo" ]]; then - __UbuntuRepo="http://ftp.ports.debian.org/debian-ports/" + __UbuntuRepo="https://ftp.debian.org/debian-ports/" fi ;; esac - - if [[ -e "$__KeyringFile" ]]; then - __Keyring="--keyring $__KeyringFile" - fi ;; tizen) __CodeName= @@ -383,10 +364,14 @@ while :; do ;; freebsd14) __CodeName=freebsd - __FreeBSDBase="14.2-RELEASE" + __FreeBSDBase="14.3-RELEASE" __FreeBSDABI="14" __SkipUnmount=1 ;; + openbsd) + __CodeName=openbsd + __SkipUnmount=1 + ;; illumos) __CodeName=illumos __SkipUnmount=1 @@ -457,7 +442,7 @@ fi __UbuntuPackages+=" ${__LLDB_Package:-}" if [[ -z "$__UbuntuRepo" ]]; then - __UbuntuRepo="http://ports.ubuntu.com/" + __UbuntuRepo="https://ports.ubuntu.com/" fi if [[ -n "$__LLVM_MajorVersion" ]]; then @@ -544,15 +529,15 @@ if [[ "$__CodeName" == "alpine" ]]; then # initialize DB # shellcheck disable=SC2086 "$__ApkToolsDir/apk.static" \ - -X "http://dl-cdn.alpinelinux.org/alpine/$version/main" \ - -X "http://dl-cdn.alpinelinux.org/alpine/$version/community" \ + -X "https://dl-cdn.alpinelinux.org/alpine/$version/main" \ + -X "https://dl-cdn.alpinelinux.org/alpine/$version/community" \ -U $__ApkSignatureArg --root "$__RootfsDir" --arch "$__AlpineArch" --initdb add if [[ "$__AlpineLlvmLibsLookup" == 1 ]]; then # shellcheck disable=SC2086 __AlpinePackages+=" $("$__ApkToolsDir/apk.static" \ - -X "http://dl-cdn.alpinelinux.org/alpine/$version/main" \ - -X "http://dl-cdn.alpinelinux.org/alpine/$version/community" \ + -X "https://dl-cdn.alpinelinux.org/alpine/$version/main" \ + -X "https://dl-cdn.alpinelinux.org/alpine/$version/community" \ -U $__ApkSignatureArg --root "$__RootfsDir" --arch "$__AlpineArch" \ search 'llvm*-libs' | grep -E '^llvm' | sort | tail -1 | sed 's/-[^-]*//2g')" fi @@ -560,8 +545,8 @@ if [[ "$__CodeName" == "alpine" ]]; then # install all packages in one go # shellcheck disable=SC2086 "$__ApkToolsDir/apk.static" \ - -X "http://dl-cdn.alpinelinux.org/alpine/$version/main" \ - -X "http://dl-cdn.alpinelinux.org/alpine/$version/community" \ + -X "https://dl-cdn.alpinelinux.org/alpine/$version/main" \ + -X "https://dl-cdn.alpinelinux.org/alpine/$version/community" \ -U $__ApkSignatureArg --root "$__RootfsDir" --arch "$__AlpineArch" $__NoEmulationArg \ add $__AlpinePackages @@ -578,7 +563,7 @@ elif [[ "$__CodeName" == "freebsd" ]]; then curl -SL "https://download.freebsd.org/ftp/releases/${__FreeBSDArch}/${__FreeBSDMachineArch}/${__FreeBSDBase}/base.txz" | tar -C "$__RootfsDir" -Jxf - ./lib ./usr/lib ./usr/libdata ./usr/include ./usr/share/keys ./etc ./bin/freebsd-version fi echo "ABI = \"FreeBSD:${__FreeBSDABI}:${__FreeBSDMachineArch}\"; FINGERPRINTS = \"${__RootfsDir}/usr/share/keys\"; REPOS_DIR = [\"${__RootfsDir}/etc/pkg\"]; REPO_AUTOUPDATE = NO; RUN_SCRIPTS = NO;" > "${__RootfsDir}"/usr/local/etc/pkg.conf - echo "FreeBSD: { url: \"pkg+http://pkg.FreeBSD.org/\${ABI}/quarterly\", mirror_type: \"srv\", signature_type: \"fingerprints\", fingerprints: \"/usr/share/keys/pkg\", enabled: yes }" > "${__RootfsDir}"/etc/pkg/FreeBSD.conf + echo "FreeBSD: { url: \"pkg+https://pkg.FreeBSD.org/\${ABI}/quarterly\", mirror_type: \"srv\", signature_type: \"fingerprints\", fingerprints: \"/usr/share/keys/pkg\", enabled: yes }" > "${__RootfsDir}"/etc/pkg/FreeBSD.conf mkdir -p "$__RootfsDir"/tmp # get and build package manager if [[ "$__hasWget" == 1 ]]; then @@ -595,6 +580,62 @@ elif [[ "$__CodeName" == "freebsd" ]]; then INSTALL_AS_USER=$(whoami) "$__RootfsDir"/host/sbin/pkg -r "$__RootfsDir" -C "$__RootfsDir"/usr/local/etc/pkg.conf update # shellcheck disable=SC2086 INSTALL_AS_USER=$(whoami) "$__RootfsDir"/host/sbin/pkg -r "$__RootfsDir" -C "$__RootfsDir"/usr/local/etc/pkg.conf install --yes $__FreeBSDPackages +elif [[ "$__CodeName" == "openbsd" ]]; then + # determine mirrors + OPENBSD_MIRROR="https://cdn.openbsd.org/pub/OpenBSD/$__OpenBSDVersion/$__OpenBSDMachineArch" + + # download base system sets + ensureDownloadTool + + BASE_SETS=(base comp) + for set in "${BASE_SETS[@]}"; do + FILE="${set}${__OpenBSDVersion//./}.tgz" + echo "Downloading $FILE..." + if [[ "$__hasWget" == 1 ]]; then + wget -O- "$OPENBSD_MIRROR/$FILE" | tar -C "$__RootfsDir" -xzpf - + else + curl -SL "$OPENBSD_MIRROR/$FILE" | tar -C "$__RootfsDir" -xzpf - + fi + done + + PKG_MIRROR="https://cdn.openbsd.org/pub/OpenBSD/${__OpenBSDVersion}/packages/${__OpenBSDMachineArch}" + + echo "Installing packages into sysroot..." + + # Fetch package index once + if [[ "$__hasWget" == 1 ]]; then + PKG_INDEX=$(wget -qO- "$PKG_MIRROR/") + else + PKG_INDEX=$(curl -s "$PKG_MIRROR/") + fi + + for pkg in $__OpenBSDPackages; do + PKG_FILE=$(echo "$PKG_INDEX" | grep -Po ">\K${pkg}-[0-9][^\" ]*\.tgz" \ + | sort -V | tail -n1) + + echo "Resolved package filename for $pkg: $PKG_FILE" + + [[ -z "$PKG_FILE" ]] && { echo "ERROR: Package $pkg not found"; exit 1; } + + if [[ "$__hasWget" == 1 ]]; then + wget -O- "$PKG_MIRROR/$PKG_FILE" | tar -C "$__RootfsDir" -xzpf - + else + curl -SL "$PKG_MIRROR/$PKG_FILE" | tar -C "$__RootfsDir" -xzpf - + fi + done + + echo "Creating versionless symlinks for shared libraries..." + # Find all versioned .so files and create the base .so symlink + for lib in "$__RootfsDir/usr/lib/libc++.so."* "$__RootfsDir/usr/lib/libc++abi.so."* "$__RootfsDir/usr/lib/libpthread.so."*; do + if [ -f "$lib" ]; then + # Extract the filename (e.g., libc++.so.12.0) + VERSIONED_NAME=$(basename "$lib") + # Remove the trailing version numbers (e.g., libc++.so) + BASE_NAME=${VERSIONED_NAME%.so.*}.so + # Create the symlink in the same directory + ln -sf "$VERSIONED_NAME" "$__RootfsDir/usr/lib/$BASE_NAME" + fi + done elif [[ "$__CodeName" == "illumos" ]]; then mkdir "$__RootfsDir/tmp" pushd "$__RootfsDir/tmp" @@ -759,6 +800,14 @@ elif [[ "$__CodeName" == "haiku" ]]; then elif [[ -n "$__CodeName" ]]; then __Suites="$__CodeName $(for suite in $__UbuntuSuites; do echo -n "$__CodeName-$suite "; done)" + __SigCheckArgs= + if [[ "$__SkipSigCheck" == "0" ]]; then + if [[ -e "$__KeyringFile" ]]; then + __SigCheckArgs="--keyring $__KeyringFile" + fi + __SigCheckArgs="$__SigCheckArgs --force-check-gpg" + fi + if [[ "$__SkipEmulation" == "1" ]]; then if [[ -z "$AR" ]]; then if command -v ar &>/dev/null; then @@ -774,31 +823,23 @@ elif [[ -n "$__CodeName" ]]; then PYTHON=${PYTHON_EXECUTABLE:-python3} # shellcheck disable=SC2086,SC2046 - echo running "$PYTHON" "$__CrossDir/install-debs.py" --arch "$__UbuntuArch" --mirror "$__UbuntuRepo" --rootfsdir "$__RootfsDir" --artool "$AR" \ + echo running "$PYTHON" "$__CrossDir/install-debs.py" $__SigCheckArgs --arch "$__UbuntuArch" --mirror "$__UbuntuRepo" --rootfsdir "$__RootfsDir" --artool "$AR" \ $(for suite in $__Suites; do echo -n "--suite $suite "; done) \ $__UbuntuPackages # shellcheck disable=SC2086,SC2046 - "$PYTHON" "$__CrossDir/install-debs.py" --arch "$__UbuntuArch" --mirror "$__UbuntuRepo" --rootfsdir "$__RootfsDir" --artool "$AR" \ + "$PYTHON" "$__CrossDir/install-debs.py" $__SigCheckArgs --arch "$__UbuntuArch" --mirror "$__UbuntuRepo" --rootfsdir "$__RootfsDir" --artool "$AR" \ $(for suite in $__Suites; do echo -n "--suite $suite "; done) \ $__UbuntuPackages exit 0 fi - __UpdateOptions= - if [[ "$__SkipSigCheck" == "0" ]]; then - __Keyring="$__Keyring --force-check-gpg" - else - __Keyring= - __UpdateOptions="--allow-unauthenticated --allow-insecure-repositories" - fi - # shellcheck disable=SC2086 - echo running debootstrap "--variant=minbase" $__Keyring --arch "$__UbuntuArch" "$__CodeName" "$__RootfsDir" "$__UbuntuRepo" + echo running debootstrap "--variant=minbase" $__SigCheckArgs --arch "$__UbuntuArch" "$__CodeName" "$__RootfsDir" "$__UbuntuRepo" # shellcheck disable=SC2086 - if ! debootstrap "--variant=minbase" $__Keyring --arch "$__UbuntuArch" "$__CodeName" "$__RootfsDir" "$__UbuntuRepo"; then + if ! debootstrap "--variant=minbase" $__SigCheckArgs --arch "$__UbuntuArch" "$__CodeName" "$__RootfsDir" "$__UbuntuRepo"; then echo "debootstrap failed! dumping debootstrap.log" cat "$__RootfsDir/debootstrap/debootstrap.log" exit 1 @@ -816,6 +857,11 @@ Components: main universe Signed-By: $__KeyringFile EOF + __UpdateOptions= + if [[ "$__SkipSigCheck" == "1" ]]; then + __UpdateOptions="--allow-unauthenticated --allow-insecure-repositories" + fi + # shellcheck disable=SC2086 chroot "$__RootfsDir" apt-get update $__UpdateOptions chroot "$__RootfsDir" apt-get -f -y install diff --git a/eng/common/cross/install-debs.py b/eng/common/cross/install-debs.py index c81eb37e522..100c4378da3 100644 --- a/eng/common/cross/install-debs.py +++ b/eng/common/cross/install-debs.py @@ -4,6 +4,7 @@ import asyncio import aiohttp import gzip +import hashlib import os import re import shutil @@ -16,7 +17,7 @@ from collections import deque from functools import cmp_to_key -async def download_file(session, url, dest_path, max_retries=3, retry_delay=2, timeout=60): +async def download_file(session, url, dest_path, max_retries=3, retry_delay=2, timeout=60, checksum=None): """Asynchronous file download with retries.""" attempt = 0 while attempt < max_retries: @@ -25,19 +26,25 @@ async def download_file(session, url, dest_path, max_retries=3, retry_delay=2, t if response.status == 200: with open(dest_path, "wb") as f: content = await response.read() + + # verify checksum if provided + if checksum: + sha256 = hashlib.sha256(content).hexdigest() + if sha256 != checksum: + raise Exception(f"SHA256 mismatch for {url}: expected {checksum}, got {sha256}") + f.write(content) print(f"Downloaded {url} at {dest_path}") return else: - print(f"Failed to download {url}, Status Code: {response.status}") - break + raise Exception(f"Failed to download {url}, Status Code: {response.status}") except (asyncio.CancelledError, asyncio.TimeoutError, aiohttp.ClientError) as e: print(f"Error downloading {url}: {type(e).__name__} - {e}. Retrying...") attempt += 1 await asyncio.sleep(retry_delay) - print(f"Failed to download {url} after {max_retries} attempts.") + raise Exception(f"Failed to download {url} after {max_retries} attempts.") async def download_deb_files_parallel(mirror, packages, tmp_dir): """Download .deb files in parallel.""" @@ -51,11 +58,11 @@ async def download_deb_files_parallel(mirror, packages, tmp_dir): if filename: url = f"{mirror}/{filename}" dest_path = os.path.join(tmp_dir, os.path.basename(filename)) - tasks.append(asyncio.create_task(download_file(session, url, dest_path))) + tasks.append(asyncio.create_task(download_file(session, url, dest_path, checksum=info.get("SHA256")))) await asyncio.gather(*tasks) -async def download_package_index_parallel(mirror, arch, suites): +async def download_package_index_parallel(mirror, arch, suites, check_sig, keyring): """Download package index files for specified suites and components entirely in memory.""" tasks = [] timeout = aiohttp.ClientTimeout(total=60) @@ -63,10 +70,9 @@ async def download_package_index_parallel(mirror, arch, suites): async with aiohttp.ClientSession(timeout=timeout) as session: for suite in suites: for component in ["main", "universe"]: - url = f"{mirror}/dists/{suite}/{component}/binary-{arch}/Packages.gz" - tasks.append(fetch_and_decompress(session, url)) + tasks.append(fetch_and_decompress(session, mirror, arch, suite, component, check_sig, keyring)) - results = await asyncio.gather(*tasks, return_exceptions=True) + results = await asyncio.gather(*tasks) merged_content = "" for result in results: @@ -77,20 +83,71 @@ async def download_package_index_parallel(mirror, arch, suites): return merged_content -async def fetch_and_decompress(session, url): +async def fetch_and_decompress(session, mirror, arch, suite, component, check_sig, keyring): """Fetch and decompress the Packages.gz file.""" - try: - async with session.get(url) as response: - if response.status == 200: - compressed_data = await response.read() - decompressed_data = gzip.decompress(compressed_data).decode('utf-8') - print(f"Downloaded index: {url}") - return decompressed_data - else: - print(f"Skipped index: {url} (doesn't exist)") - return None - except Exception as e: - print(f"Error fetching {url}: {e}") + + path = f"{component}/binary-{arch}/Packages.gz" + url = f"{mirror}/dists/{suite}/{path}" + + async with session.get(url) as response: + if response.status == 200: + compressed_data = await response.read() + decompressed_data = gzip.decompress(compressed_data).decode('utf-8') + print(f"Downloaded index: {url}") + + if check_sig: + # Verify the package index against the sha256 recorded in the Release file + release_file_content = await fetch_release_file(session, mirror, suite, keyring) + packages_sha = parse_release_file(release_file_content, path) + + sha256 = hashlib.sha256(compressed_data).hexdigest() + if sha256 != packages_sha: + raise Exception(f"SHA256 mismatch for {path}: expected {packages_sha}, got {sha256}") + print(f"Checksum verified for {path}") + + return decompressed_data + else: + print(f"Skipped index: {url} (doesn't exist)") + return None + +async def fetch_release_file(session, mirror, suite, keyring): + """Fetch Release and Release.gpg files and verify the signature.""" + + release_url = f"{mirror}/dists/{suite}/Release" + release_gpg_url = f"{mirror}/dists/{suite}/Release.gpg" + + with tempfile.NamedTemporaryFile() as release_file, tempfile.NamedTemporaryFile() as release_gpg_file: + await download_file(session, release_url, release_file.name) + await download_file(session, release_gpg_url, release_gpg_file.name) + + print("Verifying signature of Release with Release.gpg.") + verify_command = ["gpg"] + if keyring: + verify_command += ["--keyring", keyring] + verify_command += ["--verify", release_gpg_file.name, release_file.name] + result = subprocess.run(verify_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + if result.returncode != 0: + raise Exception(f"Signature verification failed: {result.stderr.decode('utf-8')}") + + print("Signature verified successfully.") + + with open(release_file.name) as f: + return f.read() + +def parse_release_file(content, path): + """Parses the Release file and returns sha256 checksum of the specified path.""" + + # data looks like this: + # + matches = re.findall(r'^ (\S*) +(\S*) +(\S*)$', content, re.MULTILINE) + + for entry in matches: + # the file has both md5 and sha256 checksums, we want sha256 which has a length of 64 + if entry[2] == path and len(entry[0]) == 64: + return entry[0] + + raise Exception(f"Could not find checksum for {path} in Release file.") def parse_debian_version(version): """Parse a Debian package version into epoch, upstream version, and revision.""" @@ -171,13 +228,15 @@ def parse_package_index(content): filename = fields.get("Filename") depends = fields.get("Depends") provides = fields.get("Provides", None) + sha256 = fields.get("SHA256") # Only update if package_name is not in packages or if the new version is higher if package_name not in packages or compare_debian_versions(version, packages[package_name]["Version"]) > 0: packages[package_name] = { "Version": version, "Filename": filename, - "Depends": depends + "Depends": depends, + "SHA256": sha256 } # Update aliases if package provides any alternatives @@ -233,7 +292,7 @@ def extract_deb_file(deb_file, tmp_dir, extract_dir, ar_tool): os.makedirs(extract_dir, exist_ok=True) with tempfile.TemporaryDirectory(dir=tmp_dir) as tmp_subdir: - result = subprocess.run(f"{ar_tool} t {os.path.abspath(deb_file)}", cwd=tmp_subdir, check=True, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + result = subprocess.run([ar_tool, "t", os.path.abspath(deb_file)], cwd=tmp_subdir, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) tar_filename = None for line in result.stdout.decode().splitlines(): @@ -247,7 +306,8 @@ def extract_deb_file(deb_file, tmp_dir, extract_dir, ar_tool): tar_file_path = os.path.join(tmp_subdir, tar_filename) print(f"Extracting {tar_filename} from {deb_file}..") - subprocess.run(f"{ar_tool} p {os.path.abspath(deb_file)} {tar_filename} > {tar_file_path}", check=True, shell=True) + with open(tar_file_path, "wb") as outfile: + subprocess.run([ar_tool, "p", os.path.abspath(deb_file), tar_filename], check=True, stdout=outfile, stderr=subprocess.PIPE) file_extension = os.path.splitext(tar_file_path)[1].lower() @@ -268,7 +328,7 @@ def extract_deb_file(deb_file, tmp_dir, extract_dir, ar_tool): raise ValueError(f"Unsupported compression format: {file_extension}") with tarfile.open(tar_file_path, mode) as tar: - tar.extractall(path=extract_dir, filter='fully_trusted') + tar.extractall(path=extract_dir, filter='tar') def finalize_setup(rootfsdir): lib_dir = os.path.join(rootfsdir, 'lib') @@ -295,24 +355,17 @@ def finalize_setup(rootfsdir): if __name__ == "__main__": parser = argparse.ArgumentParser(description="Generate rootfs for .NET runtime on Debian-like OS") - parser.add_argument("--distro", required=False, help="Distro name (e.g., debian, ubuntu, etc.)") parser.add_argument("--arch", required=True, help="Architecture (e.g., amd64, loong64, etc.)") parser.add_argument("--rootfsdir", required=True, help="Destination directory.") parser.add_argument('--suite', required=True, action='append', help='Specify one or more repository suites to collect index data.') - parser.add_argument("--mirror", required=False, help="Mirror (e.g., http://ftp.debian.org/debian-ports etc.)") + parser.add_argument("--mirror", required=True, help="Mirror (e.g., http://ftp.debian.org/debian-ports etc.)") parser.add_argument("--artool", required=False, default="ar", help="ar tool to extract debs (e.g., ar, llvm-ar etc.)") + parser.add_argument("--force-check-gpg", required=False, action='store_true', help="Verify the packages against signatures in Release file.") + parser.add_argument("--keyring", required=False, default='', help="Keyring file to check signature of Release file.") parser.add_argument("packages", nargs="+", help="List of package names to be installed.") args = parser.parse_args() - if args.mirror is None: - if args.distro == "ubuntu": - args.mirror = "http://archive.ubuntu.com/ubuntu" if args.arch in ["amd64", "i386"] else "http://ports.ubuntu.com/ubuntu-ports" - elif args.distro == "debian": - args.mirror = "http://ftp.debian.org/debian-ports" - else: - raise Exception("Unsupported distro") - DESIRED_PACKAGES = args.packages + [ # base packages "dpkg", "busybox", @@ -322,9 +375,16 @@ def finalize_setup(rootfsdir): "debianutils" ] - print(f"Creating rootfs. rootfsdir: {args.rootfsdir}, distro: {args.distro}, arch: {args.arch}, suites: {args.suite}, mirror: {args.mirror}") + print(f"Creating rootfs. rootfsdir: {args.rootfsdir}, arch: {args.arch}, suites: {args.suite}, mirror: {args.mirror}") + + check_sig = args.force_check_gpg + if check_sig and not args.keyring: + print("ERROR: --force-check-gpg requires --keyring to specify a keyring file for signature verification.") + print("Install the appropriate keyring package (e.g., debian-ports-archive-keyring, ubuntu-archive-keyring)") + print("or pass --skipsigcheck to build-rootfs.sh to disable signature checking.") + sys.exit(1) - package_index_content = asyncio.run(download_package_index_parallel(args.mirror, args.arch, args.suite)) + package_index_content = asyncio.run(download_package_index_parallel(args.mirror, args.arch, args.suite, check_sig, args.keyring)) packages_info, aliases = parse_package_index(package_index_content) diff --git a/eng/common/cross/toolchain.cmake b/eng/common/cross/toolchain.cmake index 0ff85cf0367..99d6dfe82dd 100644 --- a/eng/common/cross/toolchain.cmake +++ b/eng/common/cross/toolchain.cmake @@ -3,15 +3,22 @@ set(CROSS_ROOTFS $ENV{ROOTFS_DIR}) # reset platform variables (e.g. cmake 3.25 sets LINUX=1) unset(LINUX) unset(FREEBSD) +unset(OPENBSD) unset(ILLUMOS) unset(ANDROID) unset(TIZEN) unset(HAIKU) set(TARGET_ARCH_NAME $ENV{TARGET_BUILD_ARCH}) + +file(GLOB OPENBSD_PROBE "${CROSS_ROOTFS}/etc/signify/openbsd-*.pub") + if(EXISTS ${CROSS_ROOTFS}/bin/freebsd-version) set(CMAKE_SYSTEM_NAME FreeBSD) set(FREEBSD 1) +elseif(OPENBSD_PROBE) + set(CMAKE_SYSTEM_NAME OpenBSD) + set(OPENBSD 1) elseif(EXISTS ${CROSS_ROOTFS}/usr/platform/i86pc) set(CMAKE_SYSTEM_NAME SunOS) set(ILLUMOS 1) @@ -53,6 +60,8 @@ elseif(TARGET_ARCH_NAME STREQUAL "arm64") endif() elseif(FREEBSD) set(triple "aarch64-unknown-freebsd12") + elseif(OPENBSD) + set(triple "aarch64-unknown-openbsd") endif() elseif(TARGET_ARCH_NAME STREQUAL "armel") set(CMAKE_SYSTEM_PROCESSOR armv7l) @@ -109,6 +118,8 @@ elseif(TARGET_ARCH_NAME STREQUAL "x64") endif() elseif(FREEBSD) set(triple "x86_64-unknown-freebsd12") + elseif(OPENBSD) + set(triple "x86_64-unknown-openbsd") elseif(ILLUMOS) set(TOOLCHAIN "x86_64-illumos") elseif(HAIKU) @@ -193,7 +204,7 @@ if(ANDROID) # include official NDK toolchain script include(${CROSS_ROOTFS}/../build/cmake/android.toolchain.cmake) -elseif(FREEBSD) +elseif(FREEBSD OR OPENBSD) # we cross-compile by instructing clang set(CMAKE_C_COMPILER_TARGET ${triple}) set(CMAKE_CXX_COMPILER_TARGET ${triple}) @@ -214,13 +225,19 @@ elseif(ILLUMOS) locate_toolchain_exec(g++ CMAKE_CXX_COMPILER) elseif(HAIKU) set(CMAKE_SYSROOT "${CROSS_ROOTFS}") - set(CMAKE_PROGRAM_PATH "${CMAKE_PROGRAM_PATH};${CROSS_ROOTFS}/cross-tools-x86_64/bin") set(CMAKE_SYSTEM_PREFIX_PATH "${CROSS_ROOTFS}") set(CMAKE_C_STANDARD_LIBRARIES "${CMAKE_C_STANDARD_LIBRARIES} -lssp") set(CMAKE_CXX_STANDARD_LIBRARIES "${CMAKE_CXX_STANDARD_LIBRARIES} -lssp") - locate_toolchain_exec(gcc CMAKE_C_COMPILER) - locate_toolchain_exec(g++ CMAKE_CXX_COMPILER) + if ($ENV{CCC_CC} MATCHES ".*gcc.*") + set(CMAKE_PROGRAM_PATH "${CMAKE_PROGRAM_PATH};${CROSS_ROOTFS}/cross-tools-x86_64/bin") + locate_toolchain_exec(gcc CMAKE_C_COMPILER) + locate_toolchain_exec(g++ CMAKE_CXX_COMPILER) + else() + set(CMAKE_C_COMPILER_EXTERNAL_TOOLCHAIN "${CROSS_ROOTFS}/cross-tools-x86_64") + set(CMAKE_CXX_COMPILER_EXTERNAL_TOOLCHAIN "${CROSS_ROOTFS}/cross-tools-x86_64") + set(CMAKE_ASM_COMPILER_EXTERNAL_TOOLCHAIN "${CROSS_ROOTFS}/cross-tools-x86_64") + endif() # let CMake set up the correct search paths include(Platform/Haiku) @@ -291,7 +308,7 @@ endif() # Specify compile options -if((TARGET_ARCH_NAME MATCHES "^(arm|arm64|armel|armv6|loongarch64|ppc64le|riscv64|s390x|x64|x86)$" AND NOT ANDROID AND NOT FREEBSD) OR ILLUMOS OR HAIKU) +if((TARGET_ARCH_NAME MATCHES "^(arm|arm64|armel|armv6|loongarch64|ppc64le|riscv64|s390x|x64|x86)$" AND NOT ANDROID AND NOT FREEBSD AND NOT OPENBSD) OR ILLUMOS OR HAIKU) set(CMAKE_C_COMPILER_TARGET ${TOOLCHAIN}) set(CMAKE_CXX_COMPILER_TARGET ${TOOLCHAIN}) set(CMAKE_ASM_COMPILER_TARGET ${TOOLCHAIN}) diff --git a/eng/common/darc-init.sh b/eng/common/darc-init.sh index e6ba4ee28c1..b56d40e5706 100755 --- a/eng/common/darc-init.sh +++ b/eng/common/darc-init.sh @@ -5,7 +5,7 @@ darcVersion='' versionEndpoint='https://maestro.dot.net/api/assets/darc-version?api-version=2020-02-20' verbosity='minimal' -while [[ $# > 0 ]]; do +while [[ $# -gt 0 ]]; do opt="$(echo "$1" | tr "[:upper:]" "[:lower:]")" case "$opt" in --darcversion) diff --git a/eng/common/dotnet-install.ps1 b/eng/common/dotnet-install.ps1 index 811f0f717f7..50ae6273768 100644 --- a/eng/common/dotnet-install.ps1 +++ b/eng/common/dotnet-install.ps1 @@ -10,7 +10,11 @@ Param( . $PSScriptRoot\tools.ps1 -$dotnetRoot = Join-Path $RepoRoot '.dotnet' +if (-not [string]::IsNullOrEmpty($env:DOTNET_GLOBAL_INSTALL_DIR)) { + $dotnetRoot = $env:DOTNET_GLOBAL_INSTALL_DIR +} else { + $dotnetRoot = Join-Path $RepoRoot '.dotnet' +} $installdir = $dotnetRoot try { diff --git a/eng/common/dotnet-install.sh b/eng/common/dotnet-install.sh index 7b9d97e3bd4..1cb3f5abac2 100755 --- a/eng/common/dotnet-install.sh +++ b/eng/common/dotnet-install.sh @@ -18,7 +18,7 @@ architecture='' runtime='dotnet' runtimeSourceFeed='' runtimeSourceFeedKey='' -while [[ $# > 0 ]]; do +while [[ $# -gt 0 ]]; do opt="$(echo "$1" | tr "[:upper:]" "[:lower:]")" case "$opt" in -version|-v) @@ -80,7 +80,11 @@ case $cpuname in ;; esac -dotnetRoot="${repo_root}.dotnet" +if [[ -n "${DOTNET_GLOBAL_INSTALL_DIR:-}" ]]; then + dotnetRoot="$DOTNET_GLOBAL_INSTALL_DIR" +else + dotnetRoot="${repo_root}.dotnet" +fi if [[ $architecture != "" ]] && [[ $architecture != $buildarch ]]; then dotnetRoot="$dotnetRoot/$architecture" fi diff --git a/eng/common/dotnet.sh b/eng/common/dotnet.sh index 2ef68235675..f6d24871c1d 100755 --- a/eng/common/dotnet.sh +++ b/eng/common/dotnet.sh @@ -19,7 +19,7 @@ source $scriptroot/tools.sh InitializeDotNetCli true # install # Invoke acquired SDK with args if they are provided -if [[ $# > 0 ]]; then +if [[ $# -gt 0 ]]; then __dotnetDir=${_InitializeDotNetCli} dotnetPath=${__dotnetDir}/dotnet ${dotnetPath} "$@" diff --git a/eng/common/internal-feed-operations.sh b/eng/common/internal-feed-operations.sh index 9378223ba09..6299e7effd4 100755 --- a/eng/common/internal-feed-operations.sh +++ b/eng/common/internal-feed-operations.sh @@ -100,7 +100,7 @@ operation='' authToken='' repoName='' -while [[ $# > 0 ]]; do +while [[ $# -gt 0 ]]; do opt="$(echo "$1" | tr "[:upper:]" "[:lower:]")" case "$opt" in --operation) diff --git a/eng/common/native/init-distro-rid.sh b/eng/common/native/init-distro-rid.sh index 83ea7aab0e0..8fc6d2fec78 100644 --- a/eng/common/native/init-distro-rid.sh +++ b/eng/common/native/init-distro-rid.sh @@ -39,6 +39,8 @@ getNonPortableDistroRid() # $rootfsDir can be empty. freebsd-version is a shell script and should always work. __freebsd_major_version=$("$rootfsDir"/bin/freebsd-version | cut -d'.' -f1) nonPortableRid="freebsd.$__freebsd_major_version-${targetArch}" + elif [ "$targetOs" = "openbsd" ]; then + nonPortableRid="openbsd.$(uname -r)-${targetArch}" elif command -v getprop >/dev/null && getprop ro.product.system.model | grep -qi android; then __android_sdk_version=$(getprop ro.build.version.sdk) nonPortableRid="android.$__android_sdk_version-${targetArch}" diff --git a/eng/common/native/install-dependencies.sh b/eng/common/native/install-dependencies.sh index 477a44f335b..4742177a768 100644 --- a/eng/common/native/install-dependencies.sh +++ b/eng/common/native/install-dependencies.sh @@ -24,14 +24,16 @@ case "$os" in apt update apt install -y build-essential gettext locales cmake llvm clang lld lldb liblldb-dev libunwind8-dev libicu-dev liblttng-ust-dev \ - libssl-dev libkrb5-dev pigz cpio + libssl-dev libkrb5-dev pigz cpio ninja-build localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 - elif [ "$ID" = "fedora" ] || [ "$ID" = "rhel" ] || [ "$ID" = "azurelinux" ]; then + elif [ "$ID" = "fedora" ] || [ "$ID" = "rhel" ] || [ "$ID" = "azurelinux" ] || [ "$ID" = "centos" ]; then pkg_mgr="$(command -v tdnf 2>/dev/null || command -v dnf)" - $pkg_mgr install -y cmake llvm lld lldb clang python curl libicu-devel openssl-devel krb5-devel lttng-ust-devel pigz cpio + $pkg_mgr install -y cmake llvm lld lldb clang python curl libicu-devel openssl-devel krb5-devel lttng-ust-devel pigz cpio ninja-build + elif [ "$ID" = "amzn" ]; then + dnf install -y cmake llvm lld lldb clang python libicu-devel openssl-devel krb5-devel lttng-ust-devel pigz cpio ninja-build elif [ "$ID" = "alpine" ]; then - apk add build-base cmake bash curl clang llvm-dev lld lldb krb5-dev lttng-ust-dev icu-dev openssl-dev pigz cpio + apk add build-base cmake bash curl clang llvm llvm-dev lld lldb-dev krb5-dev lttng-ust-dev icu-dev openssl-dev pigz cpio ninja else echo "Unsupported distro. distro: $ID" exit 1 @@ -52,6 +54,7 @@ brew "openssl@3" brew "pkgconf" brew "python3" brew "pigz" +brew "ninja" EOF ;; diff --git a/eng/common/post-build/redact-logs.ps1 b/eng/common/post-build/redact-logs.ps1 index c1e4104b79a..672f4e2652e 100644 --- a/eng/common/post-build/redact-logs.ps1 +++ b/eng/common/post-build/redact-logs.ps1 @@ -9,7 +9,8 @@ param( [Parameter(Mandatory=$false)][string] $TokensFilePath, [Parameter(ValueFromRemainingArguments=$true)][String[]]$TokensToRedact, [Parameter(Mandatory=$false)][string] $runtimeSourceFeed, - [Parameter(Mandatory=$false)][string] $runtimeSourceFeedKey) + [Parameter(Mandatory=$false)][string] $runtimeSourceFeedKey +) try { $ErrorActionPreference = 'Stop' diff --git a/eng/common/renovate.env b/eng/common/renovate.env new file mode 100644 index 00000000000..17ecc05d9b1 --- /dev/null +++ b/eng/common/renovate.env @@ -0,0 +1,42 @@ +# Renovate Global Configuration +# https://docs.renovatebot.com/self-hosted-configuration/ +# +# NOTE: This file uses bash/shell format and is sourced via `. renovate.env`. +# Values containing spaces or special characters must be quoted. + +# Author to use for git commits made by Renovate +# https://docs.renovatebot.com/configuration-options/#gitauthor +export RENOVATE_GIT_AUTHOR='.NET Renovate ' + +# Disable rate limiting for PR creation (0 = unlimited) +# https://docs.renovatebot.com/presets-default/#prhourlylimitnone +# https://docs.renovatebot.com/presets-default/#prconcurrentlimitnone +export RENOVATE_PR_HOURLY_LIMIT=0 +export RENOVATE_PR_CONCURRENT_LIMIT=0 + +# Skip the onboarding PR that Renovate normally creates for new repos +# https://docs.renovatebot.com/config-overview/#onboarding +export RENOVATE_ONBOARDING=false + +# Any Renovate config file in the cloned repository is ignored. Only +# the Renovate config file from the repo where the pipeline is running +# is used (yes, those are the same repo but the sources may be different). +# https://docs.renovatebot.com/self-hosted-configuration/#requireconfig +export RENOVATE_REQUIRE_CONFIG=ignored + +# Customize the PR body content. This removes some of the default +# sections that aren't relevant in a self-hosted config. +# https://docs.renovatebot.com/configuration-options/#prheader +# https://docs.renovatebot.com/configuration-options/#prbodynotes +# https://docs.renovatebot.com/configuration-options/#prbodytemplate +export RENOVATE_PR_HEADER='## Automated Dependency Update' +export RENOVATE_PR_BODY_NOTES='["This PR has been created automatically by the [.NET Renovate Bot](https://github.com/dotnet/arcade/blob/main/Documentation/Renovate.md) to update one or more dependencies in your repo. Please review the changes and merge the PR if everything looks good."]' +export RENOVATE_PR_BODY_TEMPLATE='{{{header}}}{{{table}}}{{{warnings}}}{{{notes}}}{{{changelogs}}}' + +# Extend the global config with additional presets +# https://docs.renovatebot.com/self-hosted-configuration/#globalextends +# Disable the Dependency Dashboard issue that tracks all updates +export RENOVATE_GLOBAL_EXTENDS='[":disableDependencyDashboard"]' + +# Allow all commands for post-upgrade commands. +export RENOVATE_ALLOWED_COMMANDS='[".*"]' diff --git a/eng/common/sdk-task.ps1 b/eng/common/sdk-task.ps1 index b64b66a6275..68119de603e 100644 --- a/eng/common/sdk-task.ps1 +++ b/eng/common/sdk-task.ps1 @@ -22,7 +22,7 @@ $warnAsError = if ($noWarnAsError) { $false } else { $true } function Print-Usage() { Write-Host "Common settings:" - Write-Host " -task Name of Arcade task (name of a project in SdkTasks directory of the Arcade SDK package)" + Write-Host " -task Name of Arcade task (name of a project in toolset directory of the Arcade SDK package)" Write-Host " -restore Restore dependencies" Write-Host " -verbosity Msbuild verbosity: q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic]" Write-Host " -help Print help and exit" @@ -66,20 +66,7 @@ try { if( $msbuildEngine -eq "vs") { # Ensure desktop MSBuild is available for sdk tasks. - if( -not ($GlobalJson.tools.PSObject.Properties.Name -contains "vs" )) { - $GlobalJson.tools | Add-Member -Name "vs" -Value (ConvertFrom-Json "{ `"version`": `"16.5`" }") -MemberType NoteProperty - } - if( -not ($GlobalJson.tools.PSObject.Properties.Name -match "xcopy-msbuild" )) { - $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "18.0.0" -MemberType NoteProperty - } - if ($GlobalJson.tools."xcopy-msbuild".Trim() -ine "none") { - $xcopyMSBuildToolsFolder = InitializeXCopyMSBuild $GlobalJson.tools."xcopy-msbuild" -install $true - } - if ($xcopyMSBuildToolsFolder -eq $null) { - throw 'Unable to get xcopy downloadable version of msbuild' - } - - $global:_MSBuildExe = "$($xcopyMSBuildToolsFolder)\MSBuild\Current\Bin\MSBuild.exe" + $global:_MSBuildExe = InitializeVisualStudioMSBuild } $taskProject = GetSdkTaskProject $task diff --git a/eng/common/sdk-task.sh b/eng/common/sdk-task.sh index 3270f83fa9a..1cf71bb2aea 100644 --- a/eng/common/sdk-task.sh +++ b/eng/common/sdk-task.sh @@ -2,7 +2,7 @@ show_usage() { echo "Common settings:" - echo " --task Name of Arcade task (name of a project in SdkTasks directory of the Arcade SDK package)" + echo " --task Name of Arcade task (name of a project in toolset directory of the Arcade SDK package)" echo " --restore Restore dependencies" echo " --verbosity Msbuild verbosity: q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic]" echo " --help Print help and exit" diff --git a/eng/common/template-guidance.md b/eng/common/template-guidance.md index e2b07a865f1..f772aa3d78f 100644 --- a/eng/common/template-guidance.md +++ b/eng/common/template-guidance.md @@ -71,7 +71,6 @@ eng\common\ source-build.yml (shim) source-index-stage1.yml (shim) jobs\ - codeql-build.yml (shim) jobs.yml (shim) source-build.yml (shim) post-build\ @@ -88,7 +87,6 @@ eng\common\ source-build.yml (shim) variables\ pool-providers.yml (logic + redirect) # templates/variables/pool-providers.yml will redirect to templates-official/variables/pool-providers.yml if you are running in the internal project - sdl-variables.yml (logic) core-templates\ job\ job.yml (logic) @@ -97,7 +95,6 @@ eng\common\ source-build.yml (logic) source-index-stage1.yml (logic) jobs\ - codeql-build.yml (logic) jobs.yml (logic) source-build.yml (logic) post-build\ diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index 977a2d4b103..0e281df8cae 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -34,6 +34,9 @@ # Configures warning treatment in msbuild. [bool]$warnAsError = if (Test-Path variable:warnAsError) { $warnAsError } else { $true } +# Specifies semi-colon delimited list of warning codes that should not be treated as errors. +[string]$warnNotAsError = if (Test-Path variable:warnNotAsError) { $warnNotAsError } else { '' } + # Specifies which msbuild engine to use for build: 'vs', 'dotnet' or unspecified (determined based on presence of tools.vs in global.json). [string]$msbuildEngine = if (Test-Path variable:msbuildEngine) { $msbuildEngine } else { $null } @@ -157,9 +160,6 @@ function InitializeDotNetCli([bool]$install, [bool]$createSdkLocationFile) { return $global:_DotNetInstallDir } - # Don't resolve runtime, shared framework, or SDK from other locations to ensure build determinism - $env:DOTNET_MULTILEVEL_LOOKUP=0 - # Disable first run since we do not need all ASP.NET packages restored. $env:DOTNET_NOLOGO=1 @@ -168,6 +168,12 @@ function InitializeDotNetCli([bool]$install, [bool]$createSdkLocationFile) { $env:DOTNET_CLI_TELEMETRY_OPTOUT=1 } + # Keep repo builds isolated from machine-installed SDK state and workload advertising. + # This avoids preview SDK builds picking up mismatched workloads on CI images. + $env:DOTNET_MULTILEVEL_LOOKUP = '0' + $env:DOTNET_SKIP_FIRST_TIME_EXPERIENCE = '1' + $env:DOTNET_CLI_WORKLOAD_UPDATE_NOTIFY_DISABLE = '1' + # Find the first path on %PATH% that contains the dotnet.exe if ($useInstalledDotNetCli -and (-not $globalJsonHasRuntimes) -and ($env:DOTNET_INSTALL_DIR -eq $null)) { $dotnetExecutable = GetExecutableFileName 'dotnet' @@ -185,7 +191,11 @@ function InitializeDotNetCli([bool]$install, [bool]$createSdkLocationFile) { if ((-not $globalJsonHasRuntimes) -and (-not [string]::IsNullOrEmpty($env:DOTNET_INSTALL_DIR)) -and (Test-Path(Join-Path $env:DOTNET_INSTALL_DIR "sdk\$dotnetSdkVersion"))) { $dotnetRoot = $env:DOTNET_INSTALL_DIR } else { - $dotnetRoot = Join-Path $RepoRoot '.dotnet' + if (-not [string]::IsNullOrEmpty($env:DOTNET_GLOBAL_INSTALL_DIR)) { + $dotnetRoot = $env:DOTNET_GLOBAL_INSTALL_DIR + } else { + $dotnetRoot = Join-Path $RepoRoot '.dotnet' + } if (-not (Test-Path(Join-Path $dotnetRoot "sdk\$dotnetSdkVersion"))) { if ($install) { @@ -225,8 +235,10 @@ function InitializeDotNetCli([bool]$install, [bool]$createSdkLocationFile) { # Make Sure that our bootstrapped dotnet cli is available in future steps of the Azure Pipelines build Write-PipelinePrependPath -Path $dotnetRoot - Write-PipelineSetVariable -Name 'DOTNET_MULTILEVEL_LOOKUP' -Value '0' Write-PipelineSetVariable -Name 'DOTNET_NOLOGO' -Value '1' + Write-PipelineSetVariable -Name 'DOTNET_MULTILEVEL_LOOKUP' -Value '0' + Write-PipelineSetVariable -Name 'DOTNET_SKIP_FIRST_TIME_EXPERIENCE' -Value '1' + Write-PipelineSetVariable -Name 'DOTNET_CLI_WORKLOAD_UPDATE_NOTIFY_DISABLE' -Value '1' return $global:_DotNetInstallDir = $dotnetRoot } @@ -299,6 +311,8 @@ function InstallDotNet([string] $dotnetRoot, $dotnetVersionLabel = "'sdk v$version'" + # For performance this check is duplicated in src/Microsoft.DotNet.Arcade.Sdk/src/InstallDotNetCore.cs + # if you are making changes here, consider if you need to make changes there as well. if ($runtime -ne '' -and $runtime -ne 'sdk') { $runtimePath = $dotnetRoot $runtimePath = $runtimePath + "\shared" @@ -374,12 +388,11 @@ function InstallDotNet([string] $dotnetRoot, # # 1. MSBuild from an active VS command prompt # 2. MSBuild from a compatible VS installation -# 3. MSBuild from the xcopy tool package # # Returns full path to msbuild.exe. # Throws on failure. # -function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = $null) { +function InitializeVisualStudioMSBuild([object]$vsRequirements = $null) { if (-not (IsWindowsPlatform)) { throw "Cannot initialize Visual Studio on non-Windows" } @@ -389,13 +402,7 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = } # Minimum VS version to require. - $vsMinVersionReqdStr = '17.7' - $vsMinVersionReqd = [Version]::new($vsMinVersionReqdStr) - - # If the version of msbuild is going to be xcopied, - # use this version. Version matches a package here: - # https://dev.azure.com/dnceng/public/_artifacts/feed/dotnet-eng/NuGet/Microsoft.DotNet.Arcade.MSBuild.Xcopy/versions/18.0.0 - $defaultXCopyMSBuildVersion = '18.0.0' + $vsMinVersionReqdStr = '18.0' if (!$vsRequirements) { if (Get-Member -InputObject $GlobalJson.tools -Name 'vs') { @@ -425,46 +432,16 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = } } - # Locate Visual Studio installation or download x-copy msbuild. + # Locate Visual Studio installation. $vsInfo = LocateVisualStudio $vsRequirements - if ($vsInfo -ne $null -and $env:ForceUseXCopyMSBuild -eq $null) { + if ($vsInfo -ne $null) { # Ensure vsInstallDir has a trailing slash $vsInstallDir = Join-Path $vsInfo.installationPath "\" $vsMajorVersion = $vsInfo.installationVersion.Split('.')[0] InitializeVisualStudioEnvironmentVariables $vsInstallDir $vsMajorVersion } else { - if (Get-Member -InputObject $GlobalJson.tools -Name 'xcopy-msbuild') { - $xcopyMSBuildVersion = $GlobalJson.tools.'xcopy-msbuild' - $vsMajorVersion = $xcopyMSBuildVersion.Split('.')[0] - } else { - #if vs version provided in global.json is incompatible (too low) then use the default version for xcopy msbuild download - if($vsMinVersion -lt $vsMinVersionReqd){ - Write-Host "Using xcopy-msbuild version of $defaultXCopyMSBuildVersion since VS version $vsMinVersionStr provided in global.json is not compatible" - $xcopyMSBuildVersion = $defaultXCopyMSBuildVersion - $vsMajorVersion = $xcopyMSBuildVersion.Split('.')[0] - } - else{ - # If the VS version IS compatible, look for an xcopy msbuild package - # with a version matching VS. - # Note: If this version does not exist, then an explicit version of xcopy msbuild - # can be specified in global.json. This will be required for pre-release versions of msbuild. - $vsMajorVersion = $vsMinVersion.Major - $vsMinorVersion = $vsMinVersion.Minor - $xcopyMSBuildVersion = "$vsMajorVersion.$vsMinorVersion.0" - } - } - - $vsInstallDir = $null - if ($xcopyMSBuildVersion.Trim() -ine "none") { - $vsInstallDir = InitializeXCopyMSBuild $xcopyMSBuildVersion $install - if ($vsInstallDir -eq $null) { - throw "Could not xcopy msbuild. Please check that package 'Microsoft.DotNet.Arcade.MSBuild.Xcopy @ $xcopyMSBuildVersion' exists on feed 'dotnet-eng'." - } - } - if ($vsInstallDir -eq $null) { - throw 'Unable to find Visual Studio that has required version and components installed' - } + throw 'Unable to find Visual Studio that has required version and components installed' } $msbuildVersionDir = if ([int]$vsMajorVersion -lt 16) { "$vsMajorVersion.0" } else { "Current" } @@ -491,38 +468,6 @@ function InitializeVisualStudioEnvironmentVariables([string] $vsInstallDir, [str } } -function InstallXCopyMSBuild([string]$packageVersion) { - return InitializeXCopyMSBuild $packageVersion -install $true -} - -function InitializeXCopyMSBuild([string]$packageVersion, [bool]$install) { - $packageName = 'Microsoft.DotNet.Arcade.MSBuild.Xcopy' - $packageDir = Join-Path $ToolsDir "msbuild\$packageVersion" - $packagePath = Join-Path $packageDir "$packageName.$packageVersion.nupkg" - - if (!(Test-Path $packageDir)) { - if (!$install) { - return $null - } - - Create-Directory $packageDir - - Write-Host "Downloading $packageName $packageVersion" - $ProgressPreference = 'SilentlyContinue' # Don't display the console progress UI - it's a huge perf hit - Retry({ - Invoke-WebRequest "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/flat2/$packageName/$packageVersion/$packageName.$packageVersion.nupkg" -UseBasicParsing -OutFile $packagePath - }) - - if (!(Test-Path $packagePath)) { - Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "See https://dev.azure.com/dnceng/internal/_wiki/wikis/DNCEng%20Services%20Wiki/1074/Updating-Microsoft.DotNet.Arcade.MSBuild.Xcopy-WAS-RoslynTools.MSBuild-(xcopy-msbuild)-generation?anchor=troubleshooting for help troubleshooting issues with XCopy MSBuild" - throw - } - Unzip $packagePath $packageDir - } - - return Join-Path $packageDir 'tools' -} - # # Locates Visual Studio instance that meets the minimal requirements specified by tools.vs object in global.json. # @@ -544,7 +489,6 @@ function LocateVisualStudio([object]$vsRequirements = $null){ if (Get-Member -InputObject $GlobalJson.tools -Name 'vswhere') { $vswhereVersion = $GlobalJson.tools.vswhere } else { - # keep this in sync with the VSWhereVersion in DefaultVersions.props $vswhereVersion = '3.1.7' } @@ -592,6 +536,11 @@ function LocateVisualStudio([object]$vsRequirements = $null){ return $null } + if ($null -eq $vsInfo -or $vsInfo.Count -eq 0) { + throw "No instance of Visual Studio meeting the requirements specified was found. Requirements: $($args -join ' ')" + return $null + } + # use first matching instance return $vsInfo[0] } @@ -627,7 +576,7 @@ function InitializeBuildTool() { $buildTool = @{ Path = $dotnetPath; Command = 'msbuild'; Tool = 'dotnet'; Framework = 'net' } } elseif ($msbuildEngine -eq "vs") { try { - $msbuildPath = InitializeVisualStudioMSBuild -install:$restore + $msbuildPath = InitializeVisualStudioMSBuild } catch { Write-PipelineTelemetryError -Category 'InitializeToolset' -Message $_ ExitWithExitCode 1 @@ -674,7 +623,13 @@ function GetNuGetPackageCachePath() { # Returns a full path to an Arcade SDK task project file. function GetSdkTaskProject([string]$taskName) { - return Join-Path (Split-Path (InitializeToolset) -Parent) "SdkTasks\$taskName.proj" + $toolsetDir = Split-Path (InitializeToolset) -Parent + $proj = Join-Path $toolsetDir "$taskName.proj" + if (Test-Path $proj) { + return $proj + } + + throw "Unable to find $taskName.proj in toolset at: $toolsetDir" } function InitializeNativeTools() { @@ -711,13 +666,18 @@ function InitializeToolset() { $nugetCache = GetNuGetPackageCachePath $toolsetVersion = Read-ArcadeSdkVersion - $toolsetLocationFile = Join-Path $ToolsetDir "$toolsetVersion.txt" + $toolsetToolsDir = Join-Path $ToolsetDir $toolsetVersion - if (Test-Path $toolsetLocationFile) { - $path = Get-Content $toolsetLocationFile -TotalCount 1 - if (Test-Path $path) { - return $global:_InitializeToolset = $path - } + # Check if the toolset has already been extracted + $toolsetBuildProj = $null + $buildProjPath = Join-Path $toolsetToolsDir 'Build.proj' + + if (Test-Path $buildProjPath) { + $toolsetBuildProj = $buildProjPath + } + + if ($toolsetBuildProj -ne $null) { + return $global:_InitializeToolset = $toolsetBuildProj } if (-not $restore) { @@ -725,21 +685,41 @@ function InitializeToolset() { ExitWithExitCode 1 } - $buildTool = InitializeBuildTool + $downloadArgs = @("package", "download", "Microsoft.DotNet.Arcade.Sdk@$toolsetVersion", "--verbosity", "minimal", "--prerelease", "--output", "$nugetCache") + $nugetConfig = $env:NUGET_CONFIG + if (-not $nugetConfig) { + # Search for any variation of nuget.config in the RepoRoot + $configFile = Get-ChildItem -Path $RepoRoot -File | Where-Object { $_.Name -ieq "nuget.config" } | Select-Object -First 1 + + if ($configFile) { + $nugetConfig = $configFile.FullName + } + } + + if ($nugetConfig) { + $downloadArgs += "--configfile" + $downloadArgs += $nugetConfig + } + DotNet @downloadArgs - $proj = Join-Path $ToolsetDir 'restore.proj' - $bl = if ($binaryLog) { '/bl:' + (Join-Path $LogDir 'ToolsetRestore.binlog') } else { '' } + $packageDir = Join-Path $nugetCache (Join-Path 'microsoft.dotnet.arcade.sdk' $toolsetVersion) + $packageToolsetDir = Join-Path $packageDir 'toolset' - '' | Set-Content $proj + if (!(Test-Path $packageToolsetDir)) { + Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Arcade SDK package does not contain a toolset or tools folder: $packageDir" + ExitWithExitCode 3 + } - MSBuild-Core $proj $bl /t:__WriteToolsetLocation /clp:ErrorsOnly`;NoSummary /p:__ToolsetLocationOutputFile=$toolsetLocationFile + New-Item -ItemType Directory -Path $toolsetToolsDir -Force | Out-Null + Copy-Item -Path "$packageToolsetDir\*" -Destination $toolsetToolsDir -Recurse -Force - $path = Get-Content $toolsetLocationFile -Encoding UTF8 -TotalCount 1 - if (!(Test-Path $path)) { - throw "Invalid toolset path: $path" + if (Test-Path $buildProjPath) { + $toolsetBuildProj = $buildProjPath + } else { + throw "Unable to find Build.proj in toolset at: $toolsetToolsDir" } - return $global:_InitializeToolset = $path + return $global:_InitializeToolset = $toolsetBuildProj } function ExitWithExitCode([int] $exitCode) { @@ -800,6 +780,40 @@ function MSBuild() { MSBuild-Core @args } +# +# Executes a dotnet command with arguments passed to the function. +# Terminates the script if the command fails. +# +function DotNet() { + $dotnetRoot = InitializeDotNetCli -install:$restore + $dotnetPath = Join-Path $dotnetRoot (GetExecutableFileName 'dotnet') + + $cmdArgs = "" + foreach ($arg in $args) { + if ($null -ne $arg -and $arg.Trim() -ne "") { + if ($arg.EndsWith('\')) { + $arg = $arg + "\" + } + $cmdArgs += " `"$arg`"" + } + } + + $env:ARCADE_BUILD_TOOL_COMMAND = "`"$dotnetPath`" $cmdArgs" + + $exitCode = Exec-Process $dotnetPath $cmdArgs + + if ($exitCode -ne 0) { + Write-Host "dotnet command failed with exit code $exitCode. Check errors above." -ForegroundColor Red + + if ($ci -and $env:SYSTEM_TEAMPROJECT -ne $null -and !$fromVMR) { + Write-PipelineSetResult -Result "Failed" -Message "dotnet command execution failed." + ExitWithExitCode 0 + } else { + ExitWithExitCode $exitCode + } + } +} + # # Executes msbuild (or 'dotnet msbuild') with arguments passed to the function. # The arguments are automatically quoted. @@ -824,6 +838,10 @@ function MSBuild-Core() { $cmdArgs = "$($buildTool.Command) /m /nologo /clp:Summary /v:$verbosity /nr:$nodeReuse /p:ContinuousIntegrationBuild=$ci" + if ($ci -and $buildTool.Tool -eq 'dotnet') { + $cmdArgs += ' /p:MSBuildEnableWorkloadResolver=false' + } + # Add -mt flag for MSBuild multithreaded mode if enabled via environment variable if ($env:MSBUILD_MT_ENABLED -eq "1") { $cmdArgs += ' -mt' @@ -836,6 +854,10 @@ function MSBuild-Core() { $cmdArgs += ' /p:TreatWarningsAsErrors=false' } + if ($warnAsError -and $warnNotAsError) { + $cmdArgs += " /warnnotaserror:$warnNotAsError /p:AdditionalWarningsNotAsErrors=$warnNotAsError" + } + foreach ($arg in $args) { if ($null -ne $arg -and $arg.Trim() -ne "") { if ($arg.EndsWith('\')) { @@ -930,6 +952,12 @@ Create-Directory $ToolsetDir Create-Directory $TempDir Create-Directory $LogDir +# Direct MSBuild crash diagnostics (MSB4166 failure.txt files) to a known location +# under artifacts/log so they are captured as build artifacts in CI. +if (-not $env:MSBUILDDEBUGPATH) { + $env:MSBUILDDEBUGPATH = Join-Path $LogDir 'MsbuildDebugLogs' +} + Write-PipelineSetVariable -Name 'Artifacts' -Value $ArtifactsDir Write-PipelineSetVariable -Name 'Artifacts.Toolset' -Value $ToolsetDir Write-PipelineSetVariable -Name 'Artifacts.Log' -Value $LogDir diff --git a/eng/common/tools.sh b/eng/common/tools.sh index 1b296f646c2..5ff37cfb700 100755 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -52,6 +52,9 @@ fi # Configures warning treatment in msbuild. warn_as_error=${warn_as_error:-true} +# Specifies semi-colon delimited list of warning codes that should not be treated as errors. +warn_not_as_error=${warn_not_as_error:-''} + # True to attempt using .NET Core already that meets requirements specified in global.json # installed on the machine instead of downloading one. use_installed_dotnet_cli=${use_installed_dotnet_cli:-true} @@ -115,9 +118,6 @@ function InitializeDotNetCli { local install=$1 - # Don't resolve runtime, shared framework, or SDK from other locations to ensure build determinism - export DOTNET_MULTILEVEL_LOOKUP=0 - # Disable first run since we want to control all package sources export DOTNET_NOLOGO=1 @@ -126,6 +126,12 @@ function InitializeDotNetCli { export DOTNET_CLI_TELEMETRY_OPTOUT=1 fi + # Keep repo builds isolated from machine-installed SDK state and workload advertising. + # This avoids preview SDK builds picking up mismatched workloads on CI images. + export DOTNET_MULTILEVEL_LOOKUP=0 + export DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 + export DOTNET_CLI_WORKLOAD_UPDATE_NOTIFY_DISABLE=1 + # LTTNG is the logging infrastructure used by Core CLR. Need this variable set # so it doesn't output warnings to the console. export LTTNG_HOME="$HOME" @@ -148,7 +154,11 @@ function InitializeDotNetCli { if [[ $global_json_has_runtimes == false && -n "${DOTNET_INSTALL_DIR:-}" && -d "$DOTNET_INSTALL_DIR/sdk/$dotnet_sdk_version" ]]; then dotnet_root="$DOTNET_INSTALL_DIR" else - dotnet_root="${repo_root}.dotnet" + if [[ -n "${DOTNET_GLOBAL_INSTALL_DIR:-}" ]]; then + dotnet_root="$DOTNET_GLOBAL_INSTALL_DIR" + else + dotnet_root="${repo_root}.dotnet" + fi export DOTNET_INSTALL_DIR="$dotnet_root" @@ -166,8 +176,10 @@ function InitializeDotNetCli { # build steps from using anything other than what we've downloaded. Write-PipelinePrependPath -path "$dotnet_root" - Write-PipelineSetVariable -name "DOTNET_MULTILEVEL_LOOKUP" -value "0" Write-PipelineSetVariable -name "DOTNET_NOLOGO" -value "1" + Write-PipelineSetVariable -name "DOTNET_MULTILEVEL_LOOKUP" -value "0" + Write-PipelineSetVariable -name "DOTNET_SKIP_FIRST_TIME_EXPERIENCE" -value "1" + Write-PipelineSetVariable -name "DOTNET_CLI_WORKLOAD_UPDATE_NOTIFY_DISABLE" -value "1" # return value _InitializeDotNetCli="$dotnet_root" @@ -188,6 +200,8 @@ function InstallDotNet { local version=$2 local runtime=$4 + # For performance this check is duplicated in src/Microsoft.DotNet.Arcade.Sdk/src/InstallDotNetCore.cs + # if you are making changes here, consider if you need to make changes there as well. local dotnetVersionLabel="'$runtime v$version'" if [[ -n "${4:-}" ]] && [ "$4" != 'sdk' ]; then runtimePath="$root" @@ -406,15 +420,18 @@ function InitializeToolset { ReadGlobalVersion "Microsoft.DotNet.Arcade.Sdk" local toolset_version=$_ReadGlobalVersion - local toolset_location_file="$toolset_dir/$toolset_version.txt" + local toolset_tools_dir="$toolset_dir/$toolset_version" - if [[ -a "$toolset_location_file" ]]; then - local path=`cat "$toolset_location_file"` - if [[ -a "$path" ]]; then - # return value - _InitializeToolset="$path" - return - fi + # Check if the toolset has already been extracted + local toolset_build_proj="" + if [[ -a "$toolset_tools_dir/Build.proj" ]]; then + toolset_build_proj="$toolset_tools_dir/Build.proj" + fi + + if [[ -n "$toolset_build_proj" ]]; then + # return value + _InitializeToolset="$toolset_build_proj" + return fi if [[ "$restore" != true ]]; then @@ -422,20 +439,37 @@ function InitializeToolset { ExitWithExitCode 2 fi - local proj="$toolset_dir/restore.proj" + local download_args=("package" "download" "Microsoft.DotNet.Arcade.Sdk@$toolset_version" "--verbosity" "minimal" "--prerelease" "--output" "$_GetNuGetPackageCachePath") + local nuget_config="${NUGET_CONFIG:-}" + if [[ -z "$nuget_config" ]]; then + # Search for any variation of nuget.config in the RepoRoot + local found_config + found_config=$(find "$repo_root" -maxdepth 1 -type f -iname "nuget.config" -print -quit) + + if [[ -n "$found_config" ]]; then + nuget_config="$found_config" + fi + fi - local bl="" - if [[ "$binary_log" == true ]]; then - bl="/bl:$log_dir/ToolsetRestore.binlog" + if [[ -n "$nuget_config" ]]; then + download_args+=("--configfile" "$nuget_config") fi + DotNet "${download_args[@]}" + + local package_dir="$_GetNuGetPackageCachePath/microsoft.dotnet.arcade.sdk/$toolset_version" - echo '' > "$proj" - MSBuild-Core "$proj" $bl /t:__WriteToolsetLocation /clp:ErrorsOnly\;NoSummary /p:__ToolsetLocationOutputFile="$toolset_location_file" + if [[ ! -d "$package_dir/toolset" ]]; then + Write-PipelineTelemetryError -category 'InitializeToolset' "Arcade SDK package does not contain a toolset folder: $package_dir" + ExitWithExitCode 3 + fi - local toolset_build_proj=`cat "$toolset_location_file"` + mkdir -p "$toolset_tools_dir" + cp -r "$package_dir/toolset/." "$toolset_tools_dir" - if [[ ! -a "$toolset_build_proj" ]]; then - Write-PipelineTelemetryError -category 'Build' "Invalid toolset path: $toolset_build_proj" + if [[ -a "$toolset_tools_dir/Build.proj" ]]; then + toolset_build_proj="$toolset_tools_dir/Build.proj" + else + Write-PipelineTelemetryError -category 'Build' "Unable to find Build.proj in toolset at: $toolset_tools_dir" ExitWithExitCode 3 fi @@ -457,6 +491,26 @@ function StopProcesses { return 0 } +function DotNet { + InitializeDotNetCli $restore + + local dotnet_path="$_InitializeDotNetCli/dotnet" + + export ARCADE_BUILD_TOOL_COMMAND="$dotnet_path $@" + + "$dotnet_path" "$@" || { + local exit_code=$? + echo "dotnet command failed with exit code $exit_code. Check errors above." + + if [[ "$ci" == true && -n ${SYSTEM_TEAMPROJECT:-} && "$from_vmr" != true ]]; then + Write-PipelineSetResult -result "Failed" -message "dotnet command execution failed." + ExitWithExitCode 0 + else + ExitWithExitCode $exit_code + fi + } +} + function MSBuild { local args=( "$@" ) if [[ "$pipelines_log" == true ]]; then @@ -532,7 +586,17 @@ function MSBuild-Core { mt_switch="-mt" fi - RunBuildTool "$_InitializeBuildToolCommand" /m /nologo /clp:Summary /v:$verbosity /nr:$node_reuse $warnaserror_switch $mt_switch /p:TreatWarningsAsErrors=$warn_as_error /p:ContinuousIntegrationBuild=$ci "$@" + local warnnotaserror_switch="" + if [[ -n "$warn_not_as_error" && "$warn_as_error" == true ]]; then + warnnotaserror_switch="/warnnotaserror:$warn_not_as_error /p:AdditionalWarningsNotAsErrors=$warn_not_as_error" + fi + + local workload_resolver_switch="" + if [[ "$ci" == true && -n "${_InitializeBuildToolCommand:-}" ]]; then + workload_resolver_switch="/p:MSBuildEnableWorkloadResolver=false" + fi + + RunBuildTool "$_InitializeBuildToolCommand" /m /nologo /clp:Summary /v:$verbosity /nr:$node_reuse $warnaserror_switch $mt_switch $warnnotaserror_switch $workload_resolver_switch /p:TreatWarningsAsErrors=$warn_as_error /p:ContinuousIntegrationBuild=$ci "$@" } function GetDarc { @@ -549,8 +613,17 @@ function GetDarc { # Returns a full path to an Arcade SDK task project file. function GetSdkTaskProject { - taskName=$1 - echo "$(dirname $_InitializeToolset)/SdkTasks/$taskName.proj" + local taskName=$1 + local toolsetDir + toolsetDir="$(dirname "$_InitializeToolset")" + local proj="$toolsetDir/$taskName.proj" + if [[ -a "$proj" ]]; then + echo "$proj" + return + fi + + Write-PipelineTelemetryError -category 'Build' "Unable to find $taskName.proj in toolset at: $toolsetDir" + ExitWithExitCode 3 } ResolvePath "${BASH_SOURCE[0]}" @@ -588,6 +661,12 @@ mkdir -p "$toolset_dir" mkdir -p "$temp_dir" mkdir -p "$log_dir" +# Direct MSBuild crash diagnostics (MSB4166 failure.txt files) to a known location +# under artifacts/log so they are captured as build artifacts in CI. +if [[ -z "${MSBUILDDEBUGPATH:-}" ]]; then + export MSBUILDDEBUGPATH="$log_dir/MsbuildDebugLogs" +fi + Write-PipelineSetVariable -name "Artifacts" -value "$artifacts_dir" Write-PipelineSetVariable -name "Artifacts.Toolset" -value "$toolset_dir" Write-PipelineSetVariable -name "Artifacts.Log" -value "$log_dir" diff --git a/global.json b/global.json index fee65ff38ab..e6d41c7e5cc 100644 --- a/global.json +++ b/global.json @@ -1,6 +1,6 @@ { "sdk": { - "version": "10.0.202", + "version": "11.0.100-preview.5.26227.104", "allowPrerelease": true, "paths": [ ".dotnet", @@ -12,7 +12,7 @@ "runner": "Microsoft.Testing.Platform" }, "tools": { - "dotnet": "10.0.202", + "dotnet": "11.0.100-preview.5.26227.104", "vs": { "version": "18.0", "components": [ @@ -22,7 +22,7 @@ "xcopy-msbuild": "18.0.0" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "10.0.0-beta.26269.2", + "Microsoft.DotNet.Arcade.Sdk": "11.0.0-beta.26265.4", "Microsoft.DotNet.Helix.Sdk": "8.0.0-beta.23255.2" } }