diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml
index 6aaaafef5..acbaeb523 100644
--- a/eng/Version.Details.xml
+++ b/eng/Version.Details.xml
@@ -3,13 +3,13 @@
-
+
https://github.com/dotnet/arcade
- 2117ebaa7336feccd2361eb01ce33c243249bce0
+ e2e96a9587219f6619d8a1cef7581bdb358f0909
-
+
https://github.com/dotnet/arcade
- 2117ebaa7336feccd2361eb01ce33c243249bce0
+ e2e96a9587219f6619d8a1cef7581bdb358f0909
diff --git a/eng/common/AGENTS.md b/eng/common/AGENTS.md
new file mode 100644
index 000000000..a5ed8f729
--- /dev/null
+++ b/eng/common/AGENTS.md
@@ -0,0 +1,5 @@
+# `eng/common`
+
+Files under `eng/common` come from [Arcade](https://github.com/dotnet/arcade).
+Edits in `eng/common` will be overwritten by automation unless the changes are made directly in the Arcade repository.
+For more information, see the [Arcade documentation](https://github.com/dotnet/arcade/tree/main/Documentation).
diff --git a/eng/common/CIBuild.cmd b/eng/common/CIBuild.cmd
index 56c2f25ac..ac1f72bf9 100644
--- a/eng/common/CIBuild.cmd
+++ b/eng/common/CIBuild.cmd
@@ -1,2 +1,2 @@
@echo off
-powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0Build.ps1""" -restore -build -test -sign -pack -publish -ci %*"
\ No newline at end of file
+powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0Build.ps1""" -restore -build -test -sign -pack -publish -ci %*"
diff --git a/eng/common/Get-GitHubAppToken.ps1 b/eng/common/Get-GitHubAppToken.ps1
new file mode 100644
index 000000000..9c7e3dcd6
--- /dev/null
+++ b/eng/common/Get-GitHubAppToken.ps1
@@ -0,0 +1,164 @@
+# Mints a short-lived GitHub App installation access token by signing a JWT
+# with a private key stored in Azure Key Vault (RSA, RS256). The signed JWT is
+# exchanged with the GitHub API for a token scoped to a single installation.
+#
+# Requirements:
+# - A GitHub App whose private key has been uploaded into Key Vault as an RSA
+# key (the PEM converted to a Key Vault *key*, NOT stored as a secret).
+# - The caller (the federated Azure service connection used to run this script)
+# must have the `Key Vault Crypto User` role (or at minimum the `Sign`
+# action) on that key.
+# - The App must be installed on the target organization/account
+# (`InstallationOwner`) with the permissions/repositories it needs.
+#
+# Installation tokens (ghs_*) are exempt from the enterprise classic-PAT
+# lifetime policy, which is why this replaces the long-lived PAT.
+
+[CmdletBinding()]
+param(
+ # Name of the Key Vault that holds the GitHub App's RSA signing key.
+ [Parameter(Mandatory = $true)]
+ [string] $KeyVaultName,
+
+ # Name of the RSA key inside the Key Vault (the App's private key).
+ [Parameter(Mandatory = $true)]
+ [string] $KeyName,
+
+ # The GitHub App's Client ID (the value to put in the `iss` JWT claim).
+ [Parameter(Mandatory = $true)]
+ [string] $AppClientId,
+
+ # Login of the organization or user account whose installation we should
+ # mint the token for (e.g. `dotnet`, `microsoft`).
+ [Parameter(Mandatory = $true)]
+ [string] $InstallationOwner,
+
+ # Optional Azure DevOps pipeline variable name to set with the installation
+ # token (marked as a secret). When not specified, the token is written to
+ # stdout instead.
+ [Parameter(Mandatory = $false)]
+ [string] $OutputVariableName
+)
+
+$ErrorActionPreference = 'Stop'
+$PSNativeCommandUseErrorActionPreference = $true
+
+. $PSScriptRoot\pipeline-logging-functions.ps1
+
+function ConvertTo-Base64Url([byte[]] $bytes) {
+ return [Convert]::ToBase64String($bytes).TrimEnd('=').Replace('+', '-').Replace('/', '_')
+}
+
+# Build JWT header and payload. Use [ordered] hashtables so JSON
+# serialization is deterministic.
+$jwtHeader = [ordered]@{
+ alg = 'RS256'
+ typ = 'JWT'
+}
+$now = [System.DateTimeOffset]::UtcNow
+$jwtPayload = [ordered]@{
+ iat = $now.AddMinutes(-1).ToUnixTimeSeconds()
+ exp = $now.AddMinutes(5).ToUnixTimeSeconds()
+ iss = $AppClientId
+}
+
+$headerEncoded = ConvertTo-Base64Url ([System.Text.Encoding]::UTF8.GetBytes(($jwtHeader | ConvertTo-Json -Compress)))
+$payloadEncoded = ConvertTo-Base64Url ([System.Text.Encoding]::UTF8.GetBytes(($jwtPayload | ConvertTo-Json -Compress)))
+$signingInput = "$headerEncoded.$payloadEncoded"
+
+# Key Vault `sign` expects the *digest* (base64), not the raw bytes.
+$sha256 = [System.Security.Cryptography.SHA256]::Create()
+$digestBytes = $sha256.ComputeHash([System.Text.Encoding]::UTF8.GetBytes($signingInput))
+$digestBase64 = [Convert]::ToBase64String($digestBytes)
+
+Write-Host "Signing JWT with key '$KeyName' in vault '$KeyVaultName'..."
+$previousNativeCommandErrorPreference = $PSNativeCommandUseErrorActionPreference
+try {
+ # Azure CLI can emit non-fatal Python warnings to stderr even when signing succeeds.
+ # Use the exit code to determine success for this invocation.
+ $PSNativeCommandUseErrorActionPreference = $false
+ $signatureBase64 = az keyvault key sign `
+ --vault-name $KeyVaultName `
+ --name $KeyName `
+ --algorithm RS256 `
+ --digest $digestBase64 `
+ --query signature `
+ --output tsv `
+ --only-show-errors
+ $signExitCode = $LASTEXITCODE
+}
+catch {
+ Write-PipelineTelemetryError -Category 'Build' -Message "Failed to sign the JWT via Key Vault (key '$KeyName', vault '$KeyVaultName'): $_. Verify the service connection identity has the 'Key Vault Crypto User' role (Sign action) on the key."
+ exit 1
+}
+finally {
+ $PSNativeCommandUseErrorActionPreference = $previousNativeCommandErrorPreference
+}
+if ($signExitCode -ne 0 -or [string]::IsNullOrWhiteSpace($signatureBase64)) {
+ Write-PipelineTelemetryError -Category 'Build' -Message "'az keyvault key sign' exited with code $signExitCode for key '$KeyName' in vault '$KeyVaultName'. Verify the service connection identity has the 'Key Vault Crypto User' role (Sign action) on the key."
+ exit 1
+}
+$signatureUrl = $signatureBase64.Trim().TrimEnd('=').Replace('+', '-').Replace('/', '_')
+$jwt = "$signingInput.$signatureUrl"
+
+$headers = @{
+ Authorization = "Bearer $jwt"
+ 'X-GitHub-Api-Version' = '2022-11-28'
+ Accept = 'application/vnd.github+json'
+ 'User-Agent' = 'dotnet-arcade-onelocbuild'
+}
+
+Write-Host "Looking up installation for '$InstallationOwner'..."
+try {
+ $installations = @()
+ $page = 1
+ do {
+ # Assign the response before wrapping it in @(). PowerShell otherwise
+ # preserves a top-level JSON array as one nested pipeline object.
+ $pageResponse = Invoke-RestMethod `
+ -Uri "https://api.github.com/app/installations?per_page=100&page=$page" `
+ -Headers $headers `
+ -Method Get
+ $pageInstallations = @($pageResponse)
+ $installations += $pageInstallations
+ $page++
+ } while ($pageInstallations.Count -eq 100)
+}
+catch {
+ Write-PipelineTelemetryError -Category 'Build' -Message "Failed to list GitHub App installations: $_. The signed JWT may be invalid or the App's Client ID ('$AppClientId') may be incorrect."
+ exit 1
+}
+$matchingInstallations = @($installations | Where-Object { $_.account.login -ieq $InstallationOwner })
+if ($matchingInstallations.Count -eq 0) {
+ $found = ($installations | ForEach-Object { $_.account.login }) -join ', '
+ Write-PipelineTelemetryError -Category 'Build' -Message "No installation found for '$InstallationOwner'. App is installed on: $found"
+ exit 1
+}
+if ($matchingInstallations.Count -ne 1) {
+ $matchingIds = ($matchingInstallations | ForEach-Object { $_.id }) -join ', '
+ Write-PipelineTelemetryError -Category 'Build' -Message "Found multiple installations for '$InstallationOwner': $matchingIds"
+ exit 1
+}
+$installation = $matchingInstallations[0]
+Write-Host "Using installation $($installation.id) for '$($installation.account.login)'."
+
+try {
+ $tokenResponse = Invoke-RestMethod `
+ -Uri "https://api.github.com/app/installations/$($installation.id)/access_tokens" `
+ -Headers $headers `
+ -Method Post `
+ -ContentType 'application/json'
+}
+catch {
+ Write-PipelineTelemetryError -Category 'Build' -Message "Failed to mint an installation access token for '$InstallationOwner' (installation $($installation.id)): $_"
+ exit 1
+}
+
+Write-Host "Got installation token for '$InstallationOwner' (expires $($tokenResponse.expires_at))."
+if ($OutputVariableName) {
+ Write-Host "Setting pipeline variable '$OutputVariableName'."
+ Write-Host "##vso[task.setvariable variable=$OutputVariableName;issecret=true]$($tokenResponse.token)"
+}
+else {
+ Write-Host $tokenResponse.token -ForegroundColor Green
+}
diff --git a/eng/common/SetupNugetSources.ps1 b/eng/common/SetupNugetSources.ps1
index 5db4ad71e..9efd17273 100644
--- a/eng/common/SetupNugetSources.ps1
+++ b/eng/common/SetupNugetSources.ps1
@@ -7,13 +7,17 @@
# See example call for this script below.
#
# - task: PowerShell@2
-# displayName: Setup Private Feeds Credentials
+# displayName: Setup internal Feeds Credentials
# condition: eq(variables['Agent.OS'], 'Windows_NT')
# inputs:
-# filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1
-# arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token
+# filePath: $(System.DefaultWorkingDirectory)/eng/common/SetupNugetSources.ps1
+# arguments: -ConfigFile $(System.DefaultWorkingDirectory)/NuGet.config
# env:
-# Token: $(dn-bot-dnceng-artifact-feeds-rw)
+# Token: $(InternalFeedToken)
+#
+# Note: This logic is abstracted into enable-internal-sources.yml, which uses
+# NuGetAuthenticate or a WIF-backed service connection. Prefer that template
+# over calling this script directly.
#
# Note that the NuGetAuthenticate task should be called after SetupNugetSources.
# This ensures that:
@@ -25,40 +29,58 @@
[CmdletBinding()]
param (
[Parameter(Mandatory = $true)][string]$ConfigFile,
- $Password
+ # Keep the legacy name as an alias while callers migrate secrets to the Token environment variable.
+ [Alias("Password")]$Credential
)
$ErrorActionPreference = "Stop"
Set-StrictMode -Version 2.0
[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
+$feedCredential = if ($env:Token) { $env:Token } else { $Credential }
+
+# This script only consumes helper functions from tools.ps1 to configure NuGet feeds.
+# Skip importing configure-toolset.ps1 so that repo-specific toolset setup (e.g. acquiring
+# a bootstrap SDK) is not triggered as a side effect of feed configuration.
+$disableConfigureToolsetImport = $true
. $PSScriptRoot\tools.ps1
+# Adds or enables the package source with the given name
+function AddOrEnablePackageSource($sources, $disabledPackageSources, $SourceName, $SourceEndPoint, $creds, $Username, $credential) {
+ if ($disabledPackageSources -eq $null -or -not (EnableInternalPackageSource -DisabledPackageSources $disabledPackageSources -Creds $creds -PackageSourceName $SourceName -Credential $credential)) {
+ AddPackageSource -Sources $sources -SourceName $SourceName -SourceEndPoint $SourceEndPoint -Creds $creds -Username $Username -credential $credential
+ }
+}
+
# Add source entry to PackageSources
-function AddPackageSource($sources, $SourceName, $SourceEndPoint, $creds, $Username, $pwd) {
+function AddPackageSource($sources, $SourceName, $SourceEndPoint, $creds, $Username, $credential) {
$packageSource = $sources.SelectSingleNode("add[@key='$SourceName']")
if ($packageSource -eq $null)
{
+ Write-Host "Adding package source $SourceName"
+
$packageSource = $doc.CreateElement("add")
$packageSource.SetAttribute("key", $SourceName)
$packageSource.SetAttribute("value", $SourceEndPoint)
$sources.AppendChild($packageSource) | Out-Null
}
else {
- Write-Host "Package source $SourceName already present."
+ Write-Host "Package source $SourceName already present and enabled."
}
- AddCredential -Creds $creds -Source $SourceName -Username $Username -pwd $pwd
+ AddCredential -Creds $creds -Source $SourceName -Username $Username -credential $credential
}
# Add a credential node for the specified source
-function AddCredential($creds, $source, $username, $pwd) {
+function AddCredential($creds, $source, $username, $credential) {
# If no cred supplied, don't do anything.
- if (!$pwd) {
+ if (!$credential) {
return;
}
+ Write-Host "Inserting credential for feed: " $source
+
# Looks for credential configuration for the given SourceName. Create it if none is found.
$sourceElement = $creds.SelectSingleNode($Source)
if ($sourceElement -eq $null)
@@ -88,27 +110,30 @@ function AddCredential($creds, $source, $username, $pwd) {
$sourceElement.AppendChild($passwordElement) | Out-Null
}
- $passwordElement.SetAttribute("value", $pwd)
+ $passwordElement.SetAttribute("value", $credential)
}
-function InsertMaestroPrivateFeedCredentials($Sources, $Creds, $Username, $pwd) {
- $maestroPrivateSources = $Sources.SelectNodes("add[contains(@key,'darc-int')]")
-
- Write-Host "Inserting credentials for $($maestroPrivateSources.Count) Maestro's private feeds."
-
- ForEach ($PackageSource in $maestroPrivateSources) {
- Write-Host "`tInserting credential for Maestro's feed:" $PackageSource.Key
- AddCredential -Creds $creds -Source $PackageSource.Key -Username $Username -pwd $pwd
+# Enable all darc-int package sources.
+function EnableMaestroInternalPackageSources($DisabledPackageSources, $Creds, $Credential) {
+ $maestroInternalSources = $DisabledPackageSources.SelectNodes("add[contains(@key,'darc-int')]")
+ ForEach ($DisabledPackageSource in $maestroInternalSources) {
+ EnableInternalPackageSource -DisabledPackageSources $DisabledPackageSources -Creds $Creds -PackageSourceName $DisabledPackageSource.key -Credential $Credential
}
}
-function EnablePrivatePackageSources($DisabledPackageSources) {
- $maestroPrivateSources = $DisabledPackageSources.SelectNodes("add[contains(@key,'darc-int')]")
- ForEach ($DisabledPackageSource in $maestroPrivateSources) {
- Write-Host "`tEnsuring private source '$($DisabledPackageSource.key)' is enabled by deleting it from disabledPackageSource"
+# Enables an internal package source by name, if found. Returns true if the package source was found and enabled, false otherwise.
+function EnableInternalPackageSource($DisabledPackageSources, $Creds, $PackageSourceName, $Credential) {
+ $DisabledPackageSource = $DisabledPackageSources.SelectSingleNode("add[@key='$PackageSourceName']")
+ if ($DisabledPackageSource) {
+ Write-Host "Enabling internal source '$($DisabledPackageSource.key)'."
+
# Due to https://github.com/NuGet/Home/issues/10291, we must actually remove the disabled entries
$DisabledPackageSources.RemoveChild($DisabledPackageSource)
+
+ AddCredential -Creds $creds -Source $DisabledPackageSource.Key -Username $userName -credential $credential
+ return $true
}
+ return $false
}
if (!(Test-Path $ConfigFile -PathType Leaf)) {
@@ -121,15 +146,17 @@ $doc = New-Object System.Xml.XmlDocument
$filename = (Get-Item $ConfigFile).FullName
$doc.Load($filename)
-# Get reference to or create one if none exist already
+# Get reference to - fail if none exist
$sources = $doc.DocumentElement.SelectSingleNode("packageSources")
if ($sources -eq $null) {
- $sources = $doc.CreateElement("packageSources")
- $doc.DocumentElement.AppendChild($sources) | Out-Null
+ Write-PipelineTelemetryError -Category 'Build' -Message "Eng/common/SetupNugetSources.ps1 returned a non-zero exit code. NuGet config file must contain a packageSources section: $ConfigFile"
+ ExitWithExitCode 1
}
$creds = $null
-if ($Password) {
+$feedSuffix = "v3/index.json"
+if ($feedCredential) {
+ $feedSuffix = "v2"
# Looks for a node. Create it if none is found.
$creds = $doc.DocumentElement.SelectSingleNode("packageSourceCredentials")
if ($creds -eq $null) {
@@ -138,33 +165,22 @@ if ($Password) {
}
}
+$userName = "dn-bot"
+
# Check for disabledPackageSources; we'll enable any darc-int ones we find there
$disabledSources = $doc.DocumentElement.SelectSingleNode("disabledPackageSources")
if ($disabledSources -ne $null) {
Write-Host "Checking for any darc-int disabled package sources in the disabledPackageSources node"
- EnablePrivatePackageSources -DisabledPackageSources $disabledSources
+ EnableMaestroInternalPackageSources -DisabledPackageSources $disabledSources -Creds $creds -Credential $feedCredential
}
-
-$userName = "dn-bot"
-
-# Insert credential nodes for Maestro's private feeds
-InsertMaestroPrivateFeedCredentials -Sources $sources -Creds $creds -Username $userName -pwd $Password
-
-# 3.1 uses a different feed url format so it's handled differently here
-$dotnet31Source = $sources.SelectSingleNode("add[@key='dotnet3.1']")
-if ($dotnet31Source -ne $null) {
- AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v2" -Creds $creds -Username $userName -pwd $Password
- AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2" -Creds $creds -Username $userName -pwd $Password
-}
-
-$dotnetVersions = @('5','6','7','8','9')
+$dotnetVersions = @('5','6','7','8','9','10','11')
foreach ($dotnetVersion in $dotnetVersions) {
$feedPrefix = "dotnet" + $dotnetVersion;
$dotnetSource = $sources.SelectSingleNode("add[@key='$feedPrefix']")
if ($dotnetSource -ne $null) {
- AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal/nuget/v2" -Creds $creds -Username $userName -pwd $Password
- AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal-transport/nuget/v2" -Creds $creds -Username $userName -pwd $Password
+ AddOrEnablePackageSource -Sources $sources -DisabledPackageSources $disabledSources -SourceName "$feedPrefix-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal/nuget/$feedSuffix" -Creds $creds -Username $userName -credential $feedCredential
+ AddOrEnablePackageSource -Sources $sources -DisabledPackageSources $disabledSources -SourceName "$feedPrefix-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal-transport/nuget/$feedSuffix" -Creds $creds -Username $userName -credential $feedCredential
}
}
diff --git a/eng/common/SetupNugetSources.sh b/eng/common/SetupNugetSources.sh
index 4604b61b0..d4c66a98e 100644
--- a/eng/common/SetupNugetSources.sh
+++ b/eng/common/SetupNugetSources.sh
@@ -11,8 +11,8 @@
# - task: Bash@3
# displayName: Setup Internal Feeds
# inputs:
-# filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh
-# arguments: $(Build.SourcesDirectory)/NuGet.config
+# filePath: $(System.DefaultWorkingDirectory)/eng/common/SetupNugetSources.sh
+# arguments: $(System.DefaultWorkingDirectory)/NuGet.config
# condition: ne(variables['Agent.OS'], 'Windows_NT')
# - task: NuGetAuthenticate@1
#
@@ -24,7 +24,9 @@
# This logic is also abstracted into enable-internal-sources.yml.
ConfigFile=$1
-CredToken=$2
+# Prefer the environment variable so credentials do not appear in process arguments.
+# Retain the positional argument as a compatibility fallback for existing callers.
+CredToken=${Token:-$2}
NL='\n'
TB=' '
@@ -40,6 +42,11 @@ while [[ -h "$source" ]]; do
done
scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
+# This script only consumes helper functions from tools.sh to configure NuGet feeds.
+# Skip importing configure-toolset.sh so that repo-specific toolset setup (e.g. acquiring
+# a bootstrap SDK) is not triggered as a side effect of feed configuration.
+disable_configure_toolset_import=1
+
. "$scriptroot/tools.sh"
if [ ! -f "$ConfigFile" ]; then
@@ -52,78 +59,124 @@ if [[ `uname -s` == "Darwin" ]]; then
TB=''
fi
-# Ensure there is a ... section.
-grep -i "" $ConfigFile
-if [ "$?" != "0" ]; then
- echo "Adding ... section."
- ConfigNodeHeader=""
- PackageSourcesTemplate="${TB}${NL}${TB}"
+# Enables an internal package source by name, if found. Returns 0 if found and enabled, 1 if not found.
+EnableInternalPackageSource() {
+ local PackageSourceName="$1"
+
+ # Check if disabledPackageSources section exists
+ grep -i "" "$ConfigFile" > /dev/null
+ if [ "$?" != "0" ]; then
+ return 1 # No disabled sources section
+ fi
+
+ # Check if this source name is disabled
+ grep -i " /dev/null
+ if [ "$?" == "0" ]; then
+ echo "Enabling internal source '$PackageSourceName'."
+ # Remove the disabled entry (including any surrounding comments or whitespace on the same line)
+ sed -i.bak "//d" "$ConfigFile"
+
+ # Add the source name to PackageSources for credential handling
+ PackageSources+=("$PackageSourceName")
+ return 0 # Found and enabled
+ fi
+
+ return 1 # Not found in disabled sources
+}
+
+# Add source entry to PackageSources
+AddPackageSource() {
+ local SourceName="$1"
+ local SourceEndPoint="$2"
+
+ # Check if source already exists
+ grep -i " /dev/null
+ if [ "$?" == "0" ]; then
+ echo "Package source $SourceName already present and enabled."
+ PackageSources+=("$SourceName")
+ return
+ fi
+
+ echo "Adding package source $SourceName"
+ PackageSourcesNodeFooter=""
+ PackageSourceTemplate="${TB}"
+
+ sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" "$ConfigFile"
+ PackageSources+=("$SourceName")
+}
+
+# Adds or enables the package source with the given name
+AddOrEnablePackageSource() {
+ local SourceName="$1"
+ local SourceEndPoint="$2"
+
+ # Try to enable if disabled, if not found then add new source
+ EnableInternalPackageSource "$SourceName"
+ if [ "$?" != "0" ]; then
+ AddPackageSource "$SourceName" "$SourceEndPoint"
+ fi
+}
- sed -i.bak "s|$ConfigNodeHeader|$ConfigNodeHeader${NL}$PackageSourcesTemplate|" $ConfigFile
-fi
+# Enable all darc-int package sources
+EnableMaestroInternalPackageSources() {
+ # Check if disabledPackageSources section exists
+ grep -i "" "$ConfigFile" > /dev/null
+ if [ "$?" != "0" ]; then
+ return # No disabled sources section
+ fi
+
+ # Find all darc-int disabled sources
+ local DisabledDarcIntSources=()
+ DisabledDarcIntSources+=$(grep -oh '"darc-int-[^"]*" value="true"' "$ConfigFile" | tr -d '"')
+
+ for DisabledSourceName in ${DisabledDarcIntSources[@]} ; do
+ if [[ $DisabledSourceName == darc-int* ]]; then
+ EnableInternalPackageSource "$DisabledSourceName"
+ fi
+ done
+}
-# Ensure there is a ... section.
-grep -i "" $ConfigFile
+# Ensure there is a ... section.
+grep -i "" $ConfigFile
if [ "$?" != "0" ]; then
- echo "Adding ... section."
-
- PackageSourcesNodeFooter=""
- PackageSourceCredentialsTemplate="${TB}${NL}${TB}"
-
- sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourcesNodeFooter${NL}$PackageSourceCredentialsTemplate|" $ConfigFile
+ Write-PipelineTelemetryError -Category 'Build' "Error: Eng/common/SetupNugetSources.sh returned a non-zero exit code. NuGet config file must contain a packageSources section: $ConfigFile"
+ ExitWithExitCode 1
fi
PackageSources=()
-# Ensure dotnet3.1-internal and dotnet3.1-internal-transport are in the packageSources if the public dotnet3.1 feeds are present
-grep -i "... section.
+ grep -i "" $ConfigFile
if [ "$?" != "0" ]; then
- echo "Adding dotnet3.1-internal to the packageSources."
- PackageSourcesNodeFooter=""
- PackageSourceTemplate="${TB}"
-
- sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile
- fi
- PackageSources+=('dotnet3.1-internal')
+ echo "Adding ... section."
- grep -i "" $ConfigFile
- if [ "$?" != "0" ]; then
- echo "Adding dotnet3.1-internal-transport to the packageSources."
PackageSourcesNodeFooter=""
- PackageSourceTemplate="${TB}"
+ PackageSourceCredentialsTemplate="${TB}${NL}${TB}"
- sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile
+ sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourcesNodeFooter${NL}$PackageSourceCredentialsTemplate|" $ConfigFile
fi
- PackageSources+=('dotnet3.1-internal-transport')
fi
-DotNetVersions=('5' '6' '7' '8' '9')
+# Check for disabledPackageSources; we'll enable any darc-int ones we find there
+grep -i "" $ConfigFile > /dev/null
+if [ "$?" == "0" ]; then
+ echo "Checking for any darc-int disabled package sources in the disabledPackageSources node"
+ EnableMaestroInternalPackageSources
+fi
+
+DotNetVersions=('5' '6' '7' '8' '9' '10' '11')
for DotNetVersion in ${DotNetVersions[@]} ; do
FeedPrefix="dotnet${DotNetVersion}";
- grep -i " /dev/null
if [ "$?" == "0" ]; then
- grep -i ""
-
- sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile
- fi
- PackageSources+=("$FeedPrefix-internal")
-
- grep -i "" $ConfigFile
- if [ "$?" != "0" ]; then
- echo "Adding $FeedPrefix-internal-transport to the packageSources."
- PackageSourcesNodeFooter=""
- PackageSourceTemplate="${TB}"
-
- sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile
- fi
- PackageSources+=("$FeedPrefix-internal-transport")
+ AddOrEnablePackageSource "$FeedPrefix-internal" "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$FeedPrefix-internal/nuget/$FeedSuffix"
+ AddOrEnablePackageSource "$FeedPrefix-internal-transport" "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$FeedPrefix-internal-transport/nuget/$FeedSuffix"
fi
done
@@ -139,29 +192,12 @@ if [ "$CredToken" ]; then
# Check if there is no existing credential for this FeedName
grep -i "<$FeedName>" $ConfigFile
if [ "$?" != "0" ]; then
- echo "Adding credentials for $FeedName."
+ echo " Inserting credential for feed: $FeedName"
PackageSourceCredentialsNodeFooter=""
- NewCredential="${TB}${TB}<$FeedName>${NL}${NL}${NL}"
+ NewCredential="${TB}${TB}<$FeedName>${NL}${TB}${NL}${TB}${TB}${NL}${TB}${TB}"
sed -i.bak "s|$PackageSourceCredentialsNodeFooter|$NewCredential${NL}$PackageSourceCredentialsNodeFooter|" $ConfigFile
fi
done
fi
-
-# Re-enable any entries in disabledPackageSources where the feed name contains darc-int
-grep -i "" $ConfigFile
-if [ "$?" == "0" ]; then
- DisabledDarcIntSources=()
- echo "Re-enabling any disabled \"darc-int\" package sources in $ConfigFile"
- DisabledDarcIntSources+=$(grep -oh '"darc-int-[^"]*" value="true"' $ConfigFile | tr -d '"')
- for DisabledSourceName in ${DisabledDarcIntSources[@]} ; do
- if [[ $DisabledSourceName == darc-int* ]]
- then
- OldDisableValue=""
- NewDisableValue=""
- sed -i.bak "s|$OldDisableValue|$NewDisableValue|" $ConfigFile
- echo "Neutralized disablePackageSources entry for '$DisabledSourceName'"
- fi
- done
-fi
diff --git a/eng/common/build.ps1 b/eng/common/build.ps1
index 438f9920c..fee2f8399 100644
--- a/eng/common/build.ps1
+++ b/eng/common/build.ps1
@@ -6,7 +6,10 @@ Param(
[string][Alias('v')]$verbosity = "minimal",
[string] $msbuildEngine = $null,
[bool] $warnAsError = $true,
+ [string] $warnNotAsError = '',
[bool] $nodeReuse = $true,
+ [bool][Alias('mt')]$msbuildMultiThreaded = $false,
+ [switch] $buildCheck = $false,
[switch][Alias('r')]$restore,
[switch] $deployDeps,
[switch][Alias('b')]$build,
@@ -20,7 +23,10 @@ Param(
[switch] $publish,
[switch] $clean,
[switch][Alias('pb')]$productBuild,
+ [switch]$fromVMR,
+ [switch]$disablePipelineSetResult,
[switch][Alias('bl')]$binaryLog,
+ [string][Alias('bln')]$binaryLogName = '',
[switch][Alias('nobl')]$excludeCIBinarylog,
[switch] $ci,
[switch] $prepareMachine,
@@ -43,6 +49,7 @@ function Print-Usage() {
Write-Host " -platform Platform configuration: 'x86', 'x64' or any valid Platform value to pass to msbuild"
Write-Host " -verbosity Msbuild verbosity: q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic] (short: -v)"
Write-Host " -binaryLog Output binary log (short: -bl)"
+ Write-Host " -binaryLogName Binary log file name or path; implies -binaryLog (short: -bln)"
Write-Host " -help Print help and exit"
Write-Host ""
@@ -68,9 +75,15 @@ function Print-Usage() {
Write-Host " -excludeCIBinarylog Don't output binary log (short: -nobl)"
Write-Host " -prepareMachine Prepare machine for CI run, clean up processes after build"
Write-Host " -warnAsError Sets warnaserror msbuild parameter ('true' or 'false')"
+ Write-Host " -warnNotAsError Sets a semi-colon delimited list of warning codes that should not be treated as errors"
Write-Host " -msbuildEngine Msbuild engine to use to run build ('dotnet', 'vs', or unspecified)."
Write-Host " -excludePrereleaseVS Set to exclude build engines in prerelease versions of Visual Studio"
Write-Host " -nativeToolsOnMachine Sets the native tools on machine environment variable (indicating that the script should use native tools on machine)"
+ Write-Host " -nodeReuse Sets nodereuse msbuild parameter ('true' or 'false')"
+ Write-Host " -msbuildMultiThreaded Sets MSBuild's multi-threaded mode, i.e. the -mt switch ('1' or '0') (short: -mt)"
+ Write-Host " -buildCheck Sets /check msbuild parameter"
+ Write-Host " -fromVMR Set when building from within the VMR"
+ Write-Host " -disablePipelineSetResult Set to disable masking the actual exit code in the pipeline when the build fails"
Write-Host ""
Write-Host "Command line arguments not listed above are passed thru to msbuild."
@@ -95,8 +108,21 @@ function Build {
$toolsetBuildProj = InitializeToolset
InitializeCustomToolset
- $bl = if ($binaryLog) { '/bl:' + (Join-Path $LogDir 'Build.binlog') } else { '' }
+ $bl = ''
+ if ($binaryLog) {
+ $binaryLogPath = if ([string]::IsNullOrEmpty($binaryLogName)) {
+ Join-Path $LogDir 'Build.binlog'
+ } elseif ([System.IO.Path]::IsPathRooted($binaryLogName)) {
+ $binaryLogName
+ } else {
+ Join-Path $LogDir $binaryLogName
+ }
+
+ Create-Directory (Split-Path -Parent $binaryLogPath)
+ $bl = '/bl:' + $binaryLogPath
+ }
$platformArg = if ($platform) { "/p:Platform=$platform" } else { '' }
+ $check = if ($buildCheck) { '/check' } else { '' }
if ($projects) {
# Re-assign properties to a new variable because PowerShell doesn't let us append properties directly for unclear reasons.
@@ -113,6 +139,7 @@ function Build {
MSBuild $toolsetBuildProj `
$bl `
$platformArg `
+ $check `
/p:Configuration=$configuration `
/p:RepoRoot=$RepoRoot `
/p:Restore=$restore `
@@ -122,11 +149,13 @@ function Build {
/p:Deploy=$deploy `
/p:Test=$test `
/p:Pack=$pack `
- /p:DotNetBuildRepo=$productBuild `
+ /p:DotNetBuild=$productBuild `
+ /p:DotNetBuildFromVMR=$fromVMR `
/p:IntegrationTest=$integrationTest `
/p:PerformanceTest=$performanceTest `
/p:Sign=$sign `
/p:Publish=$publish `
+ /p:RestoreStaticGraphEnableBinaryLogger=$binaryLog `
@properties
}
@@ -148,7 +177,14 @@ try {
if (-not $excludeCIBinarylog) {
$binaryLog = $true
}
- $nodeReuse = $false
+ # Node reuse isn't used on CI unless it was explicitly requested via -nodeReuse.
+ if (-not $PSBoundParameters.ContainsKey('nodeReuse')) {
+ $nodeReuse = $false
+ }
+ }
+
+ if (-not [string]::IsNullOrEmpty($binaryLogName)) {
+ $binaryLog = $true
}
if ($nativeToolsOnMachine) {
diff --git a/eng/common/build.sh b/eng/common/build.sh
index 483647daf..109d83ff7 100644
--- a/eng/common/build.sh
+++ b/eng/common/build.sh
@@ -13,6 +13,7 @@ usage()
echo " --configuration Build configuration: 'Debug' or 'Release' (short: -c)"
echo " --verbosity Msbuild verbosity: q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic] (short: -v)"
echo " --binaryLog Create MSBuild binary log (short: -bl)"
+ echo " --binaryLogName Binary log file name or path; implies --binaryLog (short: -bln)"
echo " --help Print help and exit (short: -h)"
echo ""
@@ -39,9 +40,15 @@ usage()
echo " --projects Project or solution file(s) to build"
echo " --ci Set when running on CI server"
echo " --excludeCIBinarylog Don't output binary log (short: -nobl)"
+ echo " --pipelinesLog Promote msbuild errors/warnings to Azure Pipelines timeline issues; defaults to on in CI (short: -pl)"
echo " --prepareMachine Prepare machine for CI run, clean up processes after build"
echo " --nodeReuse Sets nodereuse msbuild parameter ('true' or 'false')"
+ echo " --msbuildMultiThreaded Sets MSBuild's multi-threaded mode, i.e. the -mt switch ('true' or 'false') (short: --mt)"
echo " --warnAsError Sets warnaserror msbuild parameter ('true' or 'false')"
+ echo " --warnNotAsError Sets a semi-colon delimited list of warning codes that should not be treated as errors"
+ echo " --buildCheck Sets /check msbuild parameter"
+ echo " --fromVMR Set when building from within the VMR"
+ echo " --disablePipelineSetResult Set to disable masking the actual exit code in the pipeline when the build fails"
echo ""
echo "Command line arguments not listed above are passed thru to msbuild."
echo "Arguments can also be passed in with a single hyphen."
@@ -63,6 +70,8 @@ restore=false
build=false
source_build=false
product_build=false
+from_vmr=false
+disable_pipeline_set_result=false
rebuild=false
test=false
integration_test=false
@@ -75,8 +84,13 @@ ci=false
clean=false
warn_as_error=true
-node_reuse=true
+warn_not_as_error=''
+# Empty means "not specified"; tools.sh defaults these to on for local builds and off on CI.
+node_reuse=''
+msbuild_multi_threaded=''
+build_check=false
binary_log=false
+binary_log_name=''
exclude_ci_binary_log=false
pipelines_log=false
@@ -87,8 +101,8 @@ verbosity='minimal'
runtime_source_feed=''
runtime_source_feed_key=''
-properties=''
-while [[ $# > 0 ]]; do
+properties=()
+while [[ $# -gt 0 ]]; do
opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")"
case "$opt" in
-help|-h)
@@ -109,6 +123,11 @@ while [[ $# > 0 ]]; do
-binarylog|-bl)
binary_log=true
;;
+ -binarylogname|-bln)
+ binary_log=true
+ binary_log_name=$2
+ shift
+ ;;
-excludecibinarylog|-nobl)
exclude_ci_binary_log=true
;;
@@ -127,19 +146,25 @@ while [[ $# > 0 ]]; do
-pack)
pack=true
;;
- -sourcebuild|-sb)
+ -sourcebuild|-source-build|-sb)
build=true
source_build=true
product_build=true
restore=true
pack=true
;;
- -productBuild|-pb)
+ -productbuild|-product-build|-pb)
build=true
product_build=true
restore=true
pack=true
;;
+ -fromvmr|-from-vmr)
+ from_vmr=true
+ ;;
+ -disablepipelinesetresult|-disable-pipeline-set-result)
+ disable_pipeline_set_result=true
+ ;;
-test|-t)
test=true
;;
@@ -169,10 +194,21 @@ while [[ $# > 0 ]]; do
warn_as_error=$2
shift
;;
+ -warnnotaserror)
+ warn_not_as_error=$2
+ shift
+ ;;
-nodereuse)
node_reuse=$2
shift
;;
+ -msbuildmultithreaded|-mt)
+ msbuild_multi_threaded=$2
+ shift
+ ;;
+ -buildcheck)
+ build_check=true
+ ;;
-runtimesourcefeed)
runtime_source_feed=$2
shift
@@ -182,7 +218,7 @@ while [[ $# > 0 ]]; do
shift
;;
*)
- properties="$properties $1"
+ properties+=("$1")
;;
esac
@@ -195,7 +231,6 @@ fi
if [[ "$ci" == true ]]; then
pipelines_log=true
- node_reuse=false
if [[ "$exclude_ci_binary_log" == false ]]; then
binary_log=true
fi
@@ -216,22 +251,39 @@ function Build {
InitializeCustomToolset
if [[ ! -z "$projects" ]]; then
- properties="$properties /p:Projects=$projects"
+ properties+=("/p:Projects=$projects")
fi
local bl=""
if [[ "$binary_log" == true ]]; then
- bl="/bl:\"$log_dir/Build.binlog\""
+ local binary_log_path=""
+ if [[ -z "$binary_log_name" ]]; then
+ binary_log_path="$log_dir/Build.binlog"
+ elif [[ "$binary_log_name" = /* ]]; then
+ binary_log_path="$binary_log_name"
+ else
+ binary_log_path="$log_dir/$binary_log_name"
+ fi
+
+ mkdir -p "$(dirname "$binary_log_path")"
+ bl="/bl:\"$binary_log_path\""
+ fi
+
+ local check=""
+ if [[ "$build_check" == true ]]; then
+ check="/check"
fi
MSBuild $_InitializeToolset \
$bl \
+ $check \
/p:Configuration=$configuration \
/p:RepoRoot="$repo_root" \
/p:Restore=$restore \
/p:Build=$build \
- /p:DotNetBuildRepo=$product_build \
+ /p:DotNetBuild=$product_build \
/p:DotNetBuildSourceOnly=$source_build \
+ /p:DotNetBuildFromVMR=$from_vmr \
/p:Rebuild=$rebuild \
/p:Test=$test \
/p:Pack=$pack \
@@ -239,7 +291,8 @@ function Build {
/p:PerformanceTest=$performance_test \
/p:Sign=$sign \
/p:Publish=$publish \
- $properties
+ /p:RestoreStaticGraphEnableBinaryLogger=$binary_log \
+ ${properties[@]+"${properties[@]}"}
ExitWithExitCode 0
}
diff --git a/eng/common/cibuild.sh b/eng/common/cibuild.sh
index 1a02c0dec..66e3b0ac6 100644
--- a/eng/common/cibuild.sh
+++ b/eng/common/cibuild.sh
@@ -13,4 +13,4 @@ while [[ -h $source ]]; do
done
scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
-. "$scriptroot/build.sh" --restore --build --test --pack --publish --ci $@
\ No newline at end of file
+. "$scriptroot/build.sh" --restore --build --test --pack --publish --ci $@
diff --git a/eng/common/core-templates/job/helix-job-monitor.yml b/eng/common/core-templates/job/helix-job-monitor.yml
new file mode 100644
index 000000000..81ecccdd1
--- /dev/null
+++ b/eng/common/core-templates/job/helix-job-monitor.yml
@@ -0,0 +1,270 @@
+parameters:
+# Maximum run time of the monitor job in minutes. Also used for --max-wait-minutes.
+- name: timeoutInMinutes
+ type: number
+ default: 360
+
+# Owner segment of the source repository (e.g. 'dotnet' for 'dotnet/runtime') passed via --organization.
+# Defaults to the owner segment of BUILD_REPOSITORY_NAME when empty.
+- name: organization
+ type: string
+ default: ''
+
+# Name of the source repository (e.g. 'runtime' for 'dotnet/runtime') passed via --repository.
+# Defaults to the repo segment of BUILD_REPOSITORY_NAME when empty.
+- name: repository
+ type: string
+ default: ''
+
+# Optional dependency list for the generated job.
+- name: dependsOn
+ type: object
+ default: []
+
+# Optional condition for the generated job.
+- name: condition
+ type: string
+ default: ''
+
+# Whether failures in the monitor job should allow the pipeline to continue.
+- name: continueOnError
+ type: boolean
+ default: false
+
+# NuGet package id of the Helix job monitor tool.
+- name: toolPackageId
+ type: string
+ default: Microsoft.DotNet.Helix.JobMonitor
+
+# Console command exposed by the installed tool package.
+- name: toolCommand
+ type: string
+ default: dotnet-helix-job-monitor
+
+# Optional explicit tool version. Only honored when 'toolNupkgArtifactName' is set; in the
+# default code path the version is taken from the consuming repo's .config/dotnet-tools.json.
+- name: toolVersion
+ type: string
+ default: ''
+
+# Base URI for the Helix service (--helix-base-uri).
+- name: helixBaseUri
+ type: string
+ default: https://helix.dot.net/
+
+# Helix API access token forwarded to the tool via the HELIX_ACCESSTOKEN environment variable.
+- name: helixAccessToken
+ type: string
+ default: ''
+
+# Polling interval in seconds (--polling-interval-seconds).
+- name: pollingIntervalSeconds
+ type: number
+ default: 30
+
+# When 'true' (the default), Helix work items that exit 0 but have failed AzDO test results
+# are treated as failed: they count toward the monitor's exit code and are resubmitted by a
+# later invocation's retry pass. Set to 'false' to fall back to exit-code-only outcomes.
+# Forwarded as --fail-on-failed-tests.
+- name: failWorkItemsWithFailedTests
+ type: boolean
+ default: true
+
+# When true, allow the monitor to succeed when this stage produces no Helix jobs in any attempt.
+# Forwarded as --allow-no-helix-jobs.
+- name: allowNoHelixJobs
+ type: boolean
+ default: false
+
+# When true, test results are reported to Azure DevOps using the fully qualified test name
+# (Namespace.Type.Method) as the stable automatedTestName and the visible title is qualified as
+# well (--use-fully-qualified-test-name). Opt-in because it changes AzDO test identity and display;
+# primarily useful for frameworks like MSTest whose display name is only the method name.
+- name: useFullyQualifiedTestName
+ type: boolean
+ default: false
+
+# Controls per-test output attachments. Defaults to Failed.
+- name: testResultAttachmentMode
+ type: string
+ default: Failed
+ values:
+ - Failed
+ - All
+ - None
+
+# Advanced: optional pipeline artifact (produced earlier in this run) that contains the tool
+# nupkg. When set, the artifact is downloaded and the tool is installed from the nupkg into
+# a local tool-path; this bypasses the repo's .config/dotnet-tools.json manifest and is
+# primarily intended for the Arcade repository itself, where the Helix job monitor tool is
+# built in the same pipeline that runs this template.
+#
+# When this parameter is empty (the default), the consuming repository must declare the tool
+# in its .config/dotnet-tools.json manifest (alongside other local .NET tools); the template
+# will check out the repo and run 'dotnet tool restore' to install the version pinned there.
+- name: toolNupkgArtifactName
+ type: string
+ default: ''
+
+# Advanced: sub-path within the downloaded artifact where the tool nupkg is located. Defaults
+# to the standard Arcade non-shipping packages location for a Release build (relative to the
+# pipeline artifact root, which is itself the build's 'artifacts' directory).
+- name: toolNupkgArtifactSubPath
+ type: string
+ default: 'packages/Release/NonShipping'
+
+jobs:
+- job: HelixJobMonitor
+ displayName: Monitor Helix Jobs
+ timeoutInMinutes: ${{ parameters.timeoutInMinutes }}
+ continueOnError: ${{ parameters.continueOnError }}
+ ${{ if ne(length(parameters.dependsOn), 0) }}:
+ dependsOn: ${{ parameters.dependsOn }}
+ ${{ if ne(parameters.condition, '') }}:
+ condition: ${{ parameters.condition }}
+ pool:
+ ${{ if eq(variables['System.TeamProject'], 'public') }}:
+ name: $(DncEngPublicBuildPool)
+ os: linux
+ demands: ImageOverride -equals build.azurelinux.3.amd64.open
+ ${{ else }}:
+ name: $(DncEngInternalBuildPool)
+ os: linux
+ demands: ImageOverride -equals build.azurelinux.3.amd64
+ steps:
+ - checkout: self
+ fetchDepth: 1
+
+ - ${{ if ne(parameters.toolNupkgArtifactName, '') }}:
+ - task: DownloadPipelineArtifact@2
+ displayName: Download Helix Job Monitor artifact
+ inputs:
+ buildType: current
+ artifactName: ${{ parameters.toolNupkgArtifactName }}
+ itemPattern: '${{ parameters.toolNupkgArtifactSubPath }}/${{ parameters.toolPackageId }}.*.nupkg'
+ targetPath: $(Agent.TempDirectory)/helix-job-monitor-nupkg
+
+ - bash: |
+ set -euo pipefail
+
+ toolPath="$AGENT_TEMPDIRECTORY/helix-job-monitor-tool"
+ mkdir -p "$toolPath"
+
+ packageId='${{ parameters.toolPackageId }}'
+ toolVersion='${{ parameters.toolVersion }}'
+ nupkgArtifactSubPath='${{ parameters.toolNupkgArtifactSubPath }}'
+ nupkgDir="$AGENT_TEMPDIRECTORY/helix-job-monitor-nupkg/$nupkgArtifactSubPath"
+
+ if [ ! -d "$nupkgDir" ]; then
+ echo "Expected nupkg directory '$nupkgDir' was not produced by the artifact download." >&2
+ exit 1
+ fi
+
+ nupkg=$(find "$nupkgDir" -maxdepth 1 -type f -name "$packageId.*.nupkg" | head -n 1)
+ if [ -z "$nupkg" ]; then
+ echo "No '$packageId.*.nupkg' found in '$nupkgDir'." >&2
+ exit 1
+ fi
+
+ # Derive the version from the nupkg filename so the local package is selected
+ # deterministically instead of resolving against any other configured feed.
+ nupkgBase=$(basename "$nupkg" .nupkg)
+ derivedVersion="${nupkgBase#${packageId}.}"
+ if [ -z "$toolVersion" ]; then
+ toolVersion="$derivedVersion"
+ fi
+
+ echo "Using locally built '$packageId' version '$toolVersion' from '$nupkgDir'."
+
+ # Create a minimal NuGet.config that only references the local nupkg directory.
+ # This avoids conflicts with the repo's package source mapping which blocks --add-source.
+ toolNugetConfig="$AGENT_TEMPDIRECTORY/helix-job-monitor-nuget.config"
+ printf '\n\n \n \n \n \n\n' "$nupkgDir" > "$toolNugetConfig"
+
+ pushd "$(Build.SourcesDirectory)" > /dev/null
+ ./eng/common/dotnet.sh tool install \
+ --tool-path "$toolPath" "$packageId" \
+ --version "$toolVersion" \
+ --configfile "$toolNugetConfig"
+
+ # Locate the tool DLL so the run step can invoke it via ./eng/common/dotnet.sh exec.
+ toolDll=$(find "$toolPath/.store" -path '*/tools/*/any/*.deps.json' -type f | head -n 1)
+ toolDll="${toolDll%.deps.json}.dll"
+ if [ ! -f "$toolDll" ]; then
+ echo "Could not find tool DLL in '$toolPath/.store'." >&2
+ exit 1
+ fi
+
+ echo "Tool DLL: $toolDll"
+ echo "##vso[task.setvariable variable=HelixJobMonitorDll]$toolDll"
+ displayName: Install Helix Job Monitor
+
+ - ${{ else }}:
+ - bash: ./eng/common/dotnet.sh tool restore
+ displayName: Restore Helix Job Monitor
+
+ - bash: |
+ set -euo pipefail
+
+ toolArgs=(
+ --helix-base-uri '${{ parameters.helixBaseUri }}'
+ --polling-interval-seconds '${{ parameters.pollingIntervalSeconds }}'
+ --fail-on-failed-tests '${{ parameters.failWorkItemsWithFailedTests }}'
+ --allow-no-helix-jobs '${{ parameters.allowNoHelixJobs }}'
+ --use-fully-qualified-test-name '${{ parameters.useFullyQualifiedTestName }}'
+ --max-wait-minutes "$((${{ parameters.timeoutInMinutes }} - 5))" # Set the tool's timeout slightly lower than the Azure DevOps job timeout to allow it to exit gracefully.
+ --stage-name '$(System.StageName)'
+ --stage-attempt '$(System.StageAttempt)'
+ )
+
+ organization='${{ parameters.organization }}'
+ repository='${{ parameters.repository }}'
+ testResultAttachmentMode='${{ parameters.testResultAttachmentMode }}'
+
+ # Fall back to Azure DevOps-provided environment variables when the caller did not
+ # supply organization / repository explicitly. BUILD_REPOSITORY_NAME is typically
+ # 'owner/repo' for GitHub-backed builds and 'owner-repo' for internal builds.
+ if [ -z "$organization" ] || [ -z "$repository" ]; then
+ buildRepoName="${BUILD_REPOSITORY_NAME:-}"
+ if [ -n "$buildRepoName" ] && [[ "$buildRepoName" == */* ]]; then
+ repoOwner="${buildRepoName%%/*}"
+ repoName="${buildRepoName#*/}"
+ elif [ -n "$buildRepoName" ] && [[ "$buildRepoName" == *-* ]]; then
+ repoOwner="${buildRepoName%%-*}"
+ repoName="${buildRepoName#*-}"
+ fi
+
+ if [ -n "${repoOwner:-}" ] && [ -n "${repoName:-}" ]; then
+ if [ -z "$organization" ]; then organization="$repoOwner"; fi
+ if [ -z "$repository" ]; then repository="$repoName"; fi
+ fi
+ fi
+
+ if [ -n "$organization" ]; then toolArgs+=( --organization "$organization" ); fi
+ if [ -n "$repository" ]; then toolArgs+=( --repository "$repository" ); fi
+ if [ -n "$testResultAttachmentMode" ]; then
+ toolArgs+=( --test-result-attachment-mode "$testResultAttachmentMode" )
+ fi
+
+ # Build.Reason and Build.SourceBranch are required to derive the Helix source filter
+ # the same way the Helix SDK submitter does (PR -> 'pr', internal -> 'official',
+ # otherwise -> 'ci'). Without these, manually-queued / scheduled / CI builds would
+ # be looked up under the wrong source prefix and find zero jobs.
+ toolArgs+=( --build-reason "$(Build.Reason)" )
+ toolArgs+=( --source-branch "$(Build.SourceBranch)" )
+
+ if [ -n '${{ parameters.toolNupkgArtifactName }}' ]; then
+ # Tool was installed from a local nupkg; run the DLL via the repo-local dotnet.
+ export DOTNET_ROOT="$(Build.SourcesDirectory)/.dotnet"
+ ./eng/common/dotnet.sh exec "$(HelixJobMonitorDll)" "${toolArgs[@]}"
+ else
+ # Tool was restored from the local .config/dotnet-tools.json manifest; invoke it
+ # through the manifest from the repo root.
+ pushd "$BUILD_SOURCESDIRECTORY" > /dev/null
+ trap 'popd > /dev/null' EXIT
+ ./eng/common/dotnet.sh tool run '${{ parameters.toolCommand }}' -- "${toolArgs[@]}"
+ fi
+ displayName: Monitor Helix Jobs
+ env:
+ SYSTEM_ACCESSTOKEN: $(System.AccessToken)
+ HELIX_ACCESSTOKEN: ${{ parameters.helixAccessToken }}
diff --git a/eng/common/core-templates/job/job.yml b/eng/common/core-templates/job/job.yml
index 295c9a231..cb60f5297 100644
--- a/eng/common/core-templates/job/job.yml
+++ b/eng/common/core-templates/job/job.yml
@@ -19,17 +19,19 @@ parameters:
# publishing defaults
artifacts: ''
enableMicrobuild: false
+ enablePreviewMicrobuild: false
+ microbuildPluginVersion: 'latest'
enableMicrobuildForMacAndLinux: false
+ microbuildUseESRP: true
enablePublishBuildArtifacts: false
enablePublishBuildAssets: false
enablePublishTestResults: false
- enablePublishUsingPipelines: false
+ enablePublishing: false
enableBuildRetry: false
mergeTestResults: false
testRunTitle: ''
testResultsFormat: ''
name: ''
- componentGovernanceSteps: []
preSteps: []
artifactPublishSteps: []
runAsPublic: false
@@ -71,12 +73,17 @@ jobs:
templateContext: ${{ parameters.templateContext }}
variables:
+ - name: AllowPtrToDetectTestRunRetryFiles
+ value: true
+ # Component Governance detection and CodeQL are not run in the public project
+ - ${{ if eq(variables['System.TeamProject'], 'public') }}:
+ - name: skipComponentGovernanceDetection
+ value: true
+ - name: Codeql.SkipTaskAutoInjection
+ value: true
- ${{ if ne(parameters.enableTelemetry, 'false') }}:
- name: DOTNET_CLI_TELEMETRY_PROFILE
value: '$(Build.Repository.Uri)'
- - ${{ if eq(parameters.enableRichCodeNavigation, 'true') }}:
- - name: EnableRichCodeNavigation
- value: 'true'
# Retry signature validation up to three times, waiting 2 seconds between attempts.
# See https://learn.microsoft.com/en-us/nuget/reference/errors-and-warnings/nu3028#retry-untrusted-root-failures
- name: NUGET_EXPERIMENTAL_CHAIN_BUILD_RETRY_POLICY
@@ -131,7 +138,10 @@ jobs:
- template: /eng/common/core-templates/steps/install-microbuild.yml
parameters:
enableMicrobuild: ${{ parameters.enableMicrobuild }}
+ enablePreviewMicrobuild: ${{ parameters.enablePreviewMicrobuild }}
+ microbuildPluginVersion: ${{ parameters.microbuildPluginVersion }}
enableMicrobuildForMacAndLinux: ${{ parameters.enableMicrobuildForMacAndLinux }}
+ microbuildUseESRP: ${{ parameters.microbuildUseESRP }}
continueOnError: ${{ parameters.continueOnError }}
- ${{ if and(eq(parameters.runAsPublic, 'false'), eq(variables['System.TeamProject'], 'internal')) }}:
@@ -148,23 +158,12 @@ jobs:
- ${{ each step in parameters.steps }}:
- ${{ step }}
- - ${{ if eq(parameters.enableRichCodeNavigation, true) }}:
- - task: RichCodeNavIndexer@0
- displayName: RichCodeNav Upload
- inputs:
- languages: ${{ coalesce(parameters.richCodeNavigationLanguage, 'csharp') }}
- environment: ${{ coalesce(parameters.richCodeNavigationEnvironment, 'internal') }}
- richNavLogOutputDirectory: $(Build.SourcesDirectory)/artifacts/bin
- uploadRichNavArtifacts: ${{ coalesce(parameters.richCodeNavigationUploadArtifacts, false) }}
- continueOnError: true
-
- - ${{ each step in parameters.componentGovernanceSteps }}:
- - ${{ step }}
-
- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
- template: /eng/common/core-templates/steps/cleanup-microbuild.yml
parameters:
enableMicrobuild: ${{ parameters.enableMicrobuild }}
+ enablePreviewMicrobuild: ${{ parameters.enablePreviewMicrobuild }}
+ microbuildPluginVersion: ${{ parameters.microbuildPluginVersion }}
enableMicrobuildForMacAndLinux: ${{ parameters.enableMicrobuildForMacAndLinux }}
continueOnError: ${{ parameters.continueOnError }}
@@ -175,7 +174,7 @@ jobs:
inputs:
testResultsFormat: 'xUnit'
testResultsFiles: '*.xml'
- searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)'
+ searchFolder: '$(System.DefaultWorkingDirectory)/artifacts/TestResults/$(_BuildConfig)'
testRunTitle: ${{ coalesce(parameters.testRunTitle, parameters.name, '$(System.JobName)') }}-xunit
mergeTestResults: ${{ parameters.mergeTestResults }}
continueOnError: true
@@ -186,7 +185,7 @@ jobs:
inputs:
testResultsFormat: 'VSTest'
testResultsFiles: '*.trx'
- searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)'
+ searchFolder: '$(System.DefaultWorkingDirectory)/artifacts/TestResults/$(_BuildConfig)'
testRunTitle: ${{ coalesce(parameters.testRunTitle, parameters.name, '$(System.JobName)') }}-trx
mergeTestResults: ${{ parameters.mergeTestResults }}
continueOnError: true
@@ -230,7 +229,7 @@ jobs:
- task: CopyFiles@2
displayName: Gather buildconfiguration for build retry
inputs:
- SourceFolder: '$(Build.SourcesDirectory)/eng/common/BuildConfiguration'
+ SourceFolder: '$(System.DefaultWorkingDirectory)/eng/common/BuildConfiguration'
Contents: '**'
TargetFolder: '$(Build.ArtifactStagingDirectory)/eng/common/BuildConfiguration'
continueOnError: true
diff --git a/eng/common/core-templates/job/onelocbuild.yml b/eng/common/core-templates/job/onelocbuild.yml
index 00feec8eb..ce077368e 100644
--- a/eng/common/core-templates/job/onelocbuild.yml
+++ b/eng/common/core-templates/job/onelocbuild.yml
@@ -4,11 +4,27 @@ parameters:
# Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool
pool: ''
-
+
CeapexPat: $(dn-bot-ceapex-package-r) # PAT for the loc AzDO instance https://dev.azure.com/ceapex
GithubPat: $(BotAccount-dotnet-bot-repo-PAT)
- SourcesDirectory: $(Build.SourcesDirectory)
+ # Service connection for WIF-based Entra authentication to ceapex feeds (replaces CeapexPat).
+ # When set, dnceng/internal builds acquire a federated Entra token instead of using a PAT.
+ # All other projects (e.g. DevDiv, public), where this dnceng-scoped service connection does not
+ # exist, and any pipeline that sets this to '' fall back to PAT-based auth via the CeapexPat parameter.
+ CeapexServiceConnection: 'dnceng-onelocbuild-ceapex'
+
+ # GitHub App authentication for the OneLoc check-in PR.
+ # dnceng/internal and DevDiv/DevDiv are enabled by default with their project-scoped service
+ # connections. Other projects must explicitly opt in after provisioning equivalent infrastructure.
+ UseGitHubAppAuthentication: true
+ UseGitHubAppAuthenticationInOtherProjects: false
+ GitHubAppServiceConnection: 'dnceng-oneloc-githubapp'
+ GitHubAppClientId: 'Iv23lijBU8x3gc9lDOc9'
+ GitHubAppKeyVaultName: 'EngKeyVault'
+ GitHubAppKeyName: 'oneloc-localization-app-key'
+
+ SourcesDirectory: $(System.DefaultWorkingDirectory)
CreatePr: true
AutoCompletePr: false
ReusePr: true
@@ -22,12 +38,13 @@ parameters:
GitHubOrg: dotnet
MirrorRepo: ''
MirrorBranch: main
+ xLocCustomPowerShellScript: ''
condition: ''
JobNameSuffix: ''
is1ESPipeline: ''
jobs:
- job: OneLocBuild${{ parameters.JobNameSuffix }}
-
+
dependsOn: ${{ parameters.dependsOn }}
displayName: OneLocBuild${{ parameters.JobNameSuffix }}
@@ -52,13 +69,13 @@ jobs:
# We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com)
${{ if eq(variables['System.TeamProject'], 'DevDiv') }}:
name: AzurePipelines-EO
- image: 1ESPT-Windows2022
+ image: 1ESPT-Windows2025
demands: Cmd
os: windows
# If it's not devdiv, it's dnceng
${{ if ne(variables['System.TeamProject'], 'DevDiv') }}:
name: $(DncEngInternalBuildPool)
- image: 1es-windows-2022
+ image: windows.vs2026.amd64
os: windows
steps:
@@ -68,11 +85,37 @@ jobs:
- ${{ if ne(parameters.SkipLocProjectJsonGeneration, 'true') }}:
- task: Powershell@2
inputs:
- filePath: $(Build.SourcesDirectory)/eng/common/generate-locproject.ps1
+ filePath: $(System.DefaultWorkingDirectory)/eng/common/generate-locproject.ps1
arguments: $(_GenerateLocProjectArguments)
displayName: Generate LocProject.json
condition: ${{ parameters.condition }}
+ # Acquire an Entra token for ceapex feed access via WIF (dnceng/internal only).
+ # All other projects use PAT-based auth, since the ceapex service connection is scoped to dnceng/internal.
+ - ${{ if and(ne(parameters.CeapexServiceConnection, ''), eq(variables['System.TeamProject'], 'internal')) }}:
+ - template: /eng/common/templates/steps/get-federated-access-token.yml
+ parameters:
+ federatedServiceConnection: ${{ parameters.CeapexServiceConnection }}
+ outputVariableName: 'CeapexEntraToken'
+ condition: ${{ parameters.condition }}
+
+ # Mint a short-lived GitHub App installation token for the loc check-in PR. Use the connection
+ # provisioned in each supported project; other projects must explicitly opt in and override it.
+ - ${{ if and(eq(parameters.RepoType, 'gitHub'), eq(parameters.UseGitHubAppAuthentication, true), or(eq(variables['System.TeamProject'], 'internal'), eq(variables['System.TeamProject'], 'DevDiv'), eq(parameters.UseGitHubAppAuthenticationInOtherProjects, true))) }}:
+ - template: /eng/common/core-templates/steps/get-github-app-token.yml
+ parameters:
+ is1ESPipeline: ${{ parameters.is1ESPipeline }}
+ ${{ if and(eq(variables['System.TeamProject'], 'DevDiv'), eq(parameters.GitHubAppServiceConnection, 'dnceng-oneloc-githubapp')) }}:
+ azureSubscription: 'devdiv-oneloc-githubapp'
+ ${{ else }}:
+ azureSubscription: ${{ parameters.GitHubAppServiceConnection }}
+ keyVaultName: ${{ parameters.GitHubAppKeyVaultName }}
+ keyName: ${{ parameters.GitHubAppKeyName }}
+ appClientId: ${{ parameters.GitHubAppClientId }}
+ installationOwner: ${{ parameters.GitHubOrg }}
+ outputVariableName: 'GitHubAppInstallationToken'
+ condition: ${{ parameters.condition }}
+
- task: OneLocBuild@2
displayName: OneLocBuild
env:
@@ -86,36 +129,41 @@ jobs:
isAutoCompletePrSelected: ${{ parameters.AutoCompletePr }}
${{ if eq(parameters.CreatePr, true) }}:
isUseLfLineEndingsSelected: ${{ parameters.UseLfLineEndings }}
- ${{ if eq(parameters.RepoType, 'gitHub') }}:
- isShouldReusePrSelected: ${{ parameters.ReusePr }}
+ isShouldReusePrSelected: ${{ parameters.ReusePr }}
packageSourceAuth: patAuth
- patVariable: ${{ parameters.CeapexPat }}
+ ${{ if and(ne(parameters.CeapexServiceConnection, ''), eq(variables['System.TeamProject'], 'internal')) }}:
+ patVariable: $(CeapexEntraToken)
+ ${{ if or(eq(parameters.CeapexServiceConnection, ''), ne(variables['System.TeamProject'], 'internal')) }}:
+ patVariable: ${{ parameters.CeapexPat }}
${{ if eq(parameters.RepoType, 'gitHub') }}:
repoType: ${{ parameters.RepoType }}
- gitHubPatVariable: "${{ parameters.GithubPat }}"
+ ${{ if and(eq(parameters.UseGitHubAppAuthentication, true), or(eq(variables['System.TeamProject'], 'internal'), eq(variables['System.TeamProject'], 'DevDiv'), eq(parameters.UseGitHubAppAuthenticationInOtherProjects, true))) }}:
+ gitHubPatVariable: "$(GitHubAppInstallationToken)"
+ ${{ else }}:
+ gitHubPatVariable: "${{ parameters.GithubPat }}"
${{ if ne(parameters.MirrorRepo, '') }}:
isMirrorRepoSelected: true
gitHubOrganization: ${{ parameters.GitHubOrg }}
mirrorRepo: ${{ parameters.MirrorRepo }}
mirrorBranch: ${{ parameters.MirrorBranch }}
+ ${{ if ne(parameters.xLocCustomPowerShellScript, '') }}:
+ xLocCustomPowerShellScript: ${{ parameters.xLocCustomPowerShellScript }}
condition: ${{ parameters.condition }}
- - template: /eng/common/core-templates/steps/publish-build-artifacts.yml
- parameters:
- is1ESPipeline: ${{ parameters.is1ESPipeline }}
- args:
- displayName: Publish Localization Files
- pathToPublish: '$(Build.ArtifactStagingDirectory)/loc'
- publishLocation: Container
- artifactName: Loc
- condition: ${{ parameters.condition }}
+ # Copy the locProject.json to the root of the Loc directory, then publish a pipeline artifact
+ - task: CopyFiles@2
+ displayName: Copy LocProject.json
+ inputs:
+ SourceFolder: '$(System.DefaultWorkingDirectory)/eng/Localize/'
+ Contents: 'LocProject.json'
+ TargetFolder: '$(Build.ArtifactStagingDirectory)/loc'
+ condition: ${{ parameters.condition }}
- - template: /eng/common/core-templates/steps/publish-build-artifacts.yml
+ - template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml
parameters:
is1ESPipeline: ${{ parameters.is1ESPipeline }}
args:
- displayName: Publish LocProject.json
- pathToPublish: '$(Build.SourcesDirectory)/eng/Localize/'
- publishLocation: Container
- artifactName: Loc
- condition: ${{ parameters.condition }}
\ No newline at end of file
+ targetPath: '$(Build.ArtifactStagingDirectory)/loc'
+ artifactName: 'Loc'
+ displayName: 'Publish Localization Files'
+ condition: ${{ parameters.condition }}
diff --git a/eng/common/core-templates/job/publish-build-assets.yml b/eng/common/core-templates/job/publish-build-assets.yml
index c7d59dcbf..330225ae0 100644
--- a/eng/common/core-templates/job/publish-build-assets.yml
+++ b/eng/common/core-templates/job/publish-build-assets.yml
@@ -20,9 +20,6 @@ parameters:
# if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects.
runAsPublic: false
- # Optional: whether the build's artifacts will be published using release pipelines or direct feed publishing
- publishUsingPipelines: false
-
# Optional: whether the build's artifacts will be published using release pipelines or direct feed publishing
publishAssetsImmediately: false
@@ -32,6 +29,19 @@ parameters:
is1ESPipeline: ''
+ # Optional: 🌤️ or not the build has assets it wants to publish to BAR
+ isAssetlessBuild: false
+
+ # Optional, publishing version
+ publishingVersion: 3
+
+ # Optional: A minimatch pattern for the asset manifests to publish to BAR
+ assetManifestsPattern: '*/manifests/**/*.xml'
+
+ repositoryAlias: self
+
+ officialBuildId: ''
+
jobs:
- job: Asset_Registry_Publish
@@ -48,59 +58,89 @@ jobs:
parameters:
is1ESPipeline: ${{ parameters.is1ESPipeline }}
- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
- - group: Publish-Build-Assets
- - group: AzureDevOps-Artifact-Feeds-Pats
- name: runCodesignValidationInjection
value: false
# unconditional - needed for logs publishing (redactor tool version)
- template: /eng/common/core-templates/post-build/common-variables.yml
+ - name: OfficialBuildId
+ ${{ if ne(parameters.officialBuildId, '') }}:
+ value: ${{ parameters.officialBuildId }}
+ ${{ else }}:
+ value: $(Build.BuildNumber)
pool:
# We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com)
${{ if eq(variables['System.TeamProject'], 'DevDiv') }}:
name: AzurePipelines-EO
- image: 1ESPT-Windows2022
+ image: 1ESPT-Windows2025
demands: Cmd
os: windows
# If it's not devdiv, it's dnceng
${{ if ne(variables['System.TeamProject'], 'DevDiv') }}:
name: NetCore1ESPool-Publishing-Internal
- image: windows.vs2019.amd64
+ image: windows.vs2026.amd64
os: windows
steps:
- ${{ if eq(parameters.is1ESPipeline, '') }}:
- 'Illegal entry point, is1ESPipeline is not defined. Repository yaml should not directly reference templates in core-templates folder.': error
- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
- - checkout: self
+ - checkout: ${{ parameters.repositoryAlias }}
fetchDepth: 3
clean: true
-
- - task: DownloadPipelineArtifact@2
- displayName: Download Asset Manifests
- inputs:
- artifactName: AssetManifests
- targetPath: '$(Build.StagingDirectory)/AssetManifests'
- condition: ${{ parameters.condition }}
- continueOnError: ${{ parameters.continueOnError }}
-
+
+ - ${{ if eq(parameters.isAssetlessBuild, 'false') }}:
+ - ${{ if eq(parameters.publishingVersion, 3) }}:
+ - task: DownloadPipelineArtifact@2
+ displayName: Download Asset Manifests
+ inputs:
+ artifactName: AssetManifests
+ targetPath: '$(Build.StagingDirectory)/AssetManifests'
+ condition: ${{ parameters.condition }}
+ continueOnError: ${{ parameters.continueOnError }}
+ - ${{ if eq(parameters.publishingVersion, 4) }}:
+ - task: DownloadPipelineArtifact@2
+ displayName: Download V4 asset manifests
+ inputs:
+ itemPattern: '*/manifests/**/*.xml'
+ targetPath: '$(Build.StagingDirectory)/AllAssetManifests'
+ condition: ${{ parameters.condition }}
+ continueOnError: ${{ parameters.continueOnError }}
+ - task: CopyFiles@2
+ displayName: Copy V4 asset manifests to AssetManifests
+ inputs:
+ SourceFolder: '$(Build.StagingDirectory)/AllAssetManifests'
+ Contents: ${{ parameters.assetManifestsPattern }}
+ TargetFolder: '$(Build.StagingDirectory)/AssetManifests'
+ flattenFolders: true
+ condition: ${{ parameters.condition }}
+ continueOnError: ${{ parameters.continueOnError }}
+
- task: NuGetAuthenticate@1
+ # Populate internal runtime variables.
+ - template: /eng/common/templates/steps/enable-internal-sources.yml
+
+ - template: /eng/common/templates/steps/enable-internal-runtimes.yml
+
- task: AzureCLI@2
displayName: Publish Build Assets
inputs:
azureSubscription: "Darc: Maestro Production"
scriptType: ps
scriptLocation: scriptPath
- scriptPath: $(Build.SourcesDirectory)/eng/common/sdk-task.ps1
+ scriptPath: $(System.DefaultWorkingDirectory)/eng/common/sdk-task.ps1
arguments: -task PublishBuildAssets -restore -msbuildEngine dotnet
/p:ManifestsPath='$(Build.StagingDirectory)/AssetManifests'
+ /p:IsAssetlessBuild=${{ parameters.isAssetlessBuild }}
/p:MaestroApiEndpoint=https://maestro.dot.net
- /p:PublishUsingPipelines=${{ parameters.publishUsingPipelines }}
- /p:OfficialBuildId=$(Build.BuildNumber)
+ /p:OfficialBuildId=$(OfficialBuildId)
+ -runtimeSourceFeed https://ci.dot.net/internal
+ -runtimeSourceFeedKey '$(dotnetbuilds-internal-container-read-token-base64)'
+
condition: ${{ parameters.condition }}
continueOnError: ${{ parameters.continueOnError }}
-
+
- task: powershell@2
displayName: Create ReleaseConfigs Artifact
inputs:
@@ -112,28 +152,45 @@ jobs:
Add-Content -Path $filePath -Value "$(DefaultChannels)"
Add-Content -Path $filePath -Value $(IsStableBuild)
- $symbolExclusionfile = "$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt"
+ $symbolExclusionfile = "$(System.DefaultWorkingDirectory)/eng/SymbolPublishingExclusionsFile.txt"
if (Test-Path -Path $symbolExclusionfile)
{
Write-Host "SymbolExclusionFile exists"
Copy-Item -Path $symbolExclusionfile -Destination "$(Build.StagingDirectory)/ReleaseConfigs"
}
- - template: /eng/common/core-templates/steps/publish-build-artifacts.yml
+ - ${{ if eq(parameters.publishingVersion, 4) }}:
+ - template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml
+ parameters:
+ is1ESPipeline: ${{ parameters.is1ESPipeline }}
+ args:
+ targetPath: '$(Build.ArtifactStagingDirectory)/MergedManifest.xml'
+ artifactName: AssetManifests
+ displayName: 'Publish Merged Manifest'
+ retryCountOnTaskFailure: 10 # for any files being locked
+ isProduction: false # just metadata for publishing
+
+ - template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml
parameters:
is1ESPipeline: ${{ parameters.is1ESPipeline }}
args:
displayName: Publish ReleaseConfigs Artifact
- pathToPublish: '$(Build.StagingDirectory)/ReleaseConfigs'
- publishLocation: Container
+ targetPath: '$(Build.StagingDirectory)/ReleaseConfigs'
artifactName: ReleaseConfigs
+ retryCountOnTaskFailure: 10 # for any files being locked
+ isProduction: false # just metadata for publishing
- - ${{ if eq(parameters.publishAssetsImmediately, 'true') }}:
+ - ${{ if or(eq(parameters.publishAssetsImmediately, 'true'), eq(parameters.isAssetlessBuild, 'true')) }}:
- template: /eng/common/core-templates/post-build/setup-maestro-vars.yml
parameters:
BARBuildId: ${{ parameters.BARBuildId }}
PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
is1ESPipeline: ${{ parameters.is1ESPipeline }}
+
+ # Darc is targeting 8.0, so make sure it's installed
+ - task: UseDotNet@2
+ inputs:
+ version: 8.0.x
- task: AzureCLI@2
displayName: Publish Using Darc
@@ -141,7 +198,7 @@ jobs:
azureSubscription: "Darc: Maestro Production"
scriptType: ps
scriptLocation: scriptPath
- scriptPath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1
+ scriptPath: $(System.DefaultWorkingDirectory)/eng/common/post-build/publish-using-darc.ps1
arguments: >
-BuildId $(BARBuildId)
-PublishingInfraVersion 3
@@ -149,9 +206,13 @@ jobs:
-WaitPublishingFinish true
-ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}'
-SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}'
+ -SkipAssetsPublishing '${{ parameters.isAssetlessBuild }}'
+ -runtimeSourceFeed https://ci.dot.net/internal
+ -runtimeSourceFeedKey '$(dotnetbuilds-internal-container-read-token-base64)'
- ${{ if eq(parameters.enablePublishBuildArtifacts, 'true') }}:
- template: /eng/common/core-templates/steps/publish-logs.yml
parameters:
is1ESPipeline: ${{ parameters.is1ESPipeline }}
- JobLabel: 'Publish_Artifacts_Logs'
+ StageLabel: 'BuildAssetRegistry'
+ JobLabel: 'Publish_Artifacts_Logs'
diff --git a/eng/common/core-templates/job/renovate.yml b/eng/common/core-templates/job/renovate.yml
new file mode 100644
index 000000000..ff86c80b4
--- /dev/null
+++ b/eng/common/core-templates/job/renovate.yml
@@ -0,0 +1,196 @@
+# --------------------------------------------------------------------------------------
+# Renovate Bot Job Template
+# --------------------------------------------------------------------------------------
+# This Azure DevOps pipeline job template runs Renovate (https://docs.renovatebot.com/)
+# to automatically update dependencies in a GitHub repository.
+#
+# Renovate scans the repository for dependency files and creates pull requests to update
+# outdated dependencies based on the configuration specified in the renovateConfigPath
+# parameter.
+#
+# Usage:
+# For each product repo wanting to make use of Renovate, this template is called from
+# an internal Azure DevOps pipeline, typically with a schedule trigger, to check for
+# and propose dependency updates.
+#
+# For more info, see https://github.com/dotnet/arcade/blob/main/Documentation/Renovate.md
+# --------------------------------------------------------------------------------------
+
+parameters:
+
+# Path to the Renovate configuration file within the repository.
+- name: renovateConfigPath
+ type: string
+ default: 'eng/renovate.json'
+
+# GitHub repository to run Renovate against, in the format 'owner/repo'.
+# This could technically be any repo but convention is to target the same
+# repo that contains the calling pipeline. The Renovate config file would
+# be co-located with the pipeline's repo and, in most cases, the config
+# file is specific to the repo being targeted.
+- name: gitHubRepo
+ type: string
+
+# List of base branches to target for Renovate PRs.
+# NOTE: The Renovate configuration file is always read from the branch where the
+# pipeline is run, NOT from the target branches specified here. If you need different
+# configurations for different branches, run the pipeline from each branch separately.
+- name: baseBranches
+ type: object
+ default:
+ - main
+
+# When true, Renovate will run in dry run mode, which previews changes without creating PRs.
+# See the 'Run Renovate' step log output for details of what would have been changed.
+- name: dryRun
+ type: boolean
+ default: false
+
+# By default, Renovate will not recreate a PR for a given dependency/version pair that was
+# previously closed. This allows opting in to always recreating PRs even if they were
+# previously closed.
+- name: forceRecreatePR
+ type: boolean
+ default: false
+
+# Name of the arcade repository resource in the pipeline.
+# This allows repos which haven't been onboarded to Arcade to still use this
+# template by checking out the repo as a resource with a custom name and pointing
+# this parameter to it.
+- name: arcadeRepoResource
+ type: string
+ default: self
+
+# Directory name for the self repo under $(Build.SourcesDirectory) in multi-checkout.
+# In multi-checkout (when arcadeRepoResource != 'self'), Azure DevOps checks out the
+# self repo to $(Build.SourcesDirectory)/. Set this to match the auto-generated
+# directory name. Using the auto-generated name is necessary rather than explicitly
+# defining a checkout path because container jobs expect repos to live under the agent's
+# workspace ($(Pipeline.Workspace)). On some self-hosted setups the host path
+# (e.g., /mnt/vss/_work) differs from the container path (e.g., /__w), and a custom checkout
+# path can fail validation. Using the default checkout location keeps the paths consistent
+# and avoids this issue.
+- name: selfRepoName
+ type: string
+ default: ''
+- name: arcadeRepoName
+ type: string
+ default: ''
+
+# Pool configuration for the job.
+- name: pool
+ type: object
+ default:
+ name: NetCore1ESPool-Internal
+ image: build.azurelinux.3.amd64
+ os: linux
+
+jobs:
+- job: Renovate
+ displayName: Run Renovate
+ container: RenovateContainer
+ variables:
+ - group: dotnet-renovate-bot
+ # The Renovate version is automatically updated by https://github.com/dotnet/arcade/blob/main/azure-pipelines-renovate.yml.
+ # Changing the variable name here would require updating the name in https://github.com/dotnet/arcade/blob/main/eng/renovate.json as well.
+ - name: renovateVersion
+ value: '42'
+ readonly: true
+ - name: renovateLogFilePath
+ value: '$(Build.ArtifactStagingDirectory)/renovate.json'
+ readonly: true
+ - name: dryRunArg
+ readonly: true
+ ${{ if eq(parameters.dryRun, true) }}:
+ value: 'full'
+ ${{ else }}:
+ value: ''
+ - name: recreateWhenArg
+ readonly: true
+ ${{ if eq(parameters.forceRecreatePR, true) }}:
+ value: 'always'
+ ${{ else }}:
+ value: ''
+ # In multi-checkout (without custom paths), Azure DevOps places each repo under
+ # $(Build.SourcesDirectory)/. selfRepoName must be provided in that case.
+ - name: selfRepoPath
+ readonly: true
+ ${{ if eq(parameters.arcadeRepoResource, 'self') }}:
+ value: '$(Build.SourcesDirectory)'
+ ${{ else }}:
+ value: '$(Build.SourcesDirectory)/${{ parameters.selfRepoName }}'
+ - name: arcadeRepoPath
+ readonly: true
+ ${{ if eq(parameters.arcadeRepoResource, 'self') }}:
+ value: '$(Build.SourcesDirectory)'
+ ${{ else }}:
+ value: '$(Build.SourcesDirectory)/${{ parameters.arcadeRepoName }}'
+ pool: ${{ parameters.pool }}
+
+ templateContext:
+ outputParentDirectory: $(Build.ArtifactStagingDirectory)
+ outputs:
+ - output: pipelineArtifact
+ displayName: Publish Renovate Log
+ condition: succeededOrFailed()
+ targetPath: $(Build.ArtifactStagingDirectory)
+ artifactName: $(Agent.JobName)_Logs_Attempt$(System.JobAttempt)
+ isProduction: false # logs are non-production artifacts
+
+ steps:
+ - checkout: self
+ fetchDepth: 1
+
+ - ${{ if ne(parameters.arcadeRepoResource, 'self') }}:
+ - checkout: ${{ parameters.arcadeRepoResource }}
+ fetchDepth: 1
+
+ - script: |
+ renovate-config-validator $(selfRepoPath)/${{parameters.renovateConfigPath}} 2>&1 | tee /tmp/renovate-config-validator.out
+ validatorExit=${PIPESTATUS[0]}
+ if grep -q '^ WARN:' /tmp/renovate-config-validator.out; then
+ echo "##vso[task.logissue type=warning]Renovate config validator produced warnings."
+ echo "##vso[task.complete result=SucceededWithIssues]"
+ fi
+ exit $validatorExit
+ displayName: Validate Renovate config
+ env:
+ LOG_LEVEL: info
+ LOG_FILE_LEVEL: debug
+ LOG_FILE: $(Build.ArtifactStagingDirectory)/renovate-config-validator.json
+
+ - script: |
+ . $(arcadeRepoPath)/eng/common/renovate.env
+ renovate 2>&1 | tee /tmp/renovate.out
+ renovateExit=${PIPESTATUS[0]}
+ if grep -q '^ WARN:' /tmp/renovate.out; then
+ echo "##vso[task.logissue type=warning]Renovate produced warnings."
+ echo "##vso[task.complete result=SucceededWithIssues]"
+ fi
+ exit $renovateExit
+ displayName: Run Renovate
+ env:
+ RENOVATE_FORK_TOKEN: $(BotAccount-dotnet-renovate-bot-PAT)
+ RENOVATE_TOKEN: $(BotAccount-dotnet-renovate-bot-PAT)
+ RENOVATE_REPOSITORIES: ${{parameters.gitHubRepo}}
+ RENOVATE_BASE_BRANCHES: ${{ convertToJson(parameters.baseBranches) }}
+ RENOVATE_DRY_RUN: $(dryRunArg)
+ RENOVATE_RECREATE_WHEN: $(recreateWhenArg)
+ LOG_LEVEL: info
+ LOG_FILE_LEVEL: debug
+ LOG_FILE: $(renovateLogFilePath)
+ RENOVATE_CONFIG_FILE: $(selfRepoPath)/${{parameters.renovateConfigPath}}
+
+ - script: |
+ echo "PRs created by Renovate:"
+ if [ -s "$(renovateLogFilePath)" ]; then
+ if ! jq -r 'select(.msg == "PR created" and .pr != null) | "https://github.com/\(.repository)/pull/\(.pr)"' "$(renovateLogFilePath)" | sort -u; then
+ echo "##vso[task.logissue type=warning]Failed to parse Renovate log file with jq."
+ echo "##vso[task.complete result=SucceededWithIssues]"
+ fi
+ else
+ echo "##vso[task.logissue type=warning]No Renovate log file found or file is empty."
+ echo "##vso[task.complete result=SucceededWithIssues]"
+ fi
+ displayName: List created PRs
+ condition: and(succeededOrFailed(), eq('${{ parameters.dryRun }}', false))
diff --git a/eng/common/core-templates/job/source-build.yml b/eng/common/core-templates/job/source-build.yml
index 05f7ad6ef..1997c2ae0 100644
--- a/eng/common/core-templates/job/source-build.yml
+++ b/eng/common/core-templates/job/source-build.yml
@@ -27,6 +27,8 @@ parameters:
# Specifies the build script to invoke to perform the build in the repo. The default
# './build.sh' should work for typical Arcade repositories, but this is customizable for
# difficult situations.
+ # buildArguments: ''
+ # Specifies additional build arguments to pass to the build script.
# jobProperties: {}
# A list of job properties to inject at the top level, for potential extensibility beyond
# container and pool.
@@ -58,19 +60,19 @@ jobs:
pool:
${{ if eq(variables['System.TeamProject'], 'public') }}:
name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore-Svc-Public' ), False, 'NetCore-Public')]
- demands: ImageOverride -equals build.ubuntu.2004.amd64
+ demands: ImageOverride -equals build.azurelinux.3.amd64.open
${{ if eq(variables['System.TeamProject'], 'internal') }}:
name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore1ESPool-Svc-Internal'), False, 'NetCore1ESPool-Internal')]
- image: 1es-mariner-2
+ image: build.azurelinux.3.amd64
os: linux
${{ else }}:
pool:
${{ if eq(variables['System.TeamProject'], 'public') }}:
name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore-Svc-Public' ), False, 'NetCore-Public')]
- demands: ImageOverride -equals Build.Ubuntu.2204.Amd64.Open
+ demands: ImageOverride -equals build.azurelinux.3.amd64.open
${{ if eq(variables['System.TeamProject'], 'internal') }}:
name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore1ESPool-Svc-Internal'), False, 'NetCore1ESPool-Internal')]
- demands: ImageOverride -equals Build.Ubuntu.2204.Amd64
+ demands: ImageOverride -equals build.azurelinux.3.amd64
${{ if ne(parameters.platform.pool, '') }}:
pool: ${{ parameters.platform.pool }}
diff --git a/eng/common/core-templates/job/source-index-stage1.yml b/eng/common/core-templates/job/source-index-stage1.yml
index 30530359a..bac6ac5fa 100644
--- a/eng/common/core-templates/job/source-index-stage1.yml
+++ b/eng/common/core-templates/job/source-index-stage1.yml
@@ -3,7 +3,7 @@ parameters:
sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci"
preSteps: []
binlogPath: artifacts/log/Debug/Build.binlog
- condition: ''
+ condition: eq(variables['Build.SourceBranch'], 'refs/heads/main')
dependsOn: ''
pool: ''
is1ESPipeline: ''
@@ -15,6 +15,8 @@ jobs:
variables:
- name: BinlogPath
value: ${{ parameters.binlogPath }}
+ - name: skipComponentGovernanceDetection
+ value: true
- template: /eng/common/core-templates/variables/pool-providers.yml
parameters:
is1ESPipeline: ${{ parameters.is1ESPipeline }}
@@ -25,10 +27,10 @@ jobs:
pool:
${{ if eq(variables['System.TeamProject'], 'public') }}:
name: $(DncEngPublicBuildPool)
- image: windows.vs2022.amd64.open
+ image: windows.vs2026.amd64.open
${{ if eq(variables['System.TeamProject'], 'internal') }}:
name: $(DncEngInternalBuildPool)
- image: windows.vs2022.amd64
+ image: windows.vs2026.amd64
steps:
- ${{ if eq(parameters.is1ESPipeline, '') }}:
@@ -41,4 +43,4 @@ jobs:
- template: /eng/common/core-templates/steps/source-index-stage1-publish.yml
parameters:
- binLogPath: ${{ parameters.binLogPath }}
\ No newline at end of file
+ binLogPath: ${{ parameters.binLogPath }}
diff --git a/eng/common/core-templates/jobs/codeql-build.yml b/eng/common/core-templates/jobs/codeql-build.yml
deleted file mode 100644
index f2144252c..000000000
--- a/eng/common/core-templates/jobs/codeql-build.yml
+++ /dev/null
@@ -1,33 +0,0 @@
-parameters:
- # See schema documentation in /Documentation/AzureDevOps/TemplateSchema.md
- continueOnError: false
- # Required: A collection of jobs to run - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job
- jobs: []
- # Optional: if specified, restore and use this version of Guardian instead of the default.
- overrideGuardianVersion: ''
- is1ESPipeline: ''
-
-jobs:
-- template: /eng/common/core-templates/jobs/jobs.yml
- parameters:
- is1ESPipeline: ${{ parameters.is1ESPipeline }}
- enableMicrobuild: false
- enablePublishBuildArtifacts: false
- enablePublishTestResults: false
- enablePublishBuildAssets: false
- enablePublishUsingPipelines: false
- enableTelemetry: true
-
- variables:
- - group: Publish-Build-Assets
- # The Guardian version specified in 'eng/common/sdl/packages.config'. This value must be kept in
- # sync with the packages.config file.
- - name: DefaultGuardianVersion
- value: 0.109.0
- - name: GuardianPackagesConfigFile
- value: $(Build.SourcesDirectory)\eng\common\sdl\packages.config
- - name: GuardianVersion
- value: ${{ coalesce(parameters.overrideGuardianVersion, '$(DefaultGuardianVersion)') }}
-
- jobs: ${{ parameters.jobs }}
-
diff --git a/eng/common/core-templates/jobs/jobs.yml b/eng/common/core-templates/jobs/jobs.yml
index ea69be434..cc8cce452 100644
--- a/eng/common/core-templates/jobs/jobs.yml
+++ b/eng/common/core-templates/jobs/jobs.yml
@@ -5,9 +5,6 @@ parameters:
# Optional: Include PublishBuildArtifacts task
enablePublishBuildArtifacts: false
- # Optional: Enable publishing using release pipelines
- enablePublishUsingPipelines: false
-
# Optional: Enable running the source-build jobs to build repo from source
enableSourceBuild: false
@@ -30,6 +27,9 @@ parameters:
# Optional: Publish the assets as soon as the publish to BAR stage is complete, rather doing so in a separate stage.
publishAssetsImmediately: false
+ # Optional: 🌤️ or not the build has assets it wants to publish to BAR
+ isAssetlessBuild: false
+
# Optional: If using publishAssetsImmediately and additional parameters are needed, can be used to send along additional parameters (normally sent to post-build.yml)
artifactsPublishingAdditionalParameters: ''
signingValidationAdditionalParameters: ''
@@ -44,6 +44,12 @@ parameters:
artifacts: {}
is1ESPipeline: ''
+ # Publishing version w/default.
+ publishingVersion: 3
+
+ repositoryAlias: self
+ officialBuildId: ''
+
# Internal resources (telemetry, microbuild) can only be accessed from non-public projects,
# and some (Microbuild) should only be applied to non-PR cases for internal builds.
@@ -83,7 +89,6 @@ jobs:
- template: /eng/common/core-templates/jobs/source-build.yml
parameters:
is1ESPipeline: ${{ parameters.is1ESPipeline }}
- allCompletedJobId: Source_Build_Complete
${{ each parameter in parameters.sourceBuildParameters }}:
${{ parameter.key }}: ${{ parameter.value }}
@@ -96,11 +101,12 @@ jobs:
${{ parameter.key }}: ${{ parameter.value }}
- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
- - ${{ if or(eq(parameters.enablePublishBuildAssets, true), eq(parameters.artifacts.publish.manifests, 'true'), ne(parameters.artifacts.publish.manifests, '')) }}:
+ - ${{ if or(eq(parameters.enablePublishBuildAssets, true), eq(parameters.artifacts.publish.manifests, 'true'), ne(parameters.artifacts.publish.manifests, ''), eq(parameters.isAssetlessBuild, true)) }}:
- template: ../job/publish-build-assets.yml
parameters:
is1ESPipeline: ${{ parameters.is1ESPipeline }}
continueOnError: ${{ parameters.continueOnError }}
+ publishingVersion: ${{ parameters.publishingVersion }}
dependsOn:
- ${{ if ne(parameters.publishBuildAssetsDependsOn, '') }}:
- ${{ each job in parameters.publishBuildAssetsDependsOn }}:
@@ -108,12 +114,12 @@ jobs:
- ${{ if eq(parameters.publishBuildAssetsDependsOn, '') }}:
- ${{ each job in parameters.jobs }}:
- ${{ job.job }}
- - ${{ if eq(parameters.enableSourceBuild, true) }}:
- - Source_Build_Complete
runAsPublic: ${{ parameters.runAsPublic }}
- publishUsingPipelines: ${{ parameters.enablePublishUsingPipelines }}
- publishAssetsImmediately: ${{ parameters.publishAssetsImmediately }}
+ publishAssetsImmediately: ${{ or(parameters.publishAssetsImmediately, parameters.isAssetlessBuild) }}
+ isAssetlessBuild: ${{ parameters.isAssetlessBuild }}
enablePublishBuildArtifacts: ${{ parameters.enablePublishBuildArtifacts }}
artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
signingValidationAdditionalParameters: ${{ parameters.signingValidationAdditionalParameters }}
+ repositoryAlias: ${{ parameters.repositoryAlias }}
+ officialBuildId: ${{ parameters.officialBuildId }}
diff --git a/eng/common/core-templates/jobs/source-build.yml b/eng/common/core-templates/jobs/source-build.yml
index a10ccfbee..d92860cba 100644
--- a/eng/common/core-templates/jobs/source-build.yml
+++ b/eng/common/core-templates/jobs/source-build.yml
@@ -2,19 +2,13 @@ parameters:
# This template adds arcade-powered source-build to CI. A job is created for each platform, as
# well as an optional server job that completes when all platform jobs complete.
- # The name of the "join" job for all source-build platforms. If set to empty string, the job is
- # not included. Existing repo pipelines can use this job depend on all source-build jobs
- # completing without maintaining a separate list of every single job ID: just depend on this one
- # server job. By default, not included. Recommended name if used: 'Source_Build_Complete'.
- allCompletedJobId: ''
-
# See /eng/common/core-templates/job/source-build.yml
jobNamePrefix: 'Source_Build'
# This is the default platform provided by Arcade, intended for use by a managed-only repo.
defaultManagedPlatform:
name: 'Managed'
- container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9'
+ container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream-10-amd64'
# Defines the platforms on which to run build jobs. One job is created for each platform, and the
# object in this array is sent to the job template as 'platform'. If no platforms are specified,
@@ -31,16 +25,6 @@ parameters:
jobs:
-- ${{ if ne(parameters.allCompletedJobId, '') }}:
- - job: ${{ parameters.allCompletedJobId }}
- displayName: Source-Build Complete
- pool: server
- dependsOn:
- - ${{ each platform in parameters.platforms }}:
- - ${{ parameters.jobNamePrefix }}_${{ platform.name }}
- - ${{ if eq(length(parameters.platforms), 0) }}:
- - ${{ parameters.jobNamePrefix }}_${{ parameters.defaultManagedPlatform.name }}
-
- ${{ each platform in parameters.platforms }}:
- template: /eng/common/core-templates/job/source-build.yml
parameters:
diff --git a/eng/common/core-templates/post-build/common-variables.yml b/eng/common/core-templates/post-build/common-variables.yml
index d5627a994..a3a8480e2 100644
--- a/eng/common/core-templates/post-build/common-variables.yml
+++ b/eng/common/core-templates/post-build/common-variables.yml
@@ -1,6 +1,4 @@
variables:
- - group: Publish-Build-Assets
-
# Whether the build is internal or not
- name: IsInternalBuild
value: ${{ and(ne(variables['System.TeamProject'], 'public'), contains(variables['Build.SourceBranch'], 'internal')) }}
@@ -11,8 +9,6 @@ variables:
- name: MaestroApiVersion
value: "2020-02-20"
- - name: SourceLinkCLIVersion
- value: 3.0.0
- name: SymbolToolVersion
value: 1.0.1
- name: BinlogToolVersion
diff --git a/eng/common/core-templates/post-build/post-build.yml b/eng/common/core-templates/post-build/post-build.yml
index a8c0bd3b9..6dcee6664 100644
--- a/eng/common/core-templates/post-build/post-build.yml
+++ b/eng/common/core-templates/post-build/post-build.yml
@@ -1,112 +1,108 @@
parameters:
- # Which publishing infra should be used. THIS SHOULD MATCH THE VERSION ON THE BUILD MANIFEST.
- # Publishing V1 is no longer supported
- # Publishing V2 is no longer supported
- # Publishing V3 is the default
- - name: publishingInfraVersion
- displayName: Which version of publishing should be used to promote the build definition?
- type: number
- default: 3
- values:
- - 3
-
- - name: BARBuildId
- displayName: BAR Build Id
- type: number
- default: 0
-
- - name: PromoteToChannelIds
- displayName: Channel to promote BARBuildId to
- type: string
- default: ''
-
- - name: enableSourceLinkValidation
- displayName: Enable SourceLink validation
- type: boolean
- default: false
-
- - name: enableSigningValidation
- displayName: Enable signing validation
- type: boolean
- default: true
-
- - name: enableSymbolValidation
- displayName: Enable symbol validation
- type: boolean
- default: false
-
- - name: enableNugetValidation
- displayName: Enable NuGet validation
- type: boolean
- default: true
-
- - name: publishInstallersAndChecksums
- displayName: Publish installers and checksums
- type: boolean
- default: true
-
- - name: requireDefaultChannels
- displayName: Fail the build if there are no default channel(s) registrations for the current build
- type: boolean
- default: false
-
- - name: SDLValidationParameters
- type: object
- default:
- enable: false
- publishGdn: false
- continueOnError: false
- params: ''
- artifactNames: ''
- downloadArtifacts: true
-
- # These parameters let the user customize the call to sdk-task.ps1 for publishing
- # symbols & general artifacts as well as for signing validation
- - name: symbolPublishingAdditionalParameters
- displayName: Symbol publishing additional parameters
- type: string
- default: ''
-
- - name: artifactsPublishingAdditionalParameters
- displayName: Artifact publishing additional parameters
- type: string
- default: ''
-
- - name: signingValidationAdditionalParameters
- displayName: Signing validation additional parameters
- type: string
- default: ''
-
- # Which stages should finish execution before post-build stages start
- - name: validateDependsOn
- type: object
- default:
- - build
-
- - name: publishDependsOn
- type: object
- default:
- - Validate
-
- # Optional: Call asset publishing rather than running in a separate stage
- - name: publishAssetsImmediately
- type: boolean
- default: false
-
- - name: is1ESPipeline
- type: boolean
- default: false
+# Which publishing infra should be used. THIS SHOULD MATCH THE VERSION ON THE BUILD MANIFEST.
+# Publishing V1 is no longer supported
+# Publishing V2 is no longer supported
+# Publishing V3 is the default
+- name: publishingInfraVersion
+ displayName: Which version of publishing should be used to promote the build definition?
+ type: number
+ default: 3
+ values:
+ - 3
+ - 4
+
+- name: BARBuildId
+ displayName: BAR Build Id
+ type: number
+ default: 0
+
+- name: PromoteToChannelIds
+ displayName: Channel to promote BARBuildId to
+ type: string
+ default: ''
+
+- name: enableSourceLinkValidation
+ displayName: Enable SourceLink validation
+ type: boolean
+ default: false
+
+- name: enableSigningValidation
+ displayName: Enable signing validation
+ type: boolean
+ default: true
+
+- name: enableSymbolValidation
+ displayName: Enable symbol validation
+ type: boolean
+ default: false
+
+- name: enableNugetValidation
+ displayName: Enable NuGet validation
+ type: boolean
+ default: true
+
+- name: publishInstallersAndChecksums
+ displayName: Publish installers and checksums
+ type: boolean
+ default: true
+
+- name: requireDefaultChannels
+ displayName: Fail the build if there are no default channel(s) registrations for the current build
+ type: boolean
+ default: false
+
+- name: isAssetlessBuild
+ type: boolean
+ displayName: Is Assetless Build
+ default: false
+
+# These parameters let the user customize the call to sdk-task.ps1 for publishing
+# symbols & general artifacts as well as for signing validation
+- name: symbolPublishingAdditionalParameters
+ displayName: Symbol publishing additional parameters
+ type: string
+ default: ''
+
+- name: artifactsPublishingAdditionalParameters
+ displayName: Artifact publishing additional parameters
+ type: string
+ default: ''
+
+- name: signingValidationAdditionalParameters
+ displayName: Signing validation additional parameters
+ type: string
+ default: ''
+
+# Which stages should finish execution before post-build stages start
+- name: validateDependsOn
+ type: object
+ default:
+ - build
+
+- name: publishDependsOn
+ type: object
+ default:
+ - Validate
+
+# Optional: Call asset publishing rather than running in a separate stage
+- name: publishAssetsImmediately
+ type: boolean
+ default: false
+
+- name: is1ESPipeline
+ type: boolean
+ default: false
stages:
-- ${{ if or(eq( parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}:
+- ${{ if or(eq( parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true')) }}:
- stage: Validate
dependsOn: ${{ parameters.validateDependsOn }}
displayName: Validate Build Assets
variables:
- - template: /eng/common/core-templates/post-build/common-variables.yml
- - template: /eng/common/core-templates/variables/pool-providers.yml
- parameters:
- is1ESPipeline: ${{ parameters.is1ESPipeline }}
+ - template: /eng/common/core-templates/post-build/common-variables.yml
+ - template: /eng/common/core-templates/variables/pool-providers.yml
+ parameters:
+ is1ESPipeline: ${{ parameters.is1ESPipeline }}
jobs:
- job:
displayName: NuGet Validation
@@ -115,26 +111,27 @@ stages:
# We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com)
${{ if eq(variables['System.TeamProject'], 'DevDiv') }}:
name: AzurePipelines-EO
- image: 1ESPT-Windows2022
+ image: 1ESPT-Windows2025
demands: Cmd
os: windows
# If it's not devdiv, it's dnceng
${{ else }}:
${{ if eq(parameters.is1ESPipeline, true) }}:
name: $(DncEngInternalBuildPool)
- image: windows.vs2022.amd64
+ image: windows.vs2026.amd64
os: windows
${{ else }}:
name: $(DncEngInternalBuildPool)
- demands: ImageOverride -equals windows.vs2022.amd64
+ demands: ImageOverride -equals windows.vs2026.amd64
steps:
- - template: /eng/common/core-templates/post-build/setup-maestro-vars.yml
- parameters:
- BARBuildId: ${{ parameters.BARBuildId }}
- PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
- is1ESPipeline: ${{ parameters.is1ESPipeline }}
+ - template: /eng/common/core-templates/post-build/setup-maestro-vars.yml
+ parameters:
+ BARBuildId: ${{ parameters.BARBuildId }}
+ PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
+ is1ESPipeline: ${{ parameters.is1ESPipeline }}
+ - ${{ if ne(parameters.publishingInfraVersion, 4) }}:
- task: DownloadBuildArtifacts@0
displayName: Download Package Artifacts
inputs:
@@ -145,12 +142,25 @@ stages:
buildId: $(AzDOBuildId)
artifactName: PackageArtifacts
checkDownloadedFiles: true
-
- - task: PowerShell@2
- displayName: Validate
+ - ${{ if eq(parameters.publishingInfraVersion, 4) }}:
+ - task: DownloadPipelineArtifact@2
+ displayName: Download Pipeline Artifacts (V4)
+ inputs:
+ itemPattern: '*/packages/**/*.nupkg'
+ targetPath: '$(Build.ArtifactStagingDirectory)/PipelineArtifactsDownload'
+ - task: CopyFiles@2
+ displayName: Flatten packages to PackageArtifacts
inputs:
- filePath: $(Build.SourcesDirectory)/eng/common/post-build/nuget-validation.ps1
- arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/
+ SourceFolder: '$(Build.ArtifactStagingDirectory)/PipelineArtifactsDownload'
+ Contents: '**/*.nupkg'
+ TargetFolder: '$(Build.ArtifactStagingDirectory)/PackageArtifacts'
+ flattenFolders: true
+
+ - task: PowerShell@2
+ displayName: Validate
+ inputs:
+ filePath: $(System.DefaultWorkingDirectory)/eng/common/post-build/nuget-validation.ps1
+ arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/
- job:
displayName: Signing Validation
@@ -159,25 +169,26 @@ stages:
# We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com)
${{ if eq(variables['System.TeamProject'], 'DevDiv') }}:
name: AzurePipelines-EO
- image: 1ESPT-Windows2022
+ image: 1ESPT-Windows2025
demands: Cmd
os: windows
# If it's not devdiv, it's dnceng
${{ else }}:
- ${{ if eq(parameters.is1ESPipeline, true) }}:
+ ${{ if eq(parameters.is1ESPipeline, true) }}:
name: $(DncEngInternalBuildPool)
- image: 1es-windows-2022
+ image: windows.vs2026.amd64
os: windows
${{ else }}:
name: $(DncEngInternalBuildPool)
- demands: ImageOverride -equals windows.vs2022.amd64
+ demands: ImageOverride -equals windows.vs2026.amd64
steps:
- - template: /eng/common/core-templates/post-build/setup-maestro-vars.yml
- parameters:
- BARBuildId: ${{ parameters.BARBuildId }}
- PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
- is1ESPipeline: ${{ parameters.is1ESPipeline }}
+ - template: /eng/common/core-templates/post-build/setup-maestro-vars.yml
+ parameters:
+ BARBuildId: ${{ parameters.BARBuildId }}
+ PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
+ is1ESPipeline: ${{ parameters.is1ESPipeline }}
+ - ${{ if ne(parameters.publishingInfraVersion, 4) }}:
- task: DownloadBuildArtifacts@0
displayName: Download Package Artifacts
inputs:
@@ -188,94 +199,72 @@ stages:
buildId: $(AzDOBuildId)
artifactName: PackageArtifacts
checkDownloadedFiles: true
- itemPattern: |
- **
- !**/Microsoft.SourceBuild.Intermediate.*.nupkg
-
- # This is necessary whenever we want to publish/restore to an AzDO private feed
- # Since sdk-task.ps1 tries to restore packages we need to do this authentication here
- # otherwise it'll complain about accessing a private feed.
- - task: NuGetAuthenticate@1
- displayName: 'Authenticate to AzDO Feeds'
-
- # Signing validation will optionally work with the buildmanifest file which is downloaded from
- # Azure DevOps above.
- - task: PowerShell@2
- displayName: Validate
+ - ${{ if eq(parameters.publishingInfraVersion, 4) }}:
+ - task: DownloadPipelineArtifact@2
+ displayName: Download Pipeline Artifacts (V4)
inputs:
- filePath: eng\common\sdk-task.ps1
- arguments: -task SigningValidation -restore -msbuildEngine vs
- /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts'
- /p:SignCheckExclusionsFile='$(Build.SourcesDirectory)/eng/SignCheckExclusionsFile.txt'
- ${{ parameters.signingValidationAdditionalParameters }}
-
- - template: /eng/common/core-templates/steps/publish-logs.yml
- parameters:
- is1ESPipeline: ${{ parameters.is1ESPipeline }}
- StageLabel: 'Validation'
- JobLabel: 'Signing'
- BinlogToolVersion: $(BinlogToolVersion)
+ itemPattern: '*/packages/**/*.nupkg'
+ targetPath: '$(Build.ArtifactStagingDirectory)/PipelineArtifactsDownload'
+ - task: CopyFiles@2
+ displayName: Flatten packages to PackageArtifacts
+ inputs:
+ SourceFolder: '$(Build.ArtifactStagingDirectory)/PipelineArtifactsDownload'
+ Contents: '**/*.nupkg'
+ TargetFolder: '$(Build.ArtifactStagingDirectory)/PackageArtifacts'
+ flattenFolders: true
- - job:
- displayName: SourceLink Validation
- condition: eq( ${{ parameters.enableSourceLinkValidation }}, 'true')
- pool:
- # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com)
- ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}:
- name: AzurePipelines-EO
- image: 1ESPT-Windows2022
- demands: Cmd
- os: windows
- # If it's not devdiv, it's dnceng
- ${{ else }}:
- ${{ if eq(parameters.is1ESPipeline, true) }}:
- name: $(DncEngInternalBuildPool)
- image: 1es-windows-2022
- os: windows
- ${{ else }}:
- name: $(DncEngInternalBuildPool)
- demands: ImageOverride -equals windows.vs2022.amd64
- steps:
- - template: /eng/common/core-templates/post-build/setup-maestro-vars.yml
- parameters:
- BARBuildId: ${{ parameters.BARBuildId }}
- PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
- is1ESPipeline: ${{ parameters.is1ESPipeline }}
+ # This is necessary whenever we want to publish/restore to an AzDO private feed
+ # Since sdk-task.ps1 tries to restore packages we need to do this authentication here
+ # otherwise it'll complain about accessing a private feed.
+ - task: NuGetAuthenticate@1
+ displayName: 'Authenticate to AzDO Feeds'
- - task: DownloadBuildArtifacts@0
- displayName: Download Blob Artifacts
- inputs:
- buildType: specific
- buildVersionToDownload: specific
- project: $(AzDOProjectName)
- pipeline: $(AzDOPipelineId)
- buildId: $(AzDOBuildId)
- artifactName: BlobArtifacts
- checkDownloadedFiles: true
+ # Signing validation will optionally work with the buildmanifest file which is downloaded from
+ # Azure DevOps above.
+ - task: PowerShell@2
+ displayName: Validate
+ inputs:
+ filePath: eng\common\sdk-task.ps1
+ arguments: -task SigningValidation -restore -msbuildEngine dotnet
+ /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts'
+ /p:SignCheckExclusionsFile='$(System.DefaultWorkingDirectory)/eng/SignCheckExclusionsFile.txt'
+ ${{ parameters.signingValidationAdditionalParameters }}
+
+ - template: /eng/common/core-templates/steps/publish-logs.yml
+ parameters:
+ is1ESPipeline: ${{ parameters.is1ESPipeline }}
+ StageLabel: 'Validation'
+ JobLabel: 'Signing'
+ BinlogToolVersion: $(BinlogToolVersion)
+ enableInternalRuntimes: false
- - task: PowerShell@2
- displayName: Validate
+ # SourceLink validation has been removed — the underlying CLI tool
+ # (targeting netcoreapp2.1) has not functioned for years.
+ # The enableSourceLinkValidation parameter is kept but ignored so
+ # existing pipelines that pass it are not broken.
+ # See https://github.com/dotnet/arcade/issues/16647
+ - ${{ if eq(parameters.enableSourceLinkValidation, 'true') }}:
+ - job:
+ displayName: 'SourceLink Validation Removed - please remove enableSourceLinkValidation from your pipeline'
+ pool: server
+ steps:
+ - task: Delay@1
+ displayName: 'Warning: SourceLink validation removed (see https://github.com/dotnet/arcade/issues/16647)'
inputs:
- filePath: $(Build.SourcesDirectory)/eng/common/post-build/sourcelink-validation.ps1
- arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/
- -ExtractPath $(Agent.BuildDirectory)/Extract/
- -GHRepoName $(Build.Repository.Name)
- -GHCommit $(Build.SourceVersion)
- -SourcelinkCliVersion $(SourceLinkCLIVersion)
- continueOnError: true
+ delayForMinutes: '0'
- ${{ if ne(parameters.publishAssetsImmediately, 'true') }}:
- stage: publish_using_darc
- ${{ if or(eq(parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}:
+ ${{ if or(eq(parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true')) }}:
dependsOn: ${{ parameters.publishDependsOn }}
${{ else }}:
dependsOn: ${{ parameters.validateDependsOn }}
displayName: Publish using Darc
variables:
- - template: /eng/common/core-templates/post-build/common-variables.yml
- - template: /eng/common/core-templates/variables/pool-providers.yml
- parameters:
- is1ESPipeline: ${{ parameters.is1ESPipeline }}
+ - template: /eng/common/core-templates/post-build/common-variables.yml
+ - template: /eng/common/core-templates/variables/pool-providers.yml
+ parameters:
+ is1ESPipeline: ${{ parameters.is1ESPipeline }}
jobs:
- job:
displayName: Publish Using Darc
@@ -284,39 +273,51 @@ stages:
# We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com)
${{ if eq(variables['System.TeamProject'], 'DevDiv') }}:
name: AzurePipelines-EO
- image: 1ESPT-Windows2022
+ image: 1ESPT-Windows2025
demands: Cmd
os: windows
# If it's not devdiv, it's dnceng
${{ else }}:
- ${{ if eq(parameters.is1ESPipeline, true) }}:
+ ${{ if eq(parameters.is1ESPipeline, true) }}:
name: NetCore1ESPool-Publishing-Internal
- image: windows.vs2019.amd64
+ image: windows.vs2026.amd64
os: windows
${{ else }}:
name: NetCore1ESPool-Publishing-Internal
- demands: ImageOverride -equals windows.vs2019.amd64
+ demands: ImageOverride -equals windows.vs2026.amd64
steps:
- - template: /eng/common/core-templates/post-build/setup-maestro-vars.yml
- parameters:
- BARBuildId: ${{ parameters.BARBuildId }}
- PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
- is1ESPipeline: ${{ parameters.is1ESPipeline }}
+ - template: /eng/common/core-templates/post-build/setup-maestro-vars.yml
+ parameters:
+ BARBuildId: ${{ parameters.BARBuildId }}
+ PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
+ is1ESPipeline: ${{ parameters.is1ESPipeline }}
- - task: NuGetAuthenticate@1
+ - task: NuGetAuthenticate@1
- - task: AzureCLI@2
- displayName: Publish Using Darc
- inputs:
- azureSubscription: "Darc: Maestro Production"
- scriptType: ps
- scriptLocation: scriptPath
- scriptPath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1
- arguments: >
+ # Populate internal runtime variables.
+ - template: /eng/common/templates/steps/enable-internal-sources.yml
+
+ - template: /eng/common/templates/steps/enable-internal-runtimes.yml
+
+ - task: UseDotNet@2
+ inputs:
+ version: 8.0.x
+
+ - task: AzureCLI@2
+ displayName: Publish Using Darc
+ inputs:
+ azureSubscription: "Darc: Maestro Production"
+ scriptType: ps
+ scriptLocation: scriptPath
+ scriptPath: $(System.DefaultWorkingDirectory)/eng/common/post-build/publish-using-darc.ps1
+ arguments: >
-BuildId $(BARBuildId)
- -PublishingInfraVersion ${{ parameters.publishingInfraVersion }}
+ -PublishingInfraVersion 3
-AzdoToken '$(System.AccessToken)'
-WaitPublishingFinish true
-RequireDefaultChannels ${{ parameters.requireDefaultChannels }}
-ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}'
-SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}'
+ -SkipAssetsPublishing '${{ parameters.isAssetlessBuild }}'
+ -runtimeSourceFeed https://ci.dot.net/internal
+ -runtimeSourceFeedKey '$(dotnetbuilds-internal-container-read-token-base64)'
diff --git a/eng/common/core-templates/post-build/setup-maestro-vars.yml b/eng/common/core-templates/post-build/setup-maestro-vars.yml
index f7602980d..6dfa99ec5 100644
--- a/eng/common/core-templates/post-build/setup-maestro-vars.yml
+++ b/eng/common/core-templates/post-build/setup-maestro-vars.yml
@@ -8,12 +8,11 @@ steps:
- 'Illegal entry point, is1ESPipeline is not defined. Repository yaml should not directly reference templates in core-templates folder.': error
- ${{ if eq(coalesce(parameters.PromoteToChannelIds, 0), 0) }}:
- - task: DownloadBuildArtifacts@0
+ - task: DownloadPipelineArtifact@2
displayName: Download Release Configs
inputs:
- buildType: current
artifactName: ReleaseConfigs
- checkDownloadedFiles: true
+ targetPath: '$(Build.StagingDirectory)/ReleaseConfigs'
- task: AzureCLI@2
name: setReleaseVars
@@ -36,7 +35,7 @@ steps:
$AzureDevOpsBuildId = $Env:Build_BuildId
}
else {
- . $(Build.SourcesDirectory)\eng\common\tools.ps1
+ . $(System.DefaultWorkingDirectory)\eng\common\tools.ps1
$darc = Get-Darc
$buildInfo = & $darc get-build `
--id ${{ parameters.BARBuildId }} `
diff --git a/eng/common/core-templates/stages/renovate.yml b/eng/common/core-templates/stages/renovate.yml
new file mode 100644
index 000000000..cfa968379
--- /dev/null
+++ b/eng/common/core-templates/stages/renovate.yml
@@ -0,0 +1,113 @@
+# --------------------------------------------------------------------------------------
+# Renovate Pipeline Template
+# --------------------------------------------------------------------------------------
+# This template provides a complete reusable pipeline definition for running Renovate
+# in a 1ES Official pipeline. Pipelines can extend from this template and only need
+# to pass the Renovate job parameters.
+#
+# For more info, see https://github.com/dotnet/arcade/blob/main/Documentation/Renovate.md
+# --------------------------------------------------------------------------------------
+
+parameters:
+
+# Path to the Renovate configuration file within the repository.
+- name: renovateConfigPath
+ type: string
+ default: 'eng/renovate.json'
+
+# GitHub repository to run Renovate against, in the format 'owner/repo'.
+- name: gitHubRepo
+ type: string
+
+# List of base branches to target for Renovate PRs.
+- name: baseBranches
+ type: object
+ default:
+ - main
+
+# When true, Renovate will run in dry run mode.
+- name: dryRun
+ type: boolean
+ default: false
+
+# When true, Renovate will recreate PRs even if they were previously closed.
+- name: forceRecreatePR
+ type: boolean
+ default: false
+
+# Name of the arcade repository resource in the pipeline.
+# This allows repos which haven't been onboarded to Arcade to still use this
+# template by checking out the repo as a resource with a custom name and pointing
+# this parameter to it.
+- name: arcadeRepoResource
+ type: string
+ default: 'self'
+
+- name: selfRepoName
+ type: string
+ default: ''
+- name: arcadeRepoName
+ type: string
+ default: ''
+
+# Pool configuration for the pipeline.
+- name: pool
+ type: object
+ default:
+ name: NetCore1ESPool-Internal
+ image: build.azurelinux.3.amd64
+ os: linux
+
+# Renovate version used in the container image tag.
+- name: renovateVersion
+ default: 43
+ type: number
+
+# Pool configuration for SDL analysis.
+- name: sdlPool
+ type: object
+ default:
+ name: NetCore1ESPool-Internal
+ image: windows.vs2026.amd64
+ os: windows
+
+resources:
+ repositories:
+ - repository: 1ESPipelineTemplates
+ type: git
+ name: 1ESPipelineTemplates/1ESPipelineTemplates
+ ref: refs/tags/release
+
+extends:
+ template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates
+ parameters:
+ settings:
+ networkIsolationPolicy: Permissive
+ pool: ${{ parameters.pool }}
+ sdl:
+ sourceAnalysisPool: ${{ parameters.sdlPool }}
+ # When repos that aren't onboarded to Arcade use this template, they set the
+ # arcadeRepoResource parameter to point to their Arcade repo resource. In that case,
+ # Aracde will be excluded from SDL analysis.
+ ${{ if ne(parameters.arcadeRepoResource, 'self') }}:
+ sourceRepositoriesToScan:
+ exclude:
+ - repository: ${{ parameters.arcadeRepoResource }}
+ containers:
+ RenovateContainer:
+ image: mcr.microsoft.com/dotnet-buildtools/prereqs:azurelinux-3.0-renovate-${{ parameters.renovateVersion }}-amd64
+ stages:
+ - stage: Renovate
+ displayName: Run Renovate
+ jobs:
+ - template: /eng/common/core-templates/job/renovate.yml@${{ parameters.arcadeRepoResource }}
+ parameters:
+ renovateConfigPath: ${{ parameters.renovateConfigPath }}
+ gitHubRepo: ${{ parameters.gitHubRepo }}
+ baseBranches: ${{ parameters.baseBranches }}
+ dryRun: ${{ parameters.dryRun }}
+ forceRecreatePR: ${{ parameters.forceRecreatePR }}
+ pool: ${{ parameters.pool }}
+ arcadeRepoResource: ${{ parameters.arcadeRepoResource }}
+ selfRepoName: ${{ parameters.selfRepoName }}
+ arcadeRepoName: ${{ parameters.arcadeRepoName }}
diff --git a/eng/common/core-templates/steps/component-governance.yml b/eng/common/core-templates/steps/component-governance.yml
deleted file mode 100644
index cf0649aa9..000000000
--- a/eng/common/core-templates/steps/component-governance.yml
+++ /dev/null
@@ -1,16 +0,0 @@
-parameters:
- disableComponentGovernance: false
- componentGovernanceIgnoreDirectories: ''
- is1ESPipeline: false
- displayName: 'Component Detection'
-
-steps:
-- ${{ if eq(parameters.disableComponentGovernance, 'true') }}:
- - script: echo "##vso[task.setvariable variable=skipComponentGovernanceDetection]true"
- displayName: Set skipComponentGovernanceDetection variable
-- ${{ if ne(parameters.disableComponentGovernance, 'true') }}:
- - task: ComponentGovernanceComponentDetection@0
- continueOnError: true
- displayName: ${{ parameters.displayName }}
- inputs:
- ignoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }}
diff --git a/eng/common/core-templates/steps/enable-internal-sources.yml b/eng/common/core-templates/steps/enable-internal-sources.yml
index 64f881bff..843cdff78 100644
--- a/eng/common/core-templates/steps/enable-internal-sources.yml
+++ b/eng/common/core-templates/steps/enable-internal-sources.yml
@@ -15,10 +15,20 @@ steps:
- ${{ if ne(variables['System.TeamProject'], 'public') }}:
- ${{ if ne(parameters.legacyCredential, '') }}:
- task: PowerShell@2
+ condition: and(succeeded(), eq(variables['Agent.Os'], 'Windows_NT'))
displayName: Setup Internal Feeds
inputs:
- filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1
- arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token
+ filePath: $(System.DefaultWorkingDirectory)/eng/common/SetupNugetSources.ps1
+ arguments: -ConfigFile $(System.DefaultWorkingDirectory)/NuGet.config
+ env:
+ Token: ${{ parameters.legacyCredential }}
+ - task: Bash@3
+ condition: and(succeeded(), ne(variables['Agent.Os'], 'Windows_NT'))
+ displayName: Setup Internal Feeds
+ inputs:
+ targetType: inline
+ script: |
+ "$(System.DefaultWorkingDirectory)/eng/common/SetupNugetSources.sh" "$(System.DefaultWorkingDirectory)/NuGet.config"
env:
Token: ${{ parameters.legacyCredential }}
# If running on dnceng (internal project), just use the default behavior for NuGetAuthenticate.
@@ -27,20 +37,38 @@ steps:
- ${{ else }}:
- ${{ if eq(variables['System.TeamProject'], 'internal') }}:
- task: PowerShell@2
+ condition: and(succeeded(), eq(variables['Agent.Os'], 'Windows_NT'))
displayName: Setup Internal Feeds
inputs:
- filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1
- arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config
+ filePath: $(System.DefaultWorkingDirectory)/eng/common/SetupNugetSources.ps1
+ arguments: -ConfigFile $(System.DefaultWorkingDirectory)/NuGet.config
+ - task: Bash@3
+ condition: and(succeeded(), ne(variables['Agent.Os'], 'Windows_NT'))
+ displayName: Setup Internal Feeds
+ inputs:
+ filePath: $(System.DefaultWorkingDirectory)/eng/common/SetupNugetSources.sh
+ arguments: $(System.DefaultWorkingDirectory)/NuGet.config
- ${{ else }}:
- template: /eng/common/templates/steps/get-federated-access-token.yml
parameters:
federatedServiceConnection: ${{ parameters.nugetFederatedServiceConnection }}
outputVariableName: 'dnceng-artifacts-feeds-read-access-token'
- task: PowerShell@2
+ condition: and(succeeded(), eq(variables['Agent.Os'], 'Windows_NT'))
+ displayName: Setup Internal Feeds
+ inputs:
+ filePath: $(System.DefaultWorkingDirectory)/eng/common/SetupNugetSources.ps1
+ arguments: -ConfigFile $(System.DefaultWorkingDirectory)/NuGet.config
+ env:
+ Token: $(dnceng-artifacts-feeds-read-access-token)
+ - task: Bash@3
+ condition: and(succeeded(), ne(variables['Agent.Os'], 'Windows_NT'))
displayName: Setup Internal Feeds
inputs:
- filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1
- arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $(dnceng-artifacts-feeds-read-access-token)
+ filePath: $(System.DefaultWorkingDirectory)/eng/common/SetupNugetSources.sh
+ arguments: $(System.DefaultWorkingDirectory)/NuGet.config
+ env:
+ Token: $(dnceng-artifacts-feeds-read-access-token)
# This is required in certain scenarios to install the ADO credential provider.
# It installed by default in some msbuild invocations (e.g. VS msbuild), but needs to be installed for others
# (e.g. dotnet msbuild).
diff --git a/eng/common/core-templates/steps/generate-sbom.yml b/eng/common/core-templates/steps/generate-sbom.yml
index d938b60e1..aad0a8aed 100644
--- a/eng/common/core-templates/steps/generate-sbom.yml
+++ b/eng/common/core-templates/steps/generate-sbom.yml
@@ -1,54 +1,14 @@
-# BuildDropPath - The root folder of the drop directory for which the manifest file will be generated.
-# PackageName - The name of the package this SBOM represents.
-# PackageVersion - The version of the package this SBOM represents.
-# ManifestDirPath - The path of the directory where the generated manifest files will be placed
-# IgnoreDirectories - Directories to ignore for SBOM generation. This will be passed through to the CG component detector.
-
parameters:
- PackageVersion: 9.0.0
- BuildDropPath: '$(Build.SourcesDirectory)/artifacts'
- PackageName: '.NET'
- ManifestDirPath: $(Build.ArtifactStagingDirectory)/sbom
- IgnoreDirectories: ''
- sbomContinueOnError: true
- is1ESPipeline: false
- # disable publishArtifacts if some other step is publishing the artifacts (like job.yml).
- publishArtifacts: true
+ PackageVersion: unused
+ BuildDropPath: unused
+ PackageName: unused
+ ManifestDirPath: unused
+ IgnoreDirectories: unused
+ sbomContinueOnError: unused
+ is1ESPipeline: unused
+ publishArtifacts: unused
steps:
-- task: PowerShell@2
- displayName: Prep for SBOM generation in (Non-linux)
- condition: or(eq(variables['Agent.Os'], 'Windows_NT'), eq(variables['Agent.Os'], 'Darwin'))
- inputs:
- filePath: ./eng/common/generate-sbom-prep.ps1
- arguments: ${{parameters.manifestDirPath}}
-
-# Chmodding is a workaround for https://github.com/dotnet/arcade/issues/8461
- script: |
- chmod +x ./eng/common/generate-sbom-prep.sh
- ./eng/common/generate-sbom-prep.sh ${{parameters.manifestDirPath}}
- displayName: Prep for SBOM generation in (Linux)
- condition: eq(variables['Agent.Os'], 'Linux')
- continueOnError: ${{ parameters.sbomContinueOnError }}
-
-- task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0
- displayName: 'Generate SBOM manifest'
- continueOnError: ${{ parameters.sbomContinueOnError }}
- inputs:
- PackageName: ${{ parameters.packageName }}
- BuildDropPath: ${{ parameters.buildDropPath }}
- PackageVersion: ${{ parameters.packageVersion }}
- ManifestDirPath: ${{ parameters.manifestDirPath }}
- ${{ if ne(parameters.IgnoreDirectories, '') }}:
- AdditionalComponentDetectorArgs: '--IgnoreDirectories ${{ parameters.IgnoreDirectories }}'
-
-- ${{ if eq(parameters.publishArtifacts, 'true')}}:
- - template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml
- parameters:
- is1ESPipeline: ${{ parameters.is1ESPipeline }}
- args:
- displayName: Publish SBOM manifest
- continueOnError: ${{parameters.sbomContinueOnError}}
- targetPath: '${{ parameters.manifestDirPath }}'
- artifactName: $(ARTIFACT_NAME)
-
+ echo "##vso[task.logissue type=warning]Including generate-sbom.yml is deprecated, SBOM generation is handled 1ES PT now. Remove this include."
+ displayName: Issue generate-sbom.yml deprecation warning
diff --git a/eng/common/core-templates/steps/get-github-app-token.yml b/eng/common/core-templates/steps/get-github-app-token.yml
new file mode 100644
index 000000000..6d42a48d3
--- /dev/null
+++ b/eng/common/core-templates/steps/get-github-app-token.yml
@@ -0,0 +1,79 @@
+# Mints a short-lived GitHub App installation access token by signing a JWT
+# with a private key stored in Azure Key Vault (RSA, RS256). The JWT is
+# exchanged with the GitHub API for a token scoped to a single installation.
+#
+# Requirements (per GitHub App you want to authenticate as):
+# - A GitHub App with its private key uploaded into Key Vault as an RSA key
+# (PEM converted to a key, NOT stored as a secret).
+# - The Azure service connection passed via `azureSubscription` must be
+# granted the `Key Vault Crypto User` role (or at minimum `Sign` action)
+# on that key.
+# - The App must be installed on the target organization/account
+# (`installationOwner`) with the permissions/repositories you need.
+#
+# Output: a secret pipeline variable named ${{ parameters.outputVariableName }}
+# containing the installation access token. Token lifetime is ~1 hour and is
+# automatically scrubbed from logs. Installation tokens are exempt from the
+# enterprise classic-PAT lifetime policy.
+
+parameters:
+# Azure DevOps service connection (federated) that can call
+# `az keyvault key sign` on the App's signing key.
+- name: azureSubscription
+ type: string
+
+# Name of the Key Vault that holds the GitHub App's RSA signing key.
+- name: keyVaultName
+ type: string
+
+# Name of the RSA key inside the Key Vault (the App's private key).
+- name: keyName
+ type: string
+
+# The GitHub App's Client ID (the value to put in the `iss` JWT claim).
+# Prefer this over the numeric App ID; GitHub accepts either, but Client ID
+# is the documented form going forward.
+- name: appClientId
+ type: string
+
+# Login of the organization or user account whose installation we should
+# mint the token for (e.g. `dotnet`, `microsoft`).
+- name: installationOwner
+ type: string
+
+# Name of the pipeline variable that will receive the installation token.
+- name: outputVariableName
+ type: string
+
+- name: is1ESPipeline
+ type: boolean
+
+- name: stepName
+ type: string
+ default: getGitHubAppInstallationToken
+
+- name: condition
+ type: string
+ default: ''
+
+- name: displayName
+ type: string
+ default: Get GitHub App installation token
+
+steps:
+- task: AzureCLI@2
+ displayName: ${{ parameters.displayName }}
+ name: ${{ parameters.stepName }}
+ ${{ if ne(parameters.condition, '') }}:
+ condition: ${{ parameters.condition }}
+ inputs:
+ azureSubscription: ${{ parameters.azureSubscription }}
+ scriptType: pscore
+ scriptLocation: inlineScript
+ inlineScript: |
+ & "$(System.DefaultWorkingDirectory)/eng/common/Get-GitHubAppToken.ps1" `
+ -KeyVaultName '${{ parameters.keyVaultName }}' `
+ -KeyName '${{ parameters.keyName }}' `
+ -AppClientId '${{ parameters.appClientId }}' `
+ -InstallationOwner '${{ parameters.installationOwner }}' `
+ -OutputVariableName '${{ parameters.outputVariableName }}'
diff --git a/eng/common/core-templates/steps/install-microbuild-impl.yml b/eng/common/core-templates/steps/install-microbuild-impl.yml
new file mode 100644
index 000000000..da22beb3f
--- /dev/null
+++ b/eng/common/core-templates/steps/install-microbuild-impl.yml
@@ -0,0 +1,34 @@
+parameters:
+ - name: microbuildTaskInputs
+ type: object
+ default: {}
+
+ - name: microbuildEnv
+ type: object
+ default: {}
+
+ - name: enablePreviewMicrobuild
+ type: boolean
+ default: false
+
+ - name: condition
+ type: string
+
+ - name: continueOnError
+ type: boolean
+
+steps:
+- ${{ if eq(parameters.enablePreviewMicrobuild, true) }}:
+ - task: MicroBuildSigningPluginPreview@4
+ displayName: Install Preview MicroBuild plugin
+ inputs: ${{ parameters.microbuildTaskInputs }}
+ env: ${{ parameters.microbuildEnv }}
+ continueOnError: ${{ parameters.continueOnError }}
+ condition: ${{ parameters.condition }}
+- ${{ else }}:
+ - task: MicroBuildSigningPlugin@4
+ displayName: Install MicroBuild plugin
+ inputs: ${{ parameters.microbuildTaskInputs }}
+ env: ${{ parameters.microbuildEnv }}
+ continueOnError: ${{ parameters.continueOnError }}
+ condition: ${{ parameters.condition }}
diff --git a/eng/common/core-templates/steps/install-microbuild.yml b/eng/common/core-templates/steps/install-microbuild.yml
index 2a6a52948..76a54e157 100644
--- a/eng/common/core-templates/steps/install-microbuild.yml
+++ b/eng/common/core-templates/steps/install-microbuild.yml
@@ -4,70 +4,115 @@ parameters:
# Enable install tasks for MicroBuild on Mac and Linux
# Will be ignored if 'enableMicrobuild' is false or 'Agent.Os' is 'Windows_NT'
enableMicrobuildForMacAndLinux: false
- # Location of the MicroBuild output folder
- microBuildOutputFolder: '$(Agent.TempDirectory)'
+ # Enable preview version of MB signing plugin
+ enablePreviewMicrobuild: false
+ # Determines whether the ESRP service connection information should be passed to the signing plugin.
+ # This overlaps with _SignType to some degree. We only need the service connection for real signing.
+ # It's important that the service connection not be passed to the MicroBuildSigningPlugin task in this place.
+ # Doing so will cause the service connection to be authorized for the pipeline, which isn't allowed and won't work for non-prod.
+ # Unfortunately, _SignType can't be used to exclude the use of the service connection in non-real sign scenarios. The
+ # variable is not available in template expression. _SignType has a very large proliferation across .NET, so replacing it is tough.
+ microbuildUseESRP: true
+ # Microbuild installation directory
+ microBuildOutputFolder: $(Agent.TempDirectory)/MicroBuild
+ # Microbuild version
+ microbuildPluginVersion: 'latest'
+
continueOnError: false
steps:
- ${{ if eq(parameters.enableMicrobuild, 'true') }}:
- ${{ if eq(parameters.enableMicrobuildForMacAndLinux, 'true') }}:
- # Install Python 3.12.x on when Python > 3.12.x is installed - https://github.com/dotnet/source-build/issues/4802
- - script: |
- version=$(python3 --version | awk '{print $2}')
- major=$(echo $version | cut -d. -f1)
- minor=$(echo $version | cut -d. -f2)
-
- installPython=false
- if [ "$major" -gt 3 ] || { [ "$major" -eq 3 ] && [ "$minor" -gt 12 ]; }; then
- installPython=true
- fi
-
- echo "Python version: $version."
- echo "Install Python 3.12.x: $installPython."
- echo "##vso[task.setvariable variable=installPython;isOutput=true]$installPython"
- name: InstallPython
- displayName: 'Determine Python installation'
- condition: and(succeeded(), ne(variables['Agent.Os'], 'Windows_NT'))
-
- - task: UsePythonVersion@0
- inputs:
- versionSpec: '3.12.x'
- displayName: 'Use Python 3.12.x'
- condition: and(succeeded(), eq(variables['InstallPython.installPython'], 'true'), ne(variables['Agent.Os'], 'Windows_NT'))
-
# Needed to download the MicroBuild plugin nupkgs on Mac and Linux when nuget.exe is unavailable
- task: UseDotNet@2
displayName: Install .NET 8.0 SDK for MicroBuild Plugin
inputs:
packageType: sdk
version: 8.0.x
- installationPath: ${{ parameters.microBuildOutputFolder }}/dotnet
- workingDirectory: ${{ parameters.microBuildOutputFolder }}
+ installationPath: ${{ parameters.microBuildOutputFolder }}/.dotnet-microbuild
condition: and(succeeded(), ne(variables['Agent.Os'], 'Windows_NT'))
- - task: MicroBuildSigningPlugin@4
- displayName: Install MicroBuild plugin
- inputs:
- signType: $(_SignType)
- zipSources: false
- feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json
- ${{ if and(eq(parameters.enableMicrobuildForMacAndLinux, 'true'), ne(variables['Agent.Os'], 'Windows_NT')) }}:
- azureSubscription: 'MicroBuild Signing Task (DevDiv)'
- env:
- TeamName: $(_TeamName)
- MicroBuildOutputFolderOverride: ${{ parameters.microBuildOutputFolder }}
- SYSTEM_ACCESSTOKEN: $(System.AccessToken)
- continueOnError: ${{ parameters.continueOnError }}
- condition: and(
- succeeded(),
- or(
- and(
- eq(variables['Agent.Os'], 'Windows_NT'),
- in(variables['_SignType'], 'real', 'test')
- ),
- and(
- ${{ eq(parameters.enableMicrobuildForMacAndLinux, true) }},
- ne(variables['Agent.Os'], 'Windows_NT'),
- eq(variables['_SignType'], 'real')
- )
- ))
+ - script: |
+ set -euo pipefail
+
+ # UseDotNet@2 prepends the dotnet executable path to the PATH variable, so we can call dotnet directly
+ version=$(dotnet --version)
+ cat << 'EOF' > ${{ parameters.microBuildOutputFolder }}/global.json
+ {
+ "sdk": {
+ "version": "$version",
+ "paths": [
+ "${{ parameters.microBuildOutputFolder }}/.dotnet-microbuild"
+ ],
+ "errorMessage": "The .NET SDK version $version is required to install the MicroBuild signing plugin."
+ }
+ }
+ EOF
+ displayName: 'Add global.json to MicroBuild Installation path'
+ workingDirectory: ${{ parameters.microBuildOutputFolder }}
+ condition: and(succeeded(), ne(variables['Agent.Os'], 'Windows_NT'))
+
+ - script: |
+ REM Check if ESRP is disabled while SignType is real
+ if /I "${{ parameters.microbuildUseESRP }}"=="false" if /I "$(_SignType)"=="real" (
+ echo Error: ESRP must be enabled when SignType is real.
+ exit /b 1
+ )
+ displayName: 'Validate ESRP usage (Windows)'
+ condition: and(succeeded(), eq(variables['Agent.Os'], 'Windows_NT'))
+ - script: |
+ # Check if ESRP is disabled while SignType is real
+ if [ "${{ parameters.microbuildUseESRP }}" = "false" ] && [ "$(_SignType)" = "real" ]; then
+ echo "Error: ESRP must be enabled when SignType is real."
+ exit 1
+ fi
+ displayName: 'Validate ESRP usage (Non-Windows)'
+ condition: and(succeeded(), ne(variables['Agent.Os'], 'Windows_NT'))
+
+ # Two different MB install steps. This is due to not being able to use the agent OS during
+ # YAML expansion, and Windows vs. Linux/Mac uses different service connections. However,
+ # we can avoid including the MB install step if not enabled at all. This avoids a bunch of
+ # extra pipeline authorizations, since most pipelines do not sign on non-Windows.
+ - template: /eng/common/core-templates/steps/install-microbuild-impl.yml
+ parameters:
+ enablePreviewMicrobuild: ${{ parameters.enablePreviewMicrobuild }}
+ microbuildTaskInputs:
+ signType: $(_SignType)
+ zipSources: false
+ feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json
+ version: ${{ parameters.microbuildPluginVersion }}
+ ${{ if eq(parameters.microbuildUseESRP, true) }}:
+ ConnectedServiceName: 'MicroBuild Signing Task (DevDiv)'
+ ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}:
+ ConnectedPMEServiceName: 6cc74545-d7b9-4050-9dfa-ebefcc8961ea
+ ${{ else }}:
+ ConnectedPMEServiceName: 248d384a-b39b-46e3-8ad5-c2c210d5e7ca
+ microbuildEnv:
+ TeamName: $(_TeamName)
+ MicroBuildOutputFolderOverride: ${{ parameters.microBuildOutputFolder }}
+ SYSTEM_ACCESSTOKEN: $(System.AccessToken)
+ continueOnError: ${{ parameters.continueOnError }}
+ condition: and(succeeded(), eq(variables['Agent.Os'], 'Windows_NT'), in(variables['_SignType'], 'real', 'test'))
+
+ - ${{ if eq(parameters.enableMicrobuildForMacAndLinux, true) }}:
+ - template: /eng/common/core-templates/steps/install-microbuild-impl.yml
+ parameters:
+ enablePreviewMicrobuild: ${{ parameters.enablePreviewMicrobuild }}
+ microbuildTaskInputs:
+ signType: $(_SignType)
+ zipSources: false
+ feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json
+ version: ${{ parameters.microbuildPluginVersion }}
+ workingDirectory: ${{ parameters.microBuildOutputFolder }}
+ ${{ if eq(parameters.microbuildUseESRP, true) }}:
+ ConnectedServiceName: 'MicroBuild Signing Task (DevDiv)'
+ ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}:
+ ConnectedPMEServiceName: beb8cb23-b303-4c95-ab26-9e44bc958d39
+ ${{ else }}:
+ ConnectedPMEServiceName: c24de2a5-cc7a-493d-95e4-8e5ff5cad2bc
+ microbuildEnv:
+ TeamName: $(_TeamName)
+ MicroBuildOutputFolderOverride: ${{ parameters.microBuildOutputFolder }}
+ SYSTEM_ACCESSTOKEN: $(System.AccessToken)
+ continueOnError: ${{ parameters.continueOnError }}
+ condition: and(succeeded(), ne(variables['Agent.Os'], 'Windows_NT'), eq(variables['_SignType'], 'real'))
diff --git a/eng/common/core-templates/steps/publish-logs.yml b/eng/common/core-templates/steps/publish-logs.yml
index de24d0087..244fef089 100644
--- a/eng/common/core-templates/steps/publish-logs.yml
+++ b/eng/common/core-templates/steps/publish-logs.yml
@@ -5,6 +5,7 @@ parameters:
# A default - in case value from eng/common/core-templates/post-build/common-variables.yml is not passed
BinlogToolVersion: '1.0.11'
is1ESPipeline: false
+ enableInternalRuntimes: true
steps:
- task: Powershell@2
@@ -12,50 +13,52 @@ steps:
inputs:
targetType: inline
script: |
- New-Item -ItemType Directory $(Build.SourcesDirectory)/PostBuildLogs/${{parameters.StageLabel}}/${{parameters.JobLabel}}/
- Move-Item -Path $(Build.SourcesDirectory)/artifacts/log/Debug/* $(Build.SourcesDirectory)/PostBuildLogs/${{parameters.StageLabel}}/${{parameters.JobLabel}}/
+ New-Item -ItemType Directory $(System.DefaultWorkingDirectory)/PostBuildLogs/${{parameters.StageLabel}}/${{parameters.JobLabel}}/
+ Move-Item -Path $(System.DefaultWorkingDirectory)/artifacts/log/Debug/* $(System.DefaultWorkingDirectory)/PostBuildLogs/${{parameters.StageLabel}}/${{parameters.JobLabel}}/
continueOnError: true
condition: always()
- task: PowerShell@2
displayName: Redact Logs
inputs:
- filePath: $(Build.SourcesDirectory)/eng/common/post-build/redact-logs.ps1
+ filePath: $(System.DefaultWorkingDirectory)/eng/common/post-build/redact-logs.ps1
# For now this needs to have explicit list of all sensitive data. Taken from eng/publishing/v3/publish.yml
- # Sensitive data can as well be added to $(Build.SourcesDirectory)/eng/BinlogSecretsRedactionFile.txt'
+ # Sensitive data can as well be added to $(System.DefaultWorkingDirectory)/eng/BinlogSecretsRedactionFile.txt'
# If the file exists - sensitive data for redaction will be sourced from it
# (single entry per line, lines starting with '# ' are considered comments and skipped)
- arguments: -InputPath '$(Build.SourcesDirectory)/PostBuildLogs'
- -BinlogToolVersion ${{parameters.BinlogToolVersion}}
- -TokensFilePath '$(Build.SourcesDirectory)/eng/BinlogSecretsRedactionFile.txt'
- '$(publishing-dnceng-devdiv-code-r-build-re)'
- '$(MaestroAccessToken)'
- '$(dn-bot-all-orgs-artifact-feeds-rw)'
- '$(akams-client-id)'
- '$(microsoft-symbol-server-pat)'
- '$(symweb-symbol-server-pat)'
- '$(dnceng-symbol-server-pat)'
- '$(dn-bot-all-orgs-build-rw-code-rw)'
- '$(System.AccessToken)'
- ${{parameters.CustomSensitiveDataList}}
+ ${{ if and(eq(parameters.enableInternalRuntimes, true), ne(variables['System.TeamProject'], 'public')) }}:
+ arguments: -InputPath '$(System.DefaultWorkingDirectory)/PostBuildLogs'
+ -BinlogToolVersion '${{parameters.BinlogToolVersion}}'
+ -TokensFilePath '$(System.DefaultWorkingDirectory)/eng/BinlogSecretsRedactionFile.txt'
+ -runtimeSourceFeed https://ci.dot.net/internal
+ -runtimeSourceFeedKey '$(dotnetbuilds-internal-container-read-token-base64)'
+ '$(System.AccessToken)'
+ ${{parameters.CustomSensitiveDataList}}
+ ${{ else }}:
+ arguments: -InputPath '$(System.DefaultWorkingDirectory)/PostBuildLogs'
+ -BinlogToolVersion '${{parameters.BinlogToolVersion}}'
+ -TokensFilePath '$(System.DefaultWorkingDirectory)/eng/BinlogSecretsRedactionFile.txt'
+ '$(System.AccessToken)'
+ ${{parameters.CustomSensitiveDataList}}
continueOnError: true
condition: always()
- task: CopyFiles@2
displayName: Gather post build logs
inputs:
- SourceFolder: '$(Build.SourcesDirectory)/PostBuildLogs'
+ SourceFolder: '$(System.DefaultWorkingDirectory)/PostBuildLogs'
Contents: '**'
TargetFolder: '$(Build.ArtifactStagingDirectory)/PostBuildLogs'
condition: always()
-- template: /eng/common/core-templates/steps/publish-build-artifacts.yml
+- template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml
parameters:
is1ESPipeline: ${{ parameters.is1ESPipeline }}
args:
displayName: Publish Logs
- pathToPublish: '$(Build.ArtifactStagingDirectory)/PostBuildLogs'
- publishLocation: Container
- artifactName: PostBuildLogs
+ targetPath: '$(Build.ArtifactStagingDirectory)/PostBuildLogs'
+ artifactName: PostBuildLogs_${{ parameters.StageLabel }}_${{ parameters.JobLabel }}_Attempt$(System.JobAttempt)
continueOnError: true
condition: always()
+ retryCountOnTaskFailure: 10 # for any files being locked
+ isProduction: false # logs are non-production artifacts
diff --git a/eng/common/core-templates/steps/send-to-helix.yml b/eng/common/core-templates/steps/send-to-helix.yml
index 68fa739c4..ec7a20003 100644
--- a/eng/common/core-templates/steps/send-to-helix.yml
+++ b/eng/common/core-templates/steps/send-to-helix.yml
@@ -10,6 +10,7 @@ parameters:
HelixConfiguration: '' # optional -- additional property attached to a job
HelixPreCommands: '' # optional -- commands to run before Helix work item execution
HelixPostCommands: '' # optional -- commands to run after Helix work item execution
+ UseHelixMonitor: false # optional -- true will submit Helix jobs configured for the standalone Helix Job Monitor (results are reported/waited on out-of-band; this step will not wait, and WaitForWorkItemCompletion will be overridden)
WorkItemDirectory: '' # optional -- a payload directory to zip up and send to Helix; requires WorkItemCommand; incompatible with XUnitProjects
WorkItemCommand: '' # optional -- a command to execute on the payload; requires WorkItemDirectory; incompatible with XUnitProjects
WorkItemTimeout: '' # optional -- a timeout in TimeSpan.Parse-ready value (e.g. 00:02:00) for the work item command; requires WorkItemDirectory; incompatible with XUnitProjects
@@ -31,7 +32,15 @@ parameters:
continueOnError: false # optional -- determines whether to continue the build if the step errors; defaults to false
steps:
- - powershell: 'powershell "$env:BUILD_SOURCESDIRECTORY\eng\common\msbuild.ps1 $env:BUILD_SOURCESDIRECTORY/${{ parameters.HelixProjectPath }} /restore /p:TreatWarningsAsErrors=false ${{ parameters.HelixProjectArguments }} /t:Test /bl:$env:BUILD_SOURCESDIRECTORY\artifacts\log\$env:BuildConfig\SendToHelix.binlog"'
+ - powershell: >
+ $(Build.SourcesDirectory)\eng\common\msbuild.ps1
+ $(Build.SourcesDirectory)/${{ parameters.HelixProjectPath }}
+ /restore
+ /p:TreatWarningsAsErrors=false
+ /p:EnableHelixJobMonitor=${{ parameters.UseHelixMonitor }}
+ ${{ parameters.HelixProjectArguments }}
+ /t:Test
+ /bl:$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/SendToHelix.binlog
displayName: ${{ parameters.DisplayNamePrefix }} (Windows)
env:
BuildConfig: $(_BuildConfig)
@@ -61,7 +70,15 @@ steps:
SYSTEM_ACCESSTOKEN: $(System.AccessToken)
condition: and(${{ parameters.condition }}, eq(variables['Agent.Os'], 'Windows_NT'))
continueOnError: ${{ parameters.continueOnError }}
- - script: $BUILD_SOURCESDIRECTORY/eng/common/msbuild.sh $BUILD_SOURCESDIRECTORY/${{ parameters.HelixProjectPath }} /restore /p:TreatWarningsAsErrors=false ${{ parameters.HelixProjectArguments }} /t:Test /bl:$BUILD_SOURCESDIRECTORY/artifacts/log/$BuildConfig/SendToHelix.binlog
+ - script: >
+ $(Build.SourcesDirectory)/eng/common/msbuild.sh
+ $(Build.SourcesDirectory)/${{ parameters.HelixProjectPath }}
+ /restore
+ /p:TreatWarningsAsErrors=false
+ /p:EnableHelixJobMonitor=${{ parameters.UseHelixMonitor }}
+ ${{ parameters.HelixProjectArguments }}
+ /t:Test
+ /bl:$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/SendToHelix.binlog
displayName: ${{ parameters.DisplayNamePrefix }} (Unix)
env:
BuildConfig: $(_BuildConfig)
@@ -91,3 +108,4 @@ steps:
SYSTEM_ACCESSTOKEN: $(System.AccessToken)
condition: and(${{ parameters.condition }}, ne(variables['Agent.Os'], 'Windows_NT'))
continueOnError: ${{ parameters.continueOnError }}
+
diff --git a/eng/common/core-templates/steps/source-build.yml b/eng/common/core-templates/steps/source-build.yml
index 2341706b0..b75f59c42 100644
--- a/eng/common/core-templates/steps/source-build.yml
+++ b/eng/common/core-templates/steps/source-build.yml
@@ -19,19 +19,6 @@ steps:
set -x
df -h
- # If file changes are detected, set CopyWipIntoInnerSourceBuildRepo to copy the WIP changes into the inner source build repo.
- internalRestoreArgs=
- if ! git diff --quiet; then
- internalRestoreArgs='/p:CopyWipIntoInnerSourceBuildRepo=true'
- # The 'Copy WIP' feature of source build uses git stash to apply changes from the original repo.
- # This only works if there is a username/email configured, which won't be the case in most CI runs.
- git config --get user.email
- if [ $? -ne 0 ]; then
- git config user.email dn-bot@microsoft.com
- git config user.name dn-bot
- fi
- fi
-
# If building on the internal project, the internal storage variable may be available (usually only if needed)
# In that case, add variables to allow the download of internal runtimes if the specified versions are not found
# in the default public locations.
@@ -46,36 +33,11 @@ steps:
buildConfig='$(_BuildConfig)'
fi
- officialBuildArgs=
- if [ '${{ and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}' = 'True' ]; then
- officialBuildArgs='/p:DotNetPublishUsingPipelines=true /p:OfficialBuildId=$(BUILD.BUILDNUMBER)'
- fi
-
targetRidArgs=
if [ '${{ parameters.platform.targetRID }}' != '' ]; then
targetRidArgs='/p:TargetRid=${{ parameters.platform.targetRID }}'
fi
- runtimeOsArgs=
- if [ '${{ parameters.platform.runtimeOS }}' != '' ]; then
- runtimeOsArgs='/p:RuntimeOS=${{ parameters.platform.runtimeOS }}'
- fi
-
- baseOsArgs=
- if [ '${{ parameters.platform.baseOS }}' != '' ]; then
- baseOsArgs='/p:BaseOS=${{ parameters.platform.baseOS }}'
- fi
-
- publishArgs=
- if [ '${{ parameters.platform.skipPublishValidation }}' != 'true' ]; then
- publishArgs='--publish'
- fi
-
- assetManifestFileName=SourceBuild_RidSpecific.xml
- if [ '${{ parameters.platform.name }}' != '' ]; then
- assetManifestFileName=SourceBuild_${{ parameters.platform.name }}.xml
- fi
-
portableBuildArgs=
if [ '${{ parameters.platform.portableBuild }}' != '' ]; then
portableBuildArgs='/p:PortableBuild=${{ parameters.platform.portableBuild }}'
@@ -83,51 +45,21 @@ steps:
${{ coalesce(parameters.platform.buildScript, './build.sh') }} --ci \
--configuration $buildConfig \
- --restore --build --pack $publishArgs -bl \
- $officialBuildArgs \
+ --restore --build --pack -bl \
+ --source-build \
+ ${{ parameters.platform.buildArguments }} \
$internalRuntimeDownloadArgs \
- $internalRestoreArgs \
$targetRidArgs \
- $runtimeOsArgs \
- $baseOsArgs \
$portableBuildArgs \
- /p:DotNetBuildSourceOnly=true \
- /p:DotNetBuildRepo=true \
- /p:AssetManifestFileName=$assetManifestFileName
displayName: Build
-# Upload build logs for diagnosis.
-- task: CopyFiles@2
- displayName: Prepare BuildLogs staging directory
- inputs:
- SourceFolder: '$(Build.SourcesDirectory)'
- Contents: |
- **/*.log
- **/*.binlog
- artifacts/sb/prebuilt-report/**
- TargetFolder: '$(Build.StagingDirectory)/BuildLogs'
- CleanTargetFolder: true
- continueOnError: true
- condition: succeededOrFailed()
-
- template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml
parameters:
is1ESPipeline: ${{ parameters.is1ESPipeline }}
args:
displayName: Publish BuildLogs
- targetPath: '$(Build.StagingDirectory)/BuildLogs'
+ targetPath: artifacts/log/${{ coalesce(variables._BuildConfig, 'Release') }}
artifactName: BuildLogs_SourceBuild_${{ parameters.platform.name }}_Attempt$(System.JobAttempt)
continueOnError: true
condition: succeededOrFailed()
- sbomEnabled: false # we don't need SBOM for logs
-
-# Manually inject component detection so that we can ignore the source build upstream cache, which contains
-# a nupkg cache of input packages (a local feed).
-# This path must match the upstream cache path in property 'CurrentRepoSourceBuiltNupkgCacheDir'
-# in src\Microsoft.DotNet.Arcade.Sdk\tools\SourceBuild\SourceBuildArcade.targets
-- template: /eng/common/core-templates/steps/component-governance.yml
- parameters:
- displayName: Component Detection (Exclude upstream cache)
- is1ESPipeline: ${{ parameters.is1ESPipeline }}
- componentGovernanceIgnoreDirectories: '$(Build.SourcesDirectory)/artifacts/sb/src/artifacts/obj/source-built-upstream-cache'
- disableComponentGovernance: ${{ eq(variables['System.TeamProject'], 'public') }}
+ isProduction: false # logs are non-production artifacts
diff --git a/eng/common/core-templates/steps/source-index-stage1-publish.yml b/eng/common/core-templates/steps/source-index-stage1-publish.yml
index 473a22c47..fdca62235 100644
--- a/eng/common/core-templates/steps/source-index-stage1-publish.yml
+++ b/eng/common/core-templates/steps/source-index-stage1-publish.yml
@@ -1,26 +1,26 @@
parameters:
- sourceIndexUploadPackageVersion: 2.0.0-20240522.1
- sourceIndexProcessBinlogPackageVersion: 1.0.1-20240522.1
+ sourceIndexUploadPackageVersion: 2.0.0-20260521.2
+ sourceIndexProcessBinlogPackageVersion: 1.0.1-20260521.2
sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json
binlogPath: artifacts/log/Debug/Build.binlog
steps:
- task: UseDotNet@2
- displayName: "Source Index: Use .NET 8 SDK"
+ displayName: "Source Index: Use .NET 10 SDK"
inputs:
packageType: sdk
- version: 8.0.x
+ version: 10.0.x
installationPath: $(Agent.TempDirectory)/dotnet
workingDirectory: $(Agent.TempDirectory)
- script: |
- $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version ${{parameters.sourceIndexProcessBinlogPackageVersion}} --add-source ${{parameters.SourceIndexPackageSource}} --tool-path $(Agent.TempDirectory)/.source-index/tools
- $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version ${{parameters.sourceIndexUploadPackageVersion}} --add-source ${{parameters.SourceIndexPackageSource}} --tool-path $(Agent.TempDirectory)/.source-index/tools
+ $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version ${{parameters.sourceIndexProcessBinlogPackageVersion}} --source ${{parameters.sourceIndexPackageSource}} --tool-path $(Agent.TempDirectory)/.source-index/tools
+ $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version ${{parameters.sourceIndexUploadPackageVersion}} --source ${{parameters.sourceIndexPackageSource}} --tool-path $(Agent.TempDirectory)/.source-index/tools
displayName: "Source Index: Download netsourceindex Tools"
# Set working directory to temp directory so 'dotnet' doesn't try to use global.json and use the repo's sdk.
workingDirectory: $(Agent.TempDirectory)
-- script: $(Agent.TempDirectory)/.source-index/tools/BinLogToSln -i ${{parameters.BinlogPath}} -r $(Build.SourcesDirectory) -n $(Build.Repository.Name) -o .source-index/stage1output
+- script: $(Agent.TempDirectory)/.source-index/tools/BinLogToSln -i ${{parameters.BinlogPath}} -r $(System.DefaultWorkingDirectory) -n $(Build.Repository.Name) -o .source-index/stage1output
displayName: "Source Index: Process Binlog into indexable sln"
- ${{ if and(ne(parameters.runAsPublic, 'true'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
diff --git a/eng/common/cross/arm64/tizen/tizen.patch b/eng/common/cross/arm64/tizen/tizen.patch
index af7c8be05..2cebc5473 100644
--- a/eng/common/cross/arm64/tizen/tizen.patch
+++ b/eng/common/cross/arm64/tizen/tizen.patch
@@ -5,5 +5,5 @@ diff -u -r a/usr/lib/libc.so b/usr/lib/libc.so
Use the shared library, but some functions are only in
the static library, so try that secondarily. */
OUTPUT_FORMAT(elf64-littleaarch64)
--GROUP ( /lib64/libc.so.6 /usr/lib64/libc_nonshared.a AS_NEEDED ( /lib/ld-linux-aarch64.so.1 ) )
+-GROUP ( /lib64/libc.so.6 /usr/lib64/libc_nonshared.a AS_NEEDED ( /lib64/ld-linux-aarch64.so.1 ) )
+GROUP ( libc.so.6 libc_nonshared.a AS_NEEDED ( ld-linux-aarch64.so.1 ) )
diff --git a/eng/common/cross/armel/armel.jessie.patch b/eng/common/cross/armel/armel.jessie.patch
deleted file mode 100644
index 2d2615619..000000000
--- a/eng/common/cross/armel/armel.jessie.patch
+++ /dev/null
@@ -1,43 +0,0 @@
-diff -u -r a/usr/include/urcu/uatomic/generic.h b/usr/include/urcu/uatomic/generic.h
---- a/usr/include/urcu/uatomic/generic.h 2014-10-22 15:00:58.000000000 -0700
-+++ b/usr/include/urcu/uatomic/generic.h 2020-10-30 21:38:28.550000000 -0700
-@@ -69,10 +69,10 @@
- #endif
- #ifdef UATOMIC_HAS_ATOMIC_SHORT
- case 2:
-- return __sync_val_compare_and_swap_2(addr, old, _new);
-+ return __sync_val_compare_and_swap_2((uint16_t*) addr, old, _new);
- #endif
- case 4:
-- return __sync_val_compare_and_swap_4(addr, old, _new);
-+ return __sync_val_compare_and_swap_4((uint32_t*) addr, old, _new);
- #if (CAA_BITS_PER_LONG == 64)
- case 8:
- return __sync_val_compare_and_swap_8(addr, old, _new);
-@@ -109,7 +109,7 @@
- return;
- #endif
- case 4:
-- __sync_and_and_fetch_4(addr, val);
-+ __sync_and_and_fetch_4((uint32_t*) addr, val);
- return;
- #if (CAA_BITS_PER_LONG == 64)
- case 8:
-@@ -148,7 +148,7 @@
- return;
- #endif
- case 4:
-- __sync_or_and_fetch_4(addr, val);
-+ __sync_or_and_fetch_4((uint32_t*) addr, val);
- return;
- #if (CAA_BITS_PER_LONG == 64)
- case 8:
-@@ -187,7 +187,7 @@
- return __sync_add_and_fetch_2(addr, val);
- #endif
- case 4:
-- return __sync_add_and_fetch_4(addr, val);
-+ return __sync_add_and_fetch_4((uint32_t*) addr, val);
- #if (CAA_BITS_PER_LONG == 64)
- case 8:
- return __sync_add_and_fetch_8(addr, val);
diff --git a/eng/common/cross/build-rootfs.sh b/eng/common/cross/build-rootfs.sh
index 74f399716..f58abbd2d 100644
--- a/eng/common/cross/build-rootfs.sh
+++ b/eng/common/cross/build-rootfs.sh
@@ -5,10 +5,11 @@ set -e
usage()
{
echo "Usage: $0 [BuildArch] [CodeName] [lldbx.y] [llvmx[.y]] [--skipunmount] --rootfsdir ]"
- echo "BuildArch can be: arm(default), arm64, armel, armv6, loongarch64, ppc64le, riscv64, s390x, x64, x86"
+ echo "BuildArch can be: arm(default), arm64, loongarch64, ppc64le, riscv64, s390x, x64, x86"
echo "CodeName - optional, Code name for Linux, can be: xenial(default), zesty, bionic, alpine"
echo " for alpine can be specified with version: alpineX.YY or alpineedge"
- echo " for FreeBSD can be: freebsd13, freebsd14"
+ echo " for FreeBSD can be: freebsd14, freebsd15"
+ echo " for OpenBSD can be: openbsd7.8, openbsd7.9"
echo " for illumos can be: illumos"
echo " for Haiku can be: haiku."
echo "lldbx.y - optional, LLDB version, can be: lldb3.9(default), lldb4.0, lldb5.0, lldb6.0 no-lldb. Ignored for alpine and FreeBSD"
@@ -17,7 +18,10 @@ usage()
echo "--skipsigcheck - optional, will skip package signature checks (allowing untrusted packages)."
echo "--skipemulation - optional, will skip qemu and debootstrap requirement when building environment for debian based systems."
echo "--use-mirror - optional, use mirror URL to fetch resources, when available."
- echo "--jobs N - optional, restrict to N jobs."
+ echo "--ubuntu-repo - optional, override the Ubuntu apt repository base URL."
+ echo "--debian-repo - optional, override the Debian apt repository base URL."
+ echo "--alpine-repo - optional, override the Alpine Linux repository base URL."
+ echo "--jobs N (or --use-jobs N) - optional, restrict to N jobs."
exit 1
}
@@ -27,6 +31,8 @@ __BuildArch=arm
__AlpineArch=armv7
__FreeBSDArch=arm
__FreeBSDMachineArch=armv7
+__OpenBSDArch=arm
+__OpenBSDMachineArch=armv7
__IllumosArch=arm7
__HaikuArch=arm
__QEMUArch=arm
@@ -72,9 +78,9 @@ __AlpinePackages+=" krb5-dev"
__AlpinePackages+=" openssl-dev"
__AlpinePackages+=" zlib-dev"
-__FreeBSDBase="13.4-RELEASE"
-__FreeBSDPkg="1.21.3"
-__FreeBSDABI="13"
+__FreeBSDBase="14.4-RELEASE"
+__FreeBSDPkg="2.8.0"
+__FreeBSDABI="14"
__FreeBSDPackages="libunwind"
__FreeBSDPackages+=" icu"
__FreeBSDPackages+=" libinotify"
@@ -82,6 +88,13 @@ __FreeBSDPackages+=" openssl"
__FreeBSDPackages+=" krb5"
__FreeBSDPackages+=" terminfo-db"
+__OpenBSDVersion="7.8"
+__OpenBSDPackages="heimdal-libs"
+__OpenBSDPackages+=" icu4c"
+__OpenBSDPackages+=" libinotify"
+__OpenBSDPackages+=" openssl"
+__OpenBSDPackages+=" e2fsprogs"
+
__IllumosPackages="icu"
__IllumosPackages+=" mit-krb5"
__IllumosPackages+=" openssl"
@@ -130,11 +143,13 @@ __AlpineKeys='
616db30d:MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAnpUpyWDWjlUk3smlWeA0\nlIMW+oJ38t92CRLHH3IqRhyECBRW0d0aRGtq7TY8PmxjjvBZrxTNDpJT6KUk4LRm\na6A6IuAI7QnNK8SJqM0DLzlpygd7GJf8ZL9SoHSH+gFsYF67Cpooz/YDqWrlN7Vw\ntO00s0B+eXy+PCXYU7VSfuWFGK8TGEv6HfGMALLjhqMManyvfp8hz3ubN1rK3c8C\nUS/ilRh1qckdbtPvoDPhSbTDmfU1g/EfRSIEXBrIMLg9ka/XB9PvWRrekrppnQzP\nhP9YE3x/wbFc5QqQWiRCYyQl/rgIMOXvIxhkfe8H5n1Et4VAorkpEAXdsfN8KSVv\nLSMazVlLp9GYq5SUpqYX3KnxdWBgN7BJoZ4sltsTpHQ/34SXWfu3UmyUveWj7wp0\nx9hwsPirVI00EEea9AbP7NM2rAyu6ukcm4m6ATd2DZJIViq2es6m60AE6SMCmrQF\nwmk4H/kdQgeAELVfGOm2VyJ3z69fQuywz7xu27S6zTKi05Qlnohxol4wVb6OB7qG\nLPRtK9ObgzRo/OPumyXqlzAi/Yvyd1ZQk8labZps3e16bQp8+pVPiumWioMFJDWV\nGZjCmyMSU8V6MB6njbgLHoyg2LCukCAeSjbPGGGYhnKLm1AKSoJh3IpZuqcKCk5C\n8CM1S15HxV78s9dFntEqIokCAwEAAQ==
66ba20fe:MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAtfB12w4ZgqsXWZDfUAV/\n6Y4aHUKIu3q4SXrNZ7CXF9nXoAVYrS7NAxJdAodsY3vPCN0g5O8DFXR+390LdOuQ\n+HsGKCc1k5tX5ZXld37EZNTNSbR0k+NKhd9h6X3u6wqPOx7SIKxwAQR8qeeFq4pP\nrt9GAGlxtuYgzIIcKJPwE0dZlcBCg+GnptCUZXp/38BP1eYC+xTXSL6Muq1etYfg\nodXdb7Yl+2h1IHuOwo5rjgY5kpY7GcAs8AjGk3lDD/av60OTYccknH0NCVSmPoXK\nvrxDBOn0LQRNBLcAfnTKgHrzy0Q5h4TNkkyTgxkoQw5ObDk9nnabTxql732yy9BY\ns+hM9+dSFO1HKeVXreYSA2n1ndF18YAvAumzgyqzB7I4pMHXq1kC/8bONMJxwSkS\nYm6CoXKyavp7RqGMyeVpRC7tV+blkrrUml0BwNkxE+XnwDRB3xDV6hqgWe0XrifD\nYTfvd9ScZQP83ip0r4IKlq4GMv/R5shcCRJSkSZ6QSGshH40JYSoiwJf5FHbj9ND\n7do0UAqebWo4yNx63j/wb2ULorW3AClv0BCFSdPsIrCStiGdpgJDBR2P2NZOCob3\nG9uMj+wJD6JJg2nWqNJxkANXX37Qf8plgzssrhrgOvB0fjjS7GYhfkfmZTJ0wPOw\nA8+KzFseBh4UFGgue78KwgkCAwEAAQ==
'
-__Keyring=
__KeyringFile="/usr/share/keyrings/ubuntu-archive-keyring.gpg"
__SkipSigCheck=0
__SkipEmulation=0
__UseMirror=0
+__UbuntuRepoOverride=
+__DebianRepoOverride=
+__AlpineRepoOverride=
__UnprocessedBuildArgs=
while :; do
@@ -153,6 +168,10 @@ while :; do
__AlpineArch=armv7
__QEMUArch=arm
;;
+ armel)
+ # this is only used for tizen-build-rootfs.sh
+ __BuildArch=armel
+ ;;
arm64)
__BuildArch=arm64
__UbuntuArch=arm64
@@ -160,55 +179,29 @@ while :; do
__QEMUArch=aarch64
__FreeBSDArch=arm64
__FreeBSDMachineArch=aarch64
- ;;
- armel)
- __BuildArch=armel
- __UbuntuArch=armel
- __UbuntuRepo="http://ftp.debian.org/debian/"
- __CodeName=jessie
- __KeyringFile="/usr/share/keyrings/debian-archive-keyring.gpg"
- ;;
- armv6)
- __BuildArch=armv6
- __UbuntuArch=armhf
- __QEMUArch=arm
- __UbuntuRepo="http://raspbian.raspberrypi.org/raspbian/"
- __CodeName=buster
- __KeyringFile="/usr/share/keyrings/raspbian-archive-keyring.gpg"
- __LLDB_Package="liblldb-6.0-dev"
- __UbuntuSuites=
-
- if [[ -e "$__KeyringFile" ]]; then
- __Keyring="--keyring $__KeyringFile"
- fi
+ __OpenBSDArch=arm64
+ __OpenBSDMachineArch=aarch64
;;
loongarch64)
__BuildArch=loongarch64
__AlpineArch=loongarch64
__QEMUArch=loongarch64
__UbuntuArch=loong64
- __UbuntuSuites=unreleased
__LLDB_Package="liblldb-19-dev"
-
- if [[ "$__CodeName" == "sid" ]]; then
- __UbuntuRepo="http://ftp.ports.debian.org/debian-ports/"
- fi
;;
riscv64)
__BuildArch=riscv64
__AlpineArch=riscv64
- __AlpinePackages="${__AlpinePackages// lldb-dev/}"
__QEMUArch=riscv64
__UbuntuArch=riscv64
- __UbuntuPackages="${__UbuntuPackages// libunwind8-dev/}"
- unset __LLDB_Package
+ __LLDB_Package="liblldb-19-dev"
;;
ppc64le)
__BuildArch=ppc64le
__AlpineArch=ppc64le
__QEMUArch=ppc64le
__UbuntuArch=ppc64el
- __UbuntuRepo="http://ports.ubuntu.com/ubuntu-ports/"
+ __UbuntuRepo="https://ports.ubuntu.com/ubuntu-ports/"
__UbuntuPackages="${__UbuntuPackages// libunwind8-dev/}"
__UbuntuPackages="${__UbuntuPackages// libomp-dev/}"
__UbuntuPackages="${__UbuntuPackages// libomp5/}"
@@ -219,7 +212,7 @@ while :; do
__AlpineArch=s390x
__QEMUArch=s390x
__UbuntuArch=s390x
- __UbuntuRepo="http://ports.ubuntu.com/ubuntu-ports/"
+ __UbuntuRepo="https://ports.ubuntu.com/ubuntu-ports/"
__UbuntuPackages="${__UbuntuPackages// libunwind8-dev/}"
__UbuntuPackages="${__UbuntuPackages// libomp-dev/}"
__UbuntuPackages="${__UbuntuPackages// libomp5/}"
@@ -231,15 +224,17 @@ while :; do
__UbuntuArch=amd64
__FreeBSDArch=amd64
__FreeBSDMachineArch=amd64
+ __OpenBSDArch=amd64
+ __OpenBSDMachineArch=amd64
__illumosArch=x86_64
__HaikuArch=x86_64
- __UbuntuRepo="http://archive.ubuntu.com/ubuntu/"
+ __UbuntuRepo="https://archive.ubuntu.com/ubuntu/"
;;
x86)
__BuildArch=x86
__UbuntuArch=i386
__AlpineArch=x86
- __UbuntuRepo="http://archive.ubuntu.com/ubuntu/"
+ __UbuntuRepo="https://archive.ubuntu.com/ubuntu/"
;;
lldb*)
version="$(echo "$lowerI" | tr -d '[:alpha:]-=')"
@@ -278,45 +273,24 @@ while :; do
;;
xenial) # Ubuntu 16.04
- if [[ "$__CodeName" != "jessie" ]]; then
- __CodeName=xenial
- fi
- ;;
- zesty) # Ubuntu 17.04
- if [[ "$__CodeName" != "jessie" ]]; then
- __CodeName=zesty
- fi
+ __CodeName=xenial
;;
bionic) # Ubuntu 18.04
- if [[ "$__CodeName" != "jessie" ]]; then
- __CodeName=bionic
- fi
+ __CodeName=bionic
;;
focal) # Ubuntu 20.04
- if [[ "$__CodeName" != "jessie" ]]; then
- __CodeName=focal
- fi
+ __CodeName=focal
;;
jammy) # Ubuntu 22.04
- if [[ "$__CodeName" != "jessie" ]]; then
- __CodeName=jammy
- fi
+ __CodeName=jammy
;;
noble) # Ubuntu 24.04
- if [[ "$__CodeName" != "jessie" ]]; then
- __CodeName=noble
- fi
- if [[ -n "$__LLDB_Package" ]]; then
- __LLDB_Package="liblldb-18-dev"
- fi
+ __CodeName=noble
+ __LLDB_Package="liblldb-19-dev"
;;
- jessie) # Debian 8
- __CodeName=jessie
- __KeyringFile="/usr/share/keyrings/debian-archive-keyring.gpg"
-
- if [[ -z "$__UbuntuRepo" ]]; then
- __UbuntuRepo="http://ftp.debian.org/debian/"
- fi
+ resolute) # Ubuntu 26.04
+ __CodeName=resolute
+ __LLDB_Package="liblldb-21-dev"
;;
stretch) # Debian 9
__CodeName=stretch
@@ -324,7 +298,7 @@ while :; do
__KeyringFile="/usr/share/keyrings/debian-archive-keyring.gpg"
if [[ -z "$__UbuntuRepo" ]]; then
- __UbuntuRepo="http://ftp.debian.org/debian/"
+ __UbuntuRepo="https://archive.debian.org/debian/"
fi
;;
buster) # Debian 10
@@ -333,7 +307,7 @@ while :; do
__KeyringFile="/usr/share/keyrings/debian-archive-keyring.gpg"
if [[ -z "$__UbuntuRepo" ]]; then
- __UbuntuRepo="http://ftp.debian.org/debian/"
+ __UbuntuRepo="https://archive.debian.org/debian/"
fi
;;
bullseye) # Debian 11
@@ -341,7 +315,7 @@ while :; do
__KeyringFile="/usr/share/keyrings/debian-archive-keyring.gpg"
if [[ -z "$__UbuntuRepo" ]]; then
- __UbuntuRepo="http://ftp.debian.org/debian/"
+ __UbuntuRepo="https://ftp.debian.org/debian/"
fi
;;
bookworm) # Debian 12
@@ -349,7 +323,7 @@ while :; do
__KeyringFile="/usr/share/keyrings/debian-archive-keyring.gpg"
if [[ -z "$__UbuntuRepo" ]]; then
- __UbuntuRepo="http://ftp.debian.org/debian/"
+ __UbuntuRepo="https://ftp.debian.org/debian/"
fi
;;
sid) # Debian sid
@@ -358,25 +332,21 @@ while :; do
# Debian-Ports architectures need different values
case "$__UbuntuArch" in
- amd64|arm64|armel|armhf|i386|mips64el|ppc64el|riscv64|s390x)
+ amd64|arm64|armhf|i386|mips64el|ppc64el|riscv64|loong64|s390x)
__KeyringFile="/usr/share/keyrings/debian-archive-keyring.gpg"
if [[ -z "$__UbuntuRepo" ]]; then
- __UbuntuRepo="http://ftp.debian.org/debian/"
+ __UbuntuRepo="https://ftp.debian.org/debian/"
fi
;;
*)
__KeyringFile="/usr/share/keyrings/debian-ports-archive-keyring.gpg"
if [[ -z "$__UbuntuRepo" ]]; then
- __UbuntuRepo="http://ftp.ports.debian.org/debian-ports/"
+ __UbuntuRepo="https://ftp.debian.org/debian-ports/"
fi
;;
esac
-
- if [[ -e "$__KeyringFile" ]]; then
- __Keyring="--keyring $__KeyringFile"
- fi
;;
tizen)
__CodeName=
@@ -396,14 +366,27 @@ while :; do
__AlpineVersion="$__AlpineMajorVersion.$__AlpineMinorVersion"
fi
;;
- freebsd13)
+ freebsd14)
__CodeName=freebsd
__SkipUnmount=1
;;
- freebsd14)
+ freebsd15)
__CodeName=freebsd
- __FreeBSDBase="14.2-RELEASE"
- __FreeBSDABI="14"
+ __FreeBSDBase="15.1-RELEASE"
+ __FreeBSDABI="15"
+ __SkipUnmount=1
+ ;;
+ openbsd)
+ __CodeName=openbsd
+ __SkipUnmount=1
+ ;;
+ openbsd7.8)
+ __CodeName=openbsd
+ __SkipUnmount=1
+ ;;
+ openbsd7.9)
+ __CodeName=openbsd
+ __OpenBSDVersion="7.9"
__SkipUnmount=1
;;
illumos)
@@ -430,6 +413,31 @@ while :; do
--use-mirror)
__UseMirror=1
;;
+ --ubuntu-repo|-ubuntu-repo)
+ shift
+ if [[ "$#" -le 0 ]]; then
+ echo "ERROR: --ubuntu-repo requires a URL argument."
+ usage
+ fi
+ __UbuntuRepoOverride="$1"
+ ;;
+ --debian-repo|-debian-repo)
+ shift
+ if [[ "$#" -le 0 ]]; then
+ echo "ERROR: --debian-repo requires a URL argument."
+ usage
+ fi
+ __DebianRepoOverride="$1"
+ ;;
+ --alpine-repo|-alpine-repo)
+ shift
+ if [[ "$#" -le 0 ]]; then
+ echo "ERROR: --alpine-repo requires a URL argument."
+ usage
+ fi
+ __AlpineRepoOverride="$1"
+ ;;
+ # Removed duplicate/invalid option handling block (was breaking case statement parsing).
--use-jobs)
shift
MAXJOBS=$1
@@ -455,9 +463,12 @@ case "$__AlpineVersion" in
elif [[ "$__AlpineArch" == "x86" ]]; then
__AlpineVersion=3.17 # minimum version that supports lldb-dev
__AlpinePackages+=" llvm15-libs"
- elif [[ "$__AlpineArch" == "riscv64" || "$__AlpineArch" == "loongarch64" ]]; then
+ elif [[ "$__AlpineArch" == "loongarch64" ]]; then
__AlpineVersion=3.21 # minimum version that supports lldb-dev
__AlpinePackages+=" llvm19-libs"
+ elif [[ "$__AlpineArch" == "riscv64" ]]; then
+ __AlpineVersion=3.22 # lldb-dev requires 3.21+, but 3.22+ provides the newer linux-headers needed for RISC-V extension probes
+ __AlpinePackages+=" llvm20-libs"
elif [[ -n "$__AlpineMajorVersion" ]]; then
# use whichever alpine version is provided and select the latest toolchain libs
__AlpineLlvmLibsLookup=1
@@ -473,14 +484,16 @@ if [[ "$__AlpineVersion" =~ 3\.1[345] ]]; then
__AlpinePackages="${__AlpinePackages/compiler-rt/compiler-rt-static}"
fi
-if [[ "$__BuildArch" == "armel" ]]; then
- __LLDB_Package="lldb-3.5-dev"
-fi
-
__UbuntuPackages+=" ${__LLDB_Package:-}"
if [[ -z "$__UbuntuRepo" ]]; then
- __UbuntuRepo="http://ports.ubuntu.com/"
+ __UbuntuRepo="https://ports.ubuntu.com/"
+fi
+
+if [[ -n "$__UbuntuRepoOverride" && "$__KeyringFile" == *ubuntu* ]]; then
+ __UbuntuRepo="$__UbuntuRepoOverride"
+elif [[ -n "$__DebianRepoOverride" && "$__KeyringFile" == *debian* ]]; then
+ __UbuntuRepo="$__DebianRepoOverride"
fi
if [[ -n "$__LLVM_MajorVersion" ]]; then
@@ -523,6 +536,7 @@ if [[ "$__CodeName" == "alpine" ]]; then
__ApkToolsDir="$(mktemp -d)"
__ApkKeysDir="$(mktemp -d)"
arch="$(uname -m)"
+ __AlpineRepo="${__AlpineRepoOverride:-https://dl-cdn.alpinelinux.org/alpine}"
ensureDownloadTool
@@ -567,15 +581,15 @@ if [[ "$__CodeName" == "alpine" ]]; then
# initialize DB
# shellcheck disable=SC2086
"$__ApkToolsDir/apk.static" \
- -X "http://dl-cdn.alpinelinux.org/alpine/$version/main" \
- -X "http://dl-cdn.alpinelinux.org/alpine/$version/community" \
+ -X "$__AlpineRepo/$version/main" \
+ -X "$__AlpineRepo/$version/community" \
-U $__ApkSignatureArg --root "$__RootfsDir" --arch "$__AlpineArch" --initdb add
if [[ "$__AlpineLlvmLibsLookup" == 1 ]]; then
# shellcheck disable=SC2086
__AlpinePackages+=" $("$__ApkToolsDir/apk.static" \
- -X "http://dl-cdn.alpinelinux.org/alpine/$version/main" \
- -X "http://dl-cdn.alpinelinux.org/alpine/$version/community" \
+ -X "$__AlpineRepo/$version/main" \
+ -X "$__AlpineRepo/$version/community" \
-U $__ApkSignatureArg --root "$__RootfsDir" --arch "$__AlpineArch" \
search 'llvm*-libs' | grep -E '^llvm' | sort | tail -1 | sed 's/-[^-]*//2g')"
fi
@@ -583,8 +597,8 @@ if [[ "$__CodeName" == "alpine" ]]; then
# install all packages in one go
# shellcheck disable=SC2086
"$__ApkToolsDir/apk.static" \
- -X "http://dl-cdn.alpinelinux.org/alpine/$version/main" \
- -X "http://dl-cdn.alpinelinux.org/alpine/$version/community" \
+ -X "$__AlpineRepo/$version/main" \
+ -X "$__AlpineRepo/$version/community" \
-U $__ApkSignatureArg --root "$__RootfsDir" --arch "$__AlpineArch" $__NoEmulationArg \
add $__AlpinePackages
@@ -601,7 +615,7 @@ elif [[ "$__CodeName" == "freebsd" ]]; then
curl -SL "https://download.freebsd.org/ftp/releases/${__FreeBSDArch}/${__FreeBSDMachineArch}/${__FreeBSDBase}/base.txz" | tar -C "$__RootfsDir" -Jxf - ./lib ./usr/lib ./usr/libdata ./usr/include ./usr/share/keys ./etc ./bin/freebsd-version
fi
echo "ABI = \"FreeBSD:${__FreeBSDABI}:${__FreeBSDMachineArch}\"; FINGERPRINTS = \"${__RootfsDir}/usr/share/keys\"; REPOS_DIR = [\"${__RootfsDir}/etc/pkg\"]; REPO_AUTOUPDATE = NO; RUN_SCRIPTS = NO;" > "${__RootfsDir}"/usr/local/etc/pkg.conf
- echo "FreeBSD: { url: \"pkg+http://pkg.FreeBSD.org/\${ABI}/quarterly\", mirror_type: \"srv\", signature_type: \"fingerprints\", fingerprints: \"/usr/share/keys/pkg\", enabled: yes }" > "${__RootfsDir}"/etc/pkg/FreeBSD.conf
+ echo "FreeBSD: { url: \"pkg+https://pkg.FreeBSD.org/\${ABI}/quarterly\", mirror_type: \"srv\", signature_type: \"fingerprints\", fingerprints: \"/usr/share/keys/pkg\", enabled: yes }" > "${__RootfsDir}"/etc/pkg/FreeBSD.conf
mkdir -p "$__RootfsDir"/tmp
# get and build package manager
if [[ "$__hasWget" == 1 ]]; then
@@ -615,9 +629,69 @@ elif [[ "$__CodeName" == "freebsd" ]]; then
./autogen.sh && ./configure --prefix="$__RootfsDir"/host && make -j "$JOBS" && make install
rm -rf "$__RootfsDir/tmp/pkg-${__FreeBSDPkg}"
# install packages we need.
- INSTALL_AS_USER=$(whoami) "$__RootfsDir"/host/sbin/pkg -r "$__RootfsDir" -C "$__RootfsDir"/usr/local/etc/pkg.conf update
+ INSTALL_AS_USER=$(whoami) IGNORE_OSVERSION=yes "$__RootfsDir"/host/sbin/pkg -r "$__RootfsDir" -C "$__RootfsDir"/usr/local/etc/pkg.conf update
# shellcheck disable=SC2086
INSTALL_AS_USER=$(whoami) "$__RootfsDir"/host/sbin/pkg -r "$__RootfsDir" -C "$__RootfsDir"/usr/local/etc/pkg.conf install --yes $__FreeBSDPackages
+elif [[ "$__CodeName" == "openbsd" ]]; then
+ # determine mirrors
+ OPENBSD_MIRROR="https://cdn.openbsd.org/pub/OpenBSD/$__OpenBSDVersion/$__OpenBSDMachineArch"
+
+ # download base system sets
+ ensureDownloadTool
+
+ BASE_SETS=(base comp)
+ for set in "${BASE_SETS[@]}"; do
+ FILE="${set}${__OpenBSDVersion//./}.tgz"
+ echo "Downloading $FILE..."
+ if [[ "$__hasWget" == 1 ]]; then
+ wget -O- "$OPENBSD_MIRROR/$FILE" | tar -C "$__RootfsDir" -xzpf -
+ else
+ curl -SL "$OPENBSD_MIRROR/$FILE" | tar -C "$__RootfsDir" -xzpf -
+ fi
+ done
+
+ PKG_MIRROR="https://cdn.openbsd.org/pub/OpenBSD/${__OpenBSDVersion}/packages/${__OpenBSDMachineArch}"
+
+ echo "Installing packages into sysroot..."
+
+ # Fetch package index once
+ if [[ "$__hasWget" == 1 ]]; then
+ PKG_INDEX=$(wget -qO- "$PKG_MIRROR/")
+ else
+ PKG_INDEX=$(curl -s "$PKG_MIRROR/")
+ fi
+
+ for pkg in $__OpenBSDPackages; do
+ PKG_FILE=$(echo "$PKG_INDEX" | grep -Po ">\K${pkg}-[0-9][^\" ]*\.tgz" \
+ | sort -V | tail -n1)
+
+ echo "Resolved package filename for $pkg: $PKG_FILE"
+
+ [[ -z "$PKG_FILE" ]] && { echo "ERROR: Package $pkg not found"; exit 1; }
+
+ if [[ "$__hasWget" == 1 ]]; then
+ wget -O- "$PKG_MIRROR/$PKG_FILE" | tar -C "$__RootfsDir/usr/local" -xzpf -
+ else
+ curl -SL "$PKG_MIRROR/$PKG_FILE" | tar -C "$__RootfsDir/usr/local" -xzpf -
+ fi
+ done
+
+ echo "Creating versionless symlinks for shared libraries..."
+ # Find all versioned .so files and create the base .so symlink
+ for lib in "$__RootfsDir"/usr/lib/lib*.so.*; do
+ if [ -f "$lib" ]; then
+ # Extract the filename (e.g., libc++.so.12.0)
+ VERSIONED_NAME=$(basename "$lib")
+ # Remove the trailing version numbers (e.g., libc++.so)
+ BASE_NAME=${VERSIONED_NAME%.so.*}.so
+ # Create the symlink in the same directory
+ ln -sf "$VERSIONED_NAME" "$__RootfsDir/usr/lib/$BASE_NAME"
+ fi
+ done
+
+ echo "Cleaning up unnecessary paths"
+ # we don't use executables and kernel in rootfs (as we use host's compiler with -sysroot)
+ rm -rf "$__RootfsDir/usr/share" "$__RootfsDir/usr/bin"
elif [[ "$__CodeName" == "illumos" ]]; then
mkdir "$__RootfsDir/tmp"
pushd "$__RootfsDir/tmp"
@@ -782,6 +856,14 @@ elif [[ "$__CodeName" == "haiku" ]]; then
elif [[ -n "$__CodeName" ]]; then
__Suites="$__CodeName $(for suite in $__UbuntuSuites; do echo -n "$__CodeName-$suite "; done)"
+ __SigCheckArgs=
+ if [[ "$__SkipSigCheck" == "0" ]]; then
+ if [[ -e "$__KeyringFile" ]]; then
+ __SigCheckArgs="--keyring $__KeyringFile"
+ fi
+ __SigCheckArgs="$__SigCheckArgs --force-check-gpg"
+ fi
+
if [[ "$__SkipEmulation" == "1" ]]; then
if [[ -z "$AR" ]]; then
if command -v ar &>/dev/null; then
@@ -797,31 +879,23 @@ elif [[ -n "$__CodeName" ]]; then
PYTHON=${PYTHON_EXECUTABLE:-python3}
# shellcheck disable=SC2086,SC2046
- echo running "$PYTHON" "$__CrossDir/install-debs.py" --arch "$__UbuntuArch" --mirror "$__UbuntuRepo" --rootfsdir "$__RootfsDir" --artool "$AR" \
+ echo running "$PYTHON" "$__CrossDir/install-debs.py" $__SigCheckArgs --arch "$__UbuntuArch" --mirror "$__UbuntuRepo" --rootfsdir "$__RootfsDir" --artool "$AR" \
$(for suite in $__Suites; do echo -n "--suite $suite "; done) \
$__UbuntuPackages
# shellcheck disable=SC2086,SC2046
- "$PYTHON" "$__CrossDir/install-debs.py" --arch "$__UbuntuArch" --mirror "$__UbuntuRepo" --rootfsdir "$__RootfsDir" --artool "$AR" \
+ "$PYTHON" "$__CrossDir/install-debs.py" $__SigCheckArgs --arch "$__UbuntuArch" --mirror "$__UbuntuRepo" --rootfsdir "$__RootfsDir" --artool "$AR" \
$(for suite in $__Suites; do echo -n "--suite $suite "; done) \
$__UbuntuPackages
exit 0
fi
- __UpdateOptions=
- if [[ "$__SkipSigCheck" == "0" ]]; then
- __Keyring="$__Keyring --force-check-gpg"
- else
- __Keyring=
- __UpdateOptions="--allow-unauthenticated --allow-insecure-repositories"
- fi
-
# shellcheck disable=SC2086
- echo running debootstrap "--variant=minbase" $__Keyring --arch "$__UbuntuArch" "$__CodeName" "$__RootfsDir" "$__UbuntuRepo"
+ echo running debootstrap "--variant=minbase" $__SigCheckArgs --arch "$__UbuntuArch" "$__CodeName" "$__RootfsDir" "$__UbuntuRepo"
# shellcheck disable=SC2086
- if ! debootstrap "--variant=minbase" $__Keyring --arch "$__UbuntuArch" "$__CodeName" "$__RootfsDir" "$__UbuntuRepo"; then
+ if ! debootstrap "--variant=minbase" $__SigCheckArgs --arch "$__UbuntuArch" "$__CodeName" "$__RootfsDir" "$__UbuntuRepo"; then
echo "debootstrap failed! dumping debootstrap.log"
cat "$__RootfsDir/debootstrap/debootstrap.log"
exit 1
@@ -839,6 +913,11 @@ Components: main universe
Signed-By: $__KeyringFile
EOF
+ __UpdateOptions=
+ if [[ "$__SkipSigCheck" == "1" ]]; then
+ __UpdateOptions="--allow-unauthenticated --allow-insecure-repositories"
+ fi
+
# shellcheck disable=SC2086
chroot "$__RootfsDir" apt-get update $__UpdateOptions
chroot "$__RootfsDir" apt-get -f -y install
@@ -850,12 +929,6 @@ EOF
if [[ "$__SkipUnmount" == "0" ]]; then
umount "$__RootfsDir"/* || true
fi
-
- if [[ "$__BuildArch" == "armel" && "$__CodeName" == "jessie" ]]; then
- pushd "$__RootfsDir"
- patch -p1 < "$__CrossDir/$__BuildArch/armel.jessie.patch"
- popd
- fi
elif [[ "$__Tizen" == "tizen" ]]; then
ROOTFS_DIR="$__RootfsDir" "$__CrossDir/tizen-build-rootfs.sh" "$__BuildArch"
else
diff --git a/eng/common/cross/install-debs.py b/eng/common/cross/install-debs.py
index c81eb37e5..1d1dfabf7 100644
--- a/eng/common/cross/install-debs.py
+++ b/eng/common/cross/install-debs.py
@@ -4,6 +4,7 @@
import asyncio
import aiohttp
import gzip
+import hashlib
import os
import re
import shutil
@@ -16,7 +17,7 @@
from collections import deque
from functools import cmp_to_key
-async def download_file(session, url, dest_path, max_retries=3, retry_delay=2, timeout=60):
+async def download_file(session, url, dest_path, max_retries=3, retry_delay=2, timeout=60, checksum=None):
"""Asynchronous file download with retries."""
attempt = 0
while attempt < max_retries:
@@ -25,19 +26,25 @@ async def download_file(session, url, dest_path, max_retries=3, retry_delay=2, t
if response.status == 200:
with open(dest_path, "wb") as f:
content = await response.read()
+
+ # verify checksum if provided
+ if checksum:
+ sha256 = hashlib.sha256(content).hexdigest()
+ if sha256 != checksum:
+ raise Exception(f"SHA256 mismatch for {url}: expected {checksum}, got {sha256}")
+
f.write(content)
print(f"Downloaded {url} at {dest_path}")
return
else:
- print(f"Failed to download {url}, Status Code: {response.status}")
- break
+ raise Exception(f"Failed to download {url}, Status Code: {response.status}")
except (asyncio.CancelledError, asyncio.TimeoutError, aiohttp.ClientError) as e:
print(f"Error downloading {url}: {type(e).__name__} - {e}. Retrying...")
attempt += 1
await asyncio.sleep(retry_delay)
- print(f"Failed to download {url} after {max_retries} attempts.")
+ raise Exception(f"Failed to download {url} after {max_retries} attempts.")
async def download_deb_files_parallel(mirror, packages, tmp_dir):
"""Download .deb files in parallel."""
@@ -51,11 +58,11 @@ async def download_deb_files_parallel(mirror, packages, tmp_dir):
if filename:
url = f"{mirror}/{filename}"
dest_path = os.path.join(tmp_dir, os.path.basename(filename))
- tasks.append(asyncio.create_task(download_file(session, url, dest_path)))
+ tasks.append(asyncio.create_task(download_file(session, url, dest_path, checksum=info.get("SHA256"))))
await asyncio.gather(*tasks)
-async def download_package_index_parallel(mirror, arch, suites):
+async def download_package_index_parallel(mirror, arch, suites, check_sig, keyring):
"""Download package index files for specified suites and components entirely in memory."""
tasks = []
timeout = aiohttp.ClientTimeout(total=60)
@@ -63,10 +70,9 @@ async def download_package_index_parallel(mirror, arch, suites):
async with aiohttp.ClientSession(timeout=timeout) as session:
for suite in suites:
for component in ["main", "universe"]:
- url = f"{mirror}/dists/{suite}/{component}/binary-{arch}/Packages.gz"
- tasks.append(fetch_and_decompress(session, url))
+ tasks.append(fetch_and_decompress(session, mirror, arch, suite, component, check_sig, keyring))
- results = await asyncio.gather(*tasks, return_exceptions=True)
+ results = await asyncio.gather(*tasks)
merged_content = ""
for result in results:
@@ -77,20 +83,75 @@ async def download_package_index_parallel(mirror, arch, suites):
return merged_content
-async def fetch_and_decompress(session, url):
+async def fetch_and_decompress(session, mirror, arch, suite, component, check_sig, keyring):
"""Fetch and decompress the Packages.gz file."""
- try:
- async with session.get(url) as response:
- if response.status == 200:
- compressed_data = await response.read()
- decompressed_data = gzip.decompress(compressed_data).decode('utf-8')
- print(f"Downloaded index: {url}")
- return decompressed_data
- else:
- print(f"Skipped index: {url} (doesn't exist)")
- return None
- except Exception as e:
- print(f"Error fetching {url}: {e}")
+
+ path = f"{component}/binary-{arch}/Packages.gz"
+ url = f"{mirror}/dists/{suite}/{path}"
+
+ async with session.get(url) as response:
+ if response.status == 200:
+ compressed_data = await response.read()
+ decompressed_data = gzip.decompress(compressed_data).decode('utf-8')
+ print(f"Downloaded index: {url}")
+
+ if check_sig:
+ # Verify the package index against the sha256 recorded in the Release file
+ release_file_content = await fetch_release_file(session, mirror, suite, keyring)
+ packages_sha = parse_release_file(release_file_content, path)
+
+ sha256 = hashlib.sha256(compressed_data).hexdigest()
+ if sha256 != packages_sha:
+ raise Exception(f"SHA256 mismatch for {path}: expected {packages_sha}, got {sha256}")
+ print(f"Checksum verified for {path}")
+
+ return decompressed_data
+ else:
+ print(f"Skipped index: {url} (doesn't exist)")
+ return None
+
+async def fetch_release_file(session, mirror, suite, keyring):
+ """Fetch Release and Release.gpg files and verify the signature."""
+
+ release_url = f"{mirror}/dists/{suite}/Release"
+ release_gpg_url = f"{mirror}/dists/{suite}/Release.gpg"
+
+ with tempfile.NamedTemporaryFile() as release_file, tempfile.NamedTemporaryFile() as release_gpg_file:
+ await download_file(session, release_url, release_file.name)
+ await download_file(session, release_gpg_url, release_gpg_file.name)
+
+ print("Verifying signature of Release with Release.gpg.")
+ # Use gpgv rather than gpg for verification. gpgv verifies a detached
+ # signature against a fixed keyring without involving gpg-agent or
+ # keyboxd, which makes it robust on hosts running GnuPG 2.4+ (e.g. Azure
+ # Linux) where "gpg --keyring" routes through keyboxd and can fail.
+ verify_command = ["gpgv"]
+ if keyring:
+ verify_command += ["--keyring", keyring]
+ verify_command += [release_gpg_file.name, release_file.name]
+ result = subprocess.run(verify_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+
+ if result.returncode != 0:
+ raise Exception(f"Signature verification failed: {result.stderr.decode('utf-8')}")
+
+ print("Signature verified successfully.")
+
+ with open(release_file.name) as f:
+ return f.read()
+
+def parse_release_file(content, path):
+ """Parses the Release file and returns sha256 checksum of the specified path."""
+
+ # data looks like this:
+ #
+ matches = re.findall(r'^ (\S*) +(\S*) +(\S*)$', content, re.MULTILINE)
+
+ for entry in matches:
+ # the file has both md5 and sha256 checksums, we want sha256 which has a length of 64
+ if entry[2] == path and len(entry[0]) == 64:
+ return entry[0]
+
+ raise Exception(f"Could not find checksum for {path} in Release file.")
def parse_debian_version(version):
"""Parse a Debian package version into epoch, upstream version, and revision."""
@@ -171,13 +232,15 @@ def parse_package_index(content):
filename = fields.get("Filename")
depends = fields.get("Depends")
provides = fields.get("Provides", None)
+ sha256 = fields.get("SHA256")
# Only update if package_name is not in packages or if the new version is higher
if package_name not in packages or compare_debian_versions(version, packages[package_name]["Version"]) > 0:
packages[package_name] = {
"Version": version,
"Filename": filename,
- "Depends": depends
+ "Depends": depends,
+ "SHA256": sha256
}
# Update aliases if package provides any alternatives
@@ -233,7 +296,7 @@ def extract_deb_file(deb_file, tmp_dir, extract_dir, ar_tool):
os.makedirs(extract_dir, exist_ok=True)
with tempfile.TemporaryDirectory(dir=tmp_dir) as tmp_subdir:
- result = subprocess.run(f"{ar_tool} t {os.path.abspath(deb_file)}", cwd=tmp_subdir, check=True, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ result = subprocess.run([ar_tool, "t", os.path.abspath(deb_file)], cwd=tmp_subdir, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
tar_filename = None
for line in result.stdout.decode().splitlines():
@@ -247,7 +310,8 @@ def extract_deb_file(deb_file, tmp_dir, extract_dir, ar_tool):
tar_file_path = os.path.join(tmp_subdir, tar_filename)
print(f"Extracting {tar_filename} from {deb_file}..")
- subprocess.run(f"{ar_tool} p {os.path.abspath(deb_file)} {tar_filename} > {tar_file_path}", check=True, shell=True)
+ with open(tar_file_path, "wb") as outfile:
+ subprocess.run([ar_tool, "p", os.path.abspath(deb_file), tar_filename], check=True, stdout=outfile, stderr=subprocess.PIPE)
file_extension = os.path.splitext(tar_file_path)[1].lower()
@@ -268,7 +332,18 @@ def extract_deb_file(deb_file, tmp_dir, extract_dir, ar_tool):
raise ValueError(f"Unsupported compression format: {file_extension}")
with tarfile.open(tar_file_path, mode) as tar:
- tar.extractall(path=extract_dir, filter='fully_trusted')
+ tar.extractall(path=extract_dir, filter=_rootfs_extraction_filter)
+
+def _rootfs_extraction_filter(member, dest_path):
+ """Tarfile extraction filter based on the 'data' filter that additionally
+ rewrites absolute-target symlinks/hardlinks into rootfs-relative paths.
+ """
+ if (member.issym() or member.islnk()) and os.path.isabs(member.linkname):
+ link_dir = os.path.dirname(member.name)
+ new_linkname = os.path.relpath(member.linkname.lstrip('/'),
+ start=link_dir or '.')
+ member = member.replace(linkname=new_linkname, deep=False)
+ return tarfile.data_filter(member, dest_path)
def finalize_setup(rootfsdir):
lib_dir = os.path.join(rootfsdir, 'lib')
@@ -295,24 +370,17 @@ def finalize_setup(rootfsdir):
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Generate rootfs for .NET runtime on Debian-like OS")
- parser.add_argument("--distro", required=False, help="Distro name (e.g., debian, ubuntu, etc.)")
parser.add_argument("--arch", required=True, help="Architecture (e.g., amd64, loong64, etc.)")
parser.add_argument("--rootfsdir", required=True, help="Destination directory.")
parser.add_argument('--suite', required=True, action='append', help='Specify one or more repository suites to collect index data.')
- parser.add_argument("--mirror", required=False, help="Mirror (e.g., http://ftp.debian.org/debian-ports etc.)")
+ parser.add_argument("--mirror", required=True, help="Mirror (e.g., http://ftp.debian.org/debian-ports etc.)")
parser.add_argument("--artool", required=False, default="ar", help="ar tool to extract debs (e.g., ar, llvm-ar etc.)")
+ parser.add_argument("--force-check-gpg", required=False, action='store_true', help="Verify the packages against signatures in Release file.")
+ parser.add_argument("--keyring", required=False, default='', help="Keyring file to check signature of Release file.")
parser.add_argument("packages", nargs="+", help="List of package names to be installed.")
args = parser.parse_args()
- if args.mirror is None:
- if args.distro == "ubuntu":
- args.mirror = "http://archive.ubuntu.com/ubuntu" if args.arch in ["amd64", "i386"] else "http://ports.ubuntu.com/ubuntu-ports"
- elif args.distro == "debian":
- args.mirror = "http://ftp.debian.org/debian-ports"
- else:
- raise Exception("Unsupported distro")
-
DESIRED_PACKAGES = args.packages + [ # base packages
"dpkg",
"busybox",
@@ -322,9 +390,16 @@ def finalize_setup(rootfsdir):
"debianutils"
]
- print(f"Creating rootfs. rootfsdir: {args.rootfsdir}, distro: {args.distro}, arch: {args.arch}, suites: {args.suite}, mirror: {args.mirror}")
+ print(f"Creating rootfs. rootfsdir: {args.rootfsdir}, arch: {args.arch}, suites: {args.suite}, mirror: {args.mirror}")
+
+ check_sig = args.force_check_gpg
+ if check_sig and not args.keyring:
+ print("ERROR: --force-check-gpg requires --keyring to specify a keyring file for signature verification.")
+ print("Install the appropriate keyring package (e.g., debian-ports-archive-keyring, ubuntu-archive-keyring)")
+ print("or pass --skipsigcheck to build-rootfs.sh to disable signature checking.")
+ sys.exit(1)
- package_index_content = asyncio.run(download_package_index_parallel(args.mirror, args.arch, args.suite))
+ package_index_content = asyncio.run(download_package_index_parallel(args.mirror, args.arch, args.suite, check_sig, args.keyring))
packages_info, aliases = parse_package_index(package_index_content)
diff --git a/eng/common/cross/tizen-fetch.sh b/eng/common/cross/tizen-fetch.sh
index 28936ceef..37c3a61f1 100644
--- a/eng/common/cross/tizen-fetch.sh
+++ b/eng/common/cross/tizen-fetch.sh
@@ -156,13 +156,8 @@ fetch_tizen_pkgs()
done
}
-if [ "$TIZEN_ARCH" == "riscv64" ]; then
- BASE="Tizen-Base-RISCV"
- UNIFIED="Tizen-Unified-RISCV"
-else
- BASE="Tizen-Base"
- UNIFIED="Tizen-Unified"
-fi
+BASE="Tizen-Base"
+UNIFIED="Tizen-Unified"
Inform "Initialize ${TIZEN_ARCH} base"
fetch_tizen_pkgs_init standard $BASE
diff --git a/eng/common/cross/toolchain.cmake b/eng/common/cross/toolchain.cmake
index 0ff85cf03..70b71395e 100644
--- a/eng/common/cross/toolchain.cmake
+++ b/eng/common/cross/toolchain.cmake
@@ -3,15 +3,22 @@ set(CROSS_ROOTFS $ENV{ROOTFS_DIR})
# reset platform variables (e.g. cmake 3.25 sets LINUX=1)
unset(LINUX)
unset(FREEBSD)
+unset(OPENBSD)
unset(ILLUMOS)
unset(ANDROID)
unset(TIZEN)
unset(HAIKU)
set(TARGET_ARCH_NAME $ENV{TARGET_BUILD_ARCH})
+
+file(GLOB OPENBSD_PROBE "${CROSS_ROOTFS}/etc/signify/openbsd-*.pub")
+
if(EXISTS ${CROSS_ROOTFS}/bin/freebsd-version)
set(CMAKE_SYSTEM_NAME FreeBSD)
set(FREEBSD 1)
+elseif(OPENBSD_PROBE)
+ set(CMAKE_SYSTEM_NAME OpenBSD)
+ set(OPENBSD 1)
elseif(EXISTS ${CROSS_ROOTFS}/usr/platform/i86pc)
set(CMAKE_SYSTEM_NAME SunOS)
set(ILLUMOS 1)
@@ -52,7 +59,9 @@ elseif(TARGET_ARCH_NAME STREQUAL "arm64")
set(TIZEN_TOOLCHAIN "aarch64-tizen-linux-gnu")
endif()
elseif(FREEBSD)
- set(triple "aarch64-unknown-freebsd12")
+ set(TOOLCHAIN "aarch64-unknown-freebsd14")
+ elseif(OPENBSD)
+ set(TOOLCHAIN "aarch64-unknown-openbsd")
endif()
elseif(TARGET_ARCH_NAME STREQUAL "armel")
set(CMAKE_SYSTEM_PROCESSOR armv7l)
@@ -78,6 +87,8 @@ elseif(TARGET_ARCH_NAME STREQUAL "ppc64le")
set(CMAKE_SYSTEM_PROCESSOR ppc64le)
if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/powerpc64le-alpine-linux-musl)
set(TOOLCHAIN "powerpc64le-alpine-linux-musl")
+ elseif(FREEBSD)
+ set(TOOLCHAIN "powerpc64le-unknown-freebsd14")
else()
set(TOOLCHAIN "powerpc64le-linux-gnu")
endif()
@@ -108,7 +119,9 @@ elseif(TARGET_ARCH_NAME STREQUAL "x64")
set(TIZEN_TOOLCHAIN "x86_64-tizen-linux-gnu")
endif()
elseif(FREEBSD)
- set(triple "x86_64-unknown-freebsd12")
+ set(TOOLCHAIN "x86_64-unknown-freebsd14")
+ elseif(OPENBSD)
+ set(TOOLCHAIN "x86_64-unknown-openbsd")
elseif(ILLUMOS)
set(TOOLCHAIN "x86_64-illumos")
elseif(HAIKU)
@@ -149,8 +162,6 @@ if(TIZEN)
find_toolchain_dir("${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}")
endif()
- message(STATUS "TIZEN_TOOLCHAIN_PATH set to: ${TIZEN_TOOLCHAIN_PATH}")
-
include_directories(SYSTEM ${TIZEN_TOOLCHAIN_PATH}/include/c++)
include_directories(SYSTEM ${TIZEN_TOOLCHAIN_PATH}/include/c++/${TIZEN_TOOLCHAIN})
endif()
@@ -193,11 +204,11 @@ if(ANDROID)
# include official NDK toolchain script
include(${CROSS_ROOTFS}/../build/cmake/android.toolchain.cmake)
-elseif(FREEBSD)
+elseif(FREEBSD OR OPENBSD)
# we cross-compile by instructing clang
- set(CMAKE_C_COMPILER_TARGET ${triple})
- set(CMAKE_CXX_COMPILER_TARGET ${triple})
- set(CMAKE_ASM_COMPILER_TARGET ${triple})
+ set(CMAKE_C_COMPILER_TARGET ${TOOLCHAIN})
+ set(CMAKE_CXX_COMPILER_TARGET ${TOOLCHAIN})
+ set(CMAKE_ASM_COMPILER_TARGET ${TOOLCHAIN})
set(CMAKE_SYSROOT "${CROSS_ROOTFS}")
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=lld")
set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fuse-ld=lld")
@@ -214,13 +225,19 @@ elseif(ILLUMOS)
locate_toolchain_exec(g++ CMAKE_CXX_COMPILER)
elseif(HAIKU)
set(CMAKE_SYSROOT "${CROSS_ROOTFS}")
- set(CMAKE_PROGRAM_PATH "${CMAKE_PROGRAM_PATH};${CROSS_ROOTFS}/cross-tools-x86_64/bin")
set(CMAKE_SYSTEM_PREFIX_PATH "${CROSS_ROOTFS}")
set(CMAKE_C_STANDARD_LIBRARIES "${CMAKE_C_STANDARD_LIBRARIES} -lssp")
set(CMAKE_CXX_STANDARD_LIBRARIES "${CMAKE_CXX_STANDARD_LIBRARIES} -lssp")
- locate_toolchain_exec(gcc CMAKE_C_COMPILER)
- locate_toolchain_exec(g++ CMAKE_CXX_COMPILER)
+ if ($ENV{CCC_CC} MATCHES ".*gcc.*")
+ set(CMAKE_PROGRAM_PATH "${CMAKE_PROGRAM_PATH};${CROSS_ROOTFS}/cross-tools-x86_64/bin")
+ locate_toolchain_exec(gcc CMAKE_C_COMPILER)
+ locate_toolchain_exec(g++ CMAKE_CXX_COMPILER)
+ else()
+ set(CMAKE_C_COMPILER_EXTERNAL_TOOLCHAIN "${CROSS_ROOTFS}/cross-tools-x86_64")
+ set(CMAKE_CXX_COMPILER_EXTERNAL_TOOLCHAIN "${CROSS_ROOTFS}/cross-tools-x86_64")
+ set(CMAKE_ASM_COMPILER_EXTERNAL_TOOLCHAIN "${CROSS_ROOTFS}/cross-tools-x86_64")
+ endif()
# let CMake set up the correct search paths
include(Platform/Haiku)
@@ -291,7 +308,7 @@ endif()
# Specify compile options
-if((TARGET_ARCH_NAME MATCHES "^(arm|arm64|armel|armv6|loongarch64|ppc64le|riscv64|s390x|x64|x86)$" AND NOT ANDROID AND NOT FREEBSD) OR ILLUMOS OR HAIKU)
+if((TARGET_ARCH_NAME MATCHES "^(arm|arm64|armel|armv6|loongarch64|ppc64le|riscv64|s390x|x64|x86)$" AND NOT ANDROID AND NOT FREEBSD AND NOT OPENBSD) OR ILLUMOS OR HAIKU)
set(CMAKE_C_COMPILER_TARGET ${TOOLCHAIN})
set(CMAKE_CXX_COMPILER_TARGET ${TOOLCHAIN})
set(CMAKE_ASM_COMPILER_TARGET ${TOOLCHAIN})
diff --git a/eng/common/darc-init.ps1 b/eng/common/darc-init.ps1
index e33743105..a5be41db6 100644
--- a/eng/common/darc-init.ps1
+++ b/eng/common/darc-init.ps1
@@ -29,11 +29,11 @@ function InstallDarcCli ($darcVersion, $toolpath) {
Write-Host "Installing Darc CLI version $darcVersion..."
Write-Host 'You may need to restart your command window if this is the first dotnet tool you have installed.'
if (-not $toolpath) {
- Write-Host "'$dotnet' tool install $darcCliPackageName --version $darcVersion --add-source '$arcadeServicesSource' -v $verbosity -g"
- & "$dotnet" tool install $darcCliPackageName --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity -g
+ Write-Host "'$dotnet' tool install $darcCliPackageName --version $darcVersion --source '$arcadeServicesSource' -v $verbosity -g"
+ & "$dotnet" tool install $darcCliPackageName --version $darcVersion --source "$arcadeServicesSource" -v $verbosity -g
}else {
- Write-Host "'$dotnet' tool install $darcCliPackageName --version $darcVersion --add-source '$arcadeServicesSource' -v $verbosity --tool-path '$toolpath'"
- & "$dotnet" tool install $darcCliPackageName --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity --tool-path "$toolpath"
+ Write-Host "'$dotnet' tool install $darcCliPackageName --version $darcVersion --source '$arcadeServicesSource' -v $verbosity --tool-path '$toolpath'"
+ & "$dotnet" tool install $darcCliPackageName --version $darcVersion --source "$arcadeServicesSource" -v $verbosity --tool-path "$toolpath"
}
}
diff --git a/eng/common/darc-init.sh b/eng/common/darc-init.sh
index 36dbd45e1..b56d40e57 100644
--- a/eng/common/darc-init.sh
+++ b/eng/common/darc-init.sh
@@ -5,7 +5,7 @@ darcVersion=''
versionEndpoint='https://maestro.dot.net/api/assets/darc-version?api-version=2020-02-20'
verbosity='minimal'
-while [[ $# > 0 ]]; do
+while [[ $# -gt 0 ]]; do
opt="$(echo "$1" | tr "[:upper:]" "[:lower:]")"
case "$opt" in
--darcversion)
@@ -68,14 +68,14 @@ function InstallDarcCli {
fi
fi
- local arcadeServicesSource="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json"
+ local arcadeServicesSource="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json"
echo "Installing Darc CLI version $darcVersion..."
echo "You may need to restart your command shell if this is the first dotnet tool you have installed."
if [ -z "$toolpath" ]; then
- echo $($dotnet_root/dotnet tool install $darc_cli_package_name --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity -g)
+ echo $($dotnet_root/dotnet tool install $darc_cli_package_name --version $darcVersion --source "$arcadeServicesSource" -v $verbosity -g)
else
- echo $($dotnet_root/dotnet tool install $darc_cli_package_name --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity --tool-path "$toolpath")
+ echo $($dotnet_root/dotnet tool install $darc_cli_package_name --version $darcVersion --source "$arcadeServicesSource" -v $verbosity --tool-path "$toolpath")
fi
}
diff --git a/eng/common/dotnet-install.ps1 b/eng/common/dotnet-install.ps1
index 811f0f717..b6d45f2bd 100644
--- a/eng/common/dotnet-install.ps1
+++ b/eng/common/dotnet-install.ps1
@@ -4,13 +4,20 @@ Param(
[string] $architecture = '',
[string] $version = 'Latest',
[string] $runtime = 'dotnet',
+ [string] $dotnetPath = '',
[string] $RuntimeSourceFeed = '',
[string] $RuntimeSourceFeedKey = ''
)
. $PSScriptRoot\tools.ps1
-$dotnetRoot = Join-Path $RepoRoot '.dotnet'
+if (-not [string]::IsNullOrEmpty($dotnetPath)) {
+ $dotnetRoot = $dotnetPath
+} elseif (-not [string]::IsNullOrEmpty($env:DOTNET_GLOBAL_INSTALL_DIR)) {
+ $dotnetRoot = $env:DOTNET_GLOBAL_INSTALL_DIR
+} else {
+ $dotnetRoot = Join-Path $RepoRoot '.dotnet'
+}
$installdir = $dotnetRoot
try {
diff --git a/eng/common/dotnet-install.sh b/eng/common/dotnet-install.sh
index 7b9d97e3b..58a7e6f38 100644
--- a/eng/common/dotnet-install.sh
+++ b/eng/common/dotnet-install.sh
@@ -16,9 +16,10 @@ scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
version='Latest'
architecture=''
runtime='dotnet'
+dotnetPath=''
runtimeSourceFeed=''
runtimeSourceFeedKey=''
-while [[ $# > 0 ]]; do
+while [[ $# -gt 0 ]]; do
opt="$(echo "$1" | tr "[:upper:]" "[:lower:]")"
case "$opt" in
-version|-v)
@@ -33,6 +34,10 @@ while [[ $# > 0 ]]; do
shift
runtime="$1"
;;
+ -dotnetpath)
+ shift
+ dotnetPath="$1"
+ ;;
-runtimesourcefeed)
shift
runtimeSourceFeed="$1"
@@ -80,7 +85,13 @@ case $cpuname in
;;
esac
-dotnetRoot="${repo_root}.dotnet"
+if [[ -n "${dotnetPath:-}" ]]; then
+ dotnetRoot="$dotnetPath"
+elif [[ -n "${DOTNET_GLOBAL_INSTALL_DIR:-}" ]]; then
+ dotnetRoot="$DOTNET_GLOBAL_INSTALL_DIR"
+else
+ dotnetRoot="${repo_root}.dotnet"
+fi
if [[ $architecture != "" ]] && [[ $architecture != $buildarch ]]; then
dotnetRoot="$dotnetRoot/$architecture"
fi
diff --git a/eng/common/dotnet.cmd b/eng/common/dotnet.cmd
new file mode 100644
index 000000000..527fa4bb3
--- /dev/null
+++ b/eng/common/dotnet.cmd
@@ -0,0 +1,7 @@
+@echo off
+
+:: This script is used to install the .NET SDK.
+:: It will also invoke the SDK with any provided arguments.
+
+powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0dotnet.ps1""" %*"
+exit /b %ErrorLevel%
diff --git a/eng/common/dotnet.ps1 b/eng/common/dotnet.ps1
new file mode 100644
index 000000000..ce4ea4073
--- /dev/null
+++ b/eng/common/dotnet.ps1
@@ -0,0 +1,12 @@
+# This script is used to install the .NET SDK.
+# It will also invoke the SDK with any provided arguments.
+
+. $PSScriptRoot\tools.ps1
+$dotnetRoot = InitializeDotNetCli -install:$true
+
+# Invoke acquired SDK with args if they are provided
+if ($args.count -gt 0) {
+ $env:DOTNET_NOLOGO=1
+ & "$dotnetRoot\dotnet.exe" $args
+ ExitWithExitCode $LASTEXITCODE
+}
diff --git a/eng/common/dotnet.sh b/eng/common/dotnet.sh
new file mode 100644
index 000000000..f6d24871c
--- /dev/null
+++ b/eng/common/dotnet.sh
@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+
+# This script is used to install the .NET SDK.
+# It will also invoke the SDK with any provided arguments.
+
+source="${BASH_SOURCE[0]}"
+# resolve $SOURCE until the file is no longer a symlink
+while [[ -h $source ]]; do
+ scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
+ source="$(readlink "$source")"
+
+ # if $source was a relative symlink, we need to resolve it relative to the path where the
+ # symlink file was located
+ [[ $source != /* ]] && source="$scriptroot/$source"
+done
+scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
+
+source $scriptroot/tools.sh
+InitializeDotNetCli true # install
+
+# Invoke acquired SDK with args if they are provided
+if [[ $# -gt 0 ]]; then
+ __dotnetDir=${_InitializeDotNetCli}
+ dotnetPath=${__dotnetDir}/dotnet
+ ${dotnetPath} "$@"
+fi
diff --git a/eng/common/generate-locproject.ps1 b/eng/common/generate-locproject.ps1
index 524aaa57f..fa1cdc2b3 100644
--- a/eng/common/generate-locproject.ps1
+++ b/eng/common/generate-locproject.ps1
@@ -33,15 +33,27 @@ $jsonTemplateFiles | ForEach-Object {
$jsonWinformsTemplateFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "en\\strings\.json" } # current winforms pattern
+$wxlFilesV3 = @()
+$wxlFilesV5 = @()
$wxlFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "\\.+\.wxl" -And -Not( $_.Directory.Name -Match "\d{4}" ) } # localized files live in four digit lang ID directories; this excludes them
if (-not $wxlFiles) {
$wxlEnFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "\\1033\\.+\.wxl" } # pick up en files (1033 = en) specifically so we can copy them to use as the neutral xlf files
if ($wxlEnFiles) {
- $wxlFiles = @()
- $wxlEnFiles | ForEach-Object {
- $destinationFile = "$($_.Directory.Parent.FullName)\$($_.Name)"
- $wxlFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru
- }
+ $wxlFiles = @()
+ $wxlEnFiles | ForEach-Object {
+ $destinationFile = "$($_.Directory.Parent.FullName)\$($_.Name)"
+ $content = Get-Content $_.FullName -Raw
+
+ # Split files on schema to select different parser settings in the generated project.
+ if ($content -like "*http://wixtoolset.org/schemas/v4/wxl*")
+ {
+ $wxlFilesV5 += Copy-Item $_.FullName -Destination $destinationFile -PassThru
+ }
+ elseif ($content -like "*http://schemas.microsoft.com/wix/2006/localization*")
+ {
+ $wxlFilesV3 += Copy-Item $_.FullName -Destination $destinationFile -PassThru
+ }
+ }
}
}
@@ -114,7 +126,32 @@ $locJson = @{
CloneLanguageSet = "WiX_CloneLanguages"
LssFiles = @( "wxl_loc.lss" )
LocItems = @(
- $wxlFiles | ForEach-Object {
+ $wxlFilesV3 | ForEach-Object {
+ $outputPath = "$($_.Directory.FullName | Resolve-Path -Relative)\"
+ $continue = $true
+ foreach ($exclusion in $exclusions.Exclusions) {
+ if ($_.FullName.Contains($exclusion)) {
+ $continue = $false
+ }
+ }
+ $sourceFile = ($_.FullName | Resolve-Path -Relative)
+ if ($continue)
+ {
+ return @{
+ SourceFile = $sourceFile
+ CopyOption = "LangIDOnPath"
+ OutputPath = $outputPath
+ }
+ }
+ }
+ )
+ },
+ @{
+ LanguageSet = $LanguageSet
+ CloneLanguageSet = "WiX_CloneLanguages"
+ LssFiles = @( "P210WxlSchemaV4.lss" )
+ LocItems = @(
+ $wxlFilesV5 | ForEach-Object {
$outputPath = "$($_.Directory.FullName | Resolve-Path -Relative)\"
$continue = $true
foreach ($exclusion in $exclusions.Exclusions) {
diff --git a/eng/common/generate-sbom-prep.ps1 b/eng/common/generate-sbom-prep.ps1
deleted file mode 100644
index 3e5c1c74a..000000000
--- a/eng/common/generate-sbom-prep.ps1
+++ /dev/null
@@ -1,21 +0,0 @@
-Param(
- [Parameter(Mandatory=$true)][string] $ManifestDirPath # Manifest directory where sbom will be placed
-)
-
-. $PSScriptRoot\pipeline-logging-functions.ps1
-
-Write-Host "Creating dir $ManifestDirPath"
-# create directory for sbom manifest to be placed
-if (!(Test-Path -path $ManifestDirPath))
-{
- New-Item -ItemType Directory -path $ManifestDirPath
- Write-Host "Successfully created directory $ManifestDirPath"
-}
-else{
- Write-PipelineTelemetryError -category 'Build' "Unable to create sbom folder."
-}
-
-Write-Host "Updating artifact name"
-$artifact_name = "${env:SYSTEM_STAGENAME}_${env:AGENT_JOBNAME}_SBOM" -replace '["/:<>\\|?@*"() ]', '_'
-Write-Host "Artifact name $artifact_name"
-Write-Host "##vso[task.setvariable variable=ARTIFACT_NAME]$artifact_name"
diff --git a/eng/common/generate-sbom-prep.sh b/eng/common/generate-sbom-prep.sh
deleted file mode 100644
index d5c76dc82..000000000
--- a/eng/common/generate-sbom-prep.sh
+++ /dev/null
@@ -1,34 +0,0 @@
-#!/usr/bin/env bash
-
-source="${BASH_SOURCE[0]}"
-
-# resolve $SOURCE until the file is no longer a symlink
-while [[ -h $source ]]; do
- scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
- source="$(readlink "$source")"
-
- # if $source was a relative symlink, we need to resolve it relative to the path where the
- # symlink file was located
- [[ $source != /* ]] && source="$scriptroot/$source"
-done
-scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
-. $scriptroot/pipeline-logging-functions.sh
-
-manifest_dir=$1
-
-if [ ! -d "$manifest_dir" ] ; then
- mkdir -p "$manifest_dir"
- echo "Sbom directory created." $manifest_dir
-else
- Write-PipelineTelemetryError -category 'Build' "Unable to create sbom folder."
-fi
-
-artifact_name=$SYSTEM_STAGENAME"_"$AGENT_JOBNAME"_SBOM"
-echo "Artifact name before : "$artifact_name
-# replace all special characters with _, some builds use special characters like : in Agent.Jobname, that is not a permissible name while uploading artifacts.
-safe_artifact_name="${artifact_name//["/:<>\\|?@*$" ]/_}"
-echo "Artifact name after : "$safe_artifact_name
-export ARTIFACT_NAME=$safe_artifact_name
-echo "##vso[task.setvariable variable=ARTIFACT_NAME]$safe_artifact_name"
-
-exit 0
diff --git a/eng/common/internal-feed-operations.ps1 b/eng/common/internal-feed-operations.ps1
index 92b77347d..c282d3ae4 100644
--- a/eng/common/internal-feed-operations.ps1
+++ b/eng/common/internal-feed-operations.ps1
@@ -26,7 +26,7 @@ function SetupCredProvider {
$url = 'https://raw.githubusercontent.com/microsoft/artifacts-credprovider/master/helpers/installcredprovider.ps1'
Write-Host "Writing the contents of 'installcredprovider.ps1' locally..."
- Invoke-WebRequest $url -OutFile installcredprovider.ps1
+ Invoke-WebRequest $url -UseBasicParsing -OutFile installcredprovider.ps1
Write-Host 'Installing plugin...'
.\installcredprovider.ps1 -Force
diff --git a/eng/common/internal-feed-operations.sh b/eng/common/internal-feed-operations.sh
index 9378223ba..6299e7eff 100644
--- a/eng/common/internal-feed-operations.sh
+++ b/eng/common/internal-feed-operations.sh
@@ -100,7 +100,7 @@ operation=''
authToken=''
repoName=''
-while [[ $# > 0 ]]; do
+while [[ $# -gt 0 ]]; do
opt="$(echo "$1" | tr "[:upper:]" "[:lower:]")"
case "$opt" in
--operation)
diff --git a/eng/common/internal/NuGet.config b/eng/common/internal/NuGet.config
index 19d3d311b..f70261ed6 100644
--- a/eng/common/internal/NuGet.config
+++ b/eng/common/internal/NuGet.config
@@ -4,4 +4,7 @@
+
+
+
diff --git a/eng/common/msbuild.ps1 b/eng/common/msbuild.ps1
index f041e5ddd..b6dfb570e 100644
--- a/eng/common/msbuild.ps1
+++ b/eng/common/msbuild.ps1
@@ -3,6 +3,7 @@ Param(
[string] $verbosity = 'minimal',
[bool] $warnAsError = $true,
[bool] $nodeReuse = $true,
+ [bool][Alias('mt')]$msbuildMultiThreaded = $false,
[switch] $ci,
[switch] $prepareMachine,
[switch] $excludePrereleaseVS,
@@ -13,7 +14,8 @@ Param(
. $PSScriptRoot\tools.ps1
try {
- if ($ci) {
+ # Node reuse isn't used on CI unless it was explicitly requested via -nodeReuse.
+ if ($ci -and -not $PSBoundParameters.ContainsKey('nodeReuse')) {
$nodeReuse = $false
}
diff --git a/eng/common/msbuild.sh b/eng/common/msbuild.sh
index 20d3dad54..a40c10123 100644
--- a/eng/common/msbuild.sh
+++ b/eng/common/msbuild.sh
@@ -14,7 +14,9 @@ scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
verbosity='minimal'
warn_as_error=true
-node_reuse=true
+# Empty means "not specified"; tools.sh defaults these to on for local builds and off on CI.
+node_reuse=''
+msbuild_multi_threaded=''
prepare_machine=false
extra_args=''
@@ -33,6 +35,10 @@ while (($# > 0)); do
node_reuse=$2
shift 2
;;
+ --msbuildmultithreaded|--mt)
+ msbuild_multi_threaded=$2
+ shift 2
+ ;;
--ci)
ci=true
shift 1
@@ -50,9 +56,5 @@ done
. "$scriptroot/tools.sh"
-if [[ "$ci" == true ]]; then
- node_reuse=false
-fi
-
MSBuild $extra_args
ExitWithExitCode 0
diff --git a/eng/common/native/LocateNativeCompiler.targets b/eng/common/native/LocateNativeCompiler.targets
new file mode 100644
index 000000000..028b33d94
--- /dev/null
+++ b/eng/common/native/LocateNativeCompiler.targets
@@ -0,0 +1,27 @@
+
+
+
+
+ clang
+ $(ROOTFS_DIR)
+
+
+
+
+
+
+
+ $(_CC_LDFLAGS.SubString(0, $(_CC_LDFLAGS.IndexOf(';'))))
+ <_LDFLAGS>$(_CC_LDFLAGS.SubString($([MSBuild]::Add($(_CC_LDFLAGS.IndexOf(';')), 1))))
+ lld
+
+
+
diff --git a/eng/common/native/NativeAotSupported.props b/eng/common/native/NativeAotSupported.props
new file mode 100644
index 000000000..cdff9ef03
--- /dev/null
+++ b/eng/common/native/NativeAotSupported.props
@@ -0,0 +1,28 @@
+
+
+
+
+ <_NativeAotSupportedOS Condition="
+ '$(TargetOS)' != 'browser' and
+ '$(TargetOS)' != 'haiku' and
+ '$(TargetOS)' != 'illumos' and
+ '$(TargetOS)' != 'netbsd' and
+ '$(TargetOS)' != 'solaris'
+ ">true
+
+
+ <_NativeAotSupportedArch Condition="
+ '$(TargetArchitecture)' != 'wasm' and
+ '$(TargetArchitecture)' != 's390x' and
+ '$(TargetArchitecture)' != 'ppc64le' and
+ ('$(TargetArchitecture)' != 'x86' or '$(TargetOS)' == 'windows')
+ ">true
+
+ true
+
+
+
diff --git a/eng/common/native/init-distro-rid.sh b/eng/common/native/init-distro-rid.sh
index 83ea7aab0..8fc6d2fec 100644
--- a/eng/common/native/init-distro-rid.sh
+++ b/eng/common/native/init-distro-rid.sh
@@ -39,6 +39,8 @@ getNonPortableDistroRid()
# $rootfsDir can be empty. freebsd-version is a shell script and should always work.
__freebsd_major_version=$("$rootfsDir"/bin/freebsd-version | cut -d'.' -f1)
nonPortableRid="freebsd.$__freebsd_major_version-${targetArch}"
+ elif [ "$targetOs" = "openbsd" ]; then
+ nonPortableRid="openbsd.$(uname -r)-${targetArch}"
elif command -v getprop >/dev/null && getprop ro.product.system.model | grep -qi android; then
__android_sdk_version=$(getprop ro.build.version.sdk)
nonPortableRid="android.$__android_sdk_version-${targetArch}"
diff --git a/eng/common/native/init-os-and-arch.sh b/eng/common/native/init-os-and-arch.sh
index 38921d433..62d62fed5 100644
--- a/eng/common/native/init-os-and-arch.sh
+++ b/eng/common/native/init-os-and-arch.sh
@@ -27,6 +27,10 @@ if [ "$os" = "sunos" ]; then
os="solaris"
fi
CPUName=$(isainfo -n)
+elif [ "$os" = "freebsd" ]; then
+ # FreeBSD's `uname -m` is the machine class ("powerpc" for every PowerPC
+ # variant); `uname -p` gives the specific processor (e.g. powerpc64le).
+ CPUName=$(uname -p)
else
# For the rest of the operating systems, use uname(1) to determine what the CPU is.
CPUName=$(uname -m)
@@ -75,7 +79,7 @@ case "$CPUName" in
arch=s390x
;;
- ppc64le)
+ ppc64le|powerpc64le)
arch=ppc64le
;;
*)
diff --git a/eng/common/native/install-dependencies.sh b/eng/common/native/install-dependencies.sh
index ce661e9e5..aff839fa0 100644
--- a/eng/common/native/install-dependencies.sh
+++ b/eng/common/native/install-dependencies.sh
@@ -24,13 +24,16 @@ case "$os" in
apt update
apt install -y build-essential gettext locales cmake llvm clang lld lldb liblldb-dev libunwind8-dev libicu-dev liblttng-ust-dev \
- libssl-dev libkrb5-dev pigz cpio
+ libssl-dev libkrb5-dev pigz cpio ninja-build file
localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8
- elif [ "$ID" = "fedora" ] || [ "$ID" = "rhel" ]; then
- dnf install -y cmake llvm lld lldb clang python curl libicu-devel openssl-devel krb5-devel lttng-ust-devel pigz cpio
+ elif [ "$ID" = "fedora" ] || [ "$ID" = "rhel" ] || [ "$ID" = "azurelinux" ] || [ "$ID" = "centos" ]; then
+ pkg_mgr="$(command -v tdnf 2>/dev/null || command -v dnf)"
+ $pkg_mgr install -y cmake llvm lld lldb clang python curl libicu-devel openssl-devel krb5-devel lttng-ust-devel pigz cpio ninja-build file
+ elif [ "$ID" = "amzn" ]; then
+ dnf install -y cmake llvm lld lldb clang python libicu-devel openssl-devel krb5-devel lttng-ust-devel pigz cpio ninja-build file
elif [ "$ID" = "alpine" ]; then
- apk add build-base cmake bash curl clang llvm-dev lld lldb krb5-dev lttng-ust-dev icu-dev openssl-dev pigz cpio
+ apk add build-base cmake bash curl clang llvm llvm-dev lld lldb-dev krb5-dev lttng-ust-dev icu-dev openssl-dev pigz cpio ninja file
else
echo "Unsupported distro. distro: $ID"
exit 1
@@ -44,13 +47,14 @@ case "$os" in
export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1
# Skip brew update for now, see https://github.com/actions/setup-python/issues/577
# brew update --preinstall
- brew bundle --no-upgrade --no-lock --file=- < Exist flag) used to consult whether a file exist
-# in the repository at a specific commit point. This is populated by inserting
-# all files present in the repo at a specific commit point.
-$global:RepoFiles = @{}
-
-# Maximum number of jobs to run in parallel
-$MaxParallelJobs = 16
-
-$MaxRetries = 5
-$RetryWaitTimeInSeconds = 30
-
-# Wait time between check for system load
-$SecondsBetweenLoadChecks = 10
-
-if (!$InputPath -or !(Test-Path $InputPath)){
- Write-Host "No files to validate."
- ExitWithExitCode 0
-}
-
-$ValidatePackage = {
- param(
- [string] $PackagePath # Full path to a Symbols.NuGet package
- )
-
- . $using:PSScriptRoot\..\tools.ps1
-
- # Ensure input file exist
- if (!(Test-Path $PackagePath)) {
- Write-Host "Input file does not exist: $PackagePath"
- return [pscustomobject]@{
- result = 1
- packagePath = $PackagePath
- }
- }
-
- # Extensions for which we'll look for SourceLink information
- # For now we'll only care about Portable & Embedded PDBs
- $RelevantExtensions = @('.dll', '.exe', '.pdb')
-
- Write-Host -NoNewLine 'Validating ' ([System.IO.Path]::GetFileName($PackagePath)) '...'
-
- $PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath)
- $ExtractPath = Join-Path -Path $using:ExtractPath -ChildPath $PackageId
- $FailedFiles = 0
-
- Add-Type -AssemblyName System.IO.Compression.FileSystem
-
- [System.IO.Directory]::CreateDirectory($ExtractPath) | Out-Null
-
- try {
- $zip = [System.IO.Compression.ZipFile]::OpenRead($PackagePath)
-
- $zip.Entries |
- Where-Object {$RelevantExtensions -contains [System.IO.Path]::GetExtension($_.Name)} |
- ForEach-Object {
- $FileName = $_.FullName
- $Extension = [System.IO.Path]::GetExtension($_.Name)
- $FakeName = -Join((New-Guid), $Extension)
- $TargetFile = Join-Path -Path $ExtractPath -ChildPath $FakeName
-
- # We ignore resource DLLs
- if ($FileName.EndsWith('.resources.dll')) {
- return [pscustomobject]@{
- result = 0
- packagePath = $PackagePath
- }
- }
-
- [System.IO.Compression.ZipFileExtensions]::ExtractToFile($_, $TargetFile, $true)
-
- $ValidateFile = {
- param(
- [string] $FullPath, # Full path to the module that has to be checked
- [string] $RealPath,
- [ref] $FailedFiles
- )
-
- $sourcelinkExe = "$env:USERPROFILE\.dotnet\tools"
- $sourcelinkExe = Resolve-Path "$sourcelinkExe\sourcelink.exe"
- $SourceLinkInfos = & $sourcelinkExe print-urls $FullPath | Out-String
-
- if ($LASTEXITCODE -eq 0 -and -not ([string]::IsNullOrEmpty($SourceLinkInfos))) {
- $NumFailedLinks = 0
-
- # We only care about Http addresses
- $Matches = (Select-String '(http[s]?)(:\/\/)([^\s,]+)' -Input $SourceLinkInfos -AllMatches).Matches
-
- if ($Matches.Count -ne 0) {
- $Matches.Value |
- ForEach-Object {
- $Link = $_
- $CommitUrl = "https://raw.githubusercontent.com/${using:GHRepoName}/${using:GHCommit}/"
-
- $FilePath = $Link.Replace($CommitUrl, "")
- $Status = 200
- $Cache = $using:RepoFiles
-
- $attempts = 0
-
- while ($attempts -lt $using:MaxRetries) {
- if ( !($Cache.ContainsKey($FilePath)) ) {
- try {
- $Uri = $Link -as [System.URI]
-
- if ($Link -match "submodules") {
- # Skip submodule links until sourcelink properly handles submodules
- $Status = 200
- }
- elseif ($Uri.AbsoluteURI -ne $null -and ($Uri.Host -match 'github' -or $Uri.Host -match 'githubusercontent')) {
- # Only GitHub links are valid
- $Status = (Invoke-WebRequest -Uri $Link -UseBasicParsing -Method HEAD -TimeoutSec 5).StatusCode
- }
- else {
- # If it's not a github link, we want to break out of the loop and not retry.
- $Status = 0
- $attempts = $using:MaxRetries
- }
- }
- catch {
- Write-Host $_
- $Status = 0
- }
- }
-
- if ($Status -ne 200) {
- $attempts++
-
- if ($attempts -lt $using:MaxRetries)
- {
- $attemptsLeft = $using:MaxRetries - $attempts
- Write-Warning "Download failed, $attemptsLeft attempts remaining, will retry in $using:RetryWaitTimeInSeconds seconds"
- Start-Sleep -Seconds $using:RetryWaitTimeInSeconds
- }
- else {
- if ($NumFailedLinks -eq 0) {
- if ($FailedFiles.Value -eq 0) {
- Write-Host
- }
-
- Write-Host "`tFile $RealPath has broken links:"
- }
-
- Write-Host "`t`tFailed to retrieve $Link"
-
- $NumFailedLinks++
- }
- }
- else {
- break
- }
- }
- }
- }
-
- if ($NumFailedLinks -ne 0) {
- $FailedFiles.value++
- $global:LASTEXITCODE = 1
- }
- }
- }
-
- &$ValidateFile $TargetFile $FileName ([ref]$FailedFiles)
- }
- }
- catch {
- Write-Host $_
- }
- finally {
- $zip.Dispose()
- }
-
- if ($FailedFiles -eq 0) {
- Write-Host 'Passed.'
- return [pscustomobject]@{
- result = 0
- packagePath = $PackagePath
- }
- }
- else {
- Write-PipelineTelemetryError -Category 'SourceLink' -Message "$PackagePath has broken SourceLink links."
- return [pscustomobject]@{
- result = 1
- packagePath = $PackagePath
- }
- }
-}
-
-function CheckJobResult(
- $result,
- $packagePath,
- [ref]$ValidationFailures,
- [switch]$logErrors) {
- if ($result -ne '0') {
- if ($logErrors) {
- Write-PipelineTelemetryError -Category 'SourceLink' -Message "$packagePath has broken SourceLink links."
- }
- $ValidationFailures.Value++
- }
-}
-
-function ValidateSourceLinkLinks {
- if ($GHRepoName -ne '' -and !($GHRepoName -Match '^[^\s\/]+/[^\s\/]+$')) {
- if (!($GHRepoName -Match '^[^\s-]+-[^\s]+$')) {
- Write-PipelineTelemetryError -Category 'SourceLink' -Message "GHRepoName should be in the format / or -. '$GHRepoName'"
- ExitWithExitCode 1
- }
- else {
- $GHRepoName = $GHRepoName -replace '^([^\s-]+)-([^\s]+)$', '$1/$2';
- }
- }
-
- if ($GHCommit -ne '' -and !($GHCommit -Match '^[0-9a-fA-F]{40}$')) {
- Write-PipelineTelemetryError -Category 'SourceLink' -Message "GHCommit should be a 40 chars hexadecimal string. '$GHCommit'"
- ExitWithExitCode 1
- }
-
- if ($GHRepoName -ne '' -and $GHCommit -ne '') {
- $RepoTreeURL = -Join('http://api.github.com/repos/', $GHRepoName, '/git/trees/', $GHCommit, '?recursive=1')
- $CodeExtensions = @('.cs', '.vb', '.fs', '.fsi', '.fsx', '.fsscript')
-
- try {
- # Retrieve the list of files in the repo at that particular commit point and store them in the RepoFiles hash
- $Data = Invoke-WebRequest $RepoTreeURL -UseBasicParsing | ConvertFrom-Json | Select-Object -ExpandProperty tree
-
- foreach ($file in $Data) {
- $Extension = [System.IO.Path]::GetExtension($file.path)
-
- if ($CodeExtensions.Contains($Extension)) {
- $RepoFiles[$file.path] = 1
- }
- }
- }
- catch {
- Write-Host "Problems downloading the list of files from the repo. Url used: $RepoTreeURL . Execution will proceed without caching."
- }
- }
- elseif ($GHRepoName -ne '' -or $GHCommit -ne '') {
- Write-Host 'For using the http caching mechanism both GHRepoName and GHCommit should be informed.'
- }
-
- if (Test-Path $ExtractPath) {
- Remove-Item $ExtractPath -Force -Recurse -ErrorAction SilentlyContinue
- }
-
- $ValidationFailures = 0
-
- # Process each NuGet package in parallel
- Get-ChildItem "$InputPath\*.symbols.nupkg" |
- ForEach-Object {
- Write-Host "Starting $($_.FullName)"
- Start-Job -ScriptBlock $ValidatePackage -ArgumentList $_.FullName | Out-Null
- $NumJobs = @(Get-Job -State 'Running').Count
-
- while ($NumJobs -ge $MaxParallelJobs) {
- Write-Host "There are $NumJobs validation jobs running right now. Waiting $SecondsBetweenLoadChecks seconds to check again."
- sleep $SecondsBetweenLoadChecks
- $NumJobs = @(Get-Job -State 'Running').Count
- }
-
- foreach ($Job in @(Get-Job -State 'Completed')) {
- $jobResult = Wait-Job -Id $Job.Id | Receive-Job
- CheckJobResult $jobResult.result $jobResult.packagePath ([ref]$ValidationFailures) -LogErrors
- Remove-Job -Id $Job.Id
- }
- }
-
- foreach ($Job in @(Get-Job)) {
- $jobResult = Wait-Job -Id $Job.Id | Receive-Job
- CheckJobResult $jobResult.result $jobResult.packagePath ([ref]$ValidationFailures)
- Remove-Job -Id $Job.Id
- }
- if ($ValidationFailures -gt 0) {
- Write-PipelineTelemetryError -Category 'SourceLink' -Message "$ValidationFailures package(s) failed validation."
- ExitWithExitCode 1
- }
-}
-
-function InstallSourcelinkCli {
- $sourcelinkCliPackageName = 'sourcelink'
-
- $dotnetRoot = InitializeDotNetCli -install:$true
- $dotnet = "$dotnetRoot\dotnet.exe"
- $toolList = & "$dotnet" tool list --global
-
- if (($toolList -like "*$sourcelinkCliPackageName*") -and ($toolList -like "*$sourcelinkCliVersion*")) {
- Write-Host "SourceLink CLI version $sourcelinkCliVersion is already installed."
- }
- else {
- Write-Host "Installing SourceLink CLI version $sourcelinkCliVersion..."
- Write-Host 'You may need to restart your command window if this is the first dotnet tool you have installed.'
- & "$dotnet" tool install $sourcelinkCliPackageName --version $sourcelinkCliVersion --verbosity "minimal" --global
- }
-}
-
-try {
- InstallSourcelinkCli
-
- foreach ($Job in @(Get-Job)) {
- Remove-Job -Id $Job.Id
- }
-
- ValidateSourceLinkLinks
-}
-catch {
- Write-Host $_.Exception
- Write-Host $_.ScriptStackTrace
- Write-PipelineTelemetryError -Category 'SourceLink' -Message $_
- ExitWithExitCode 1
-}
diff --git a/eng/common/renovate.env b/eng/common/renovate.env
new file mode 100644
index 000000000..17ecc05d9
--- /dev/null
+++ b/eng/common/renovate.env
@@ -0,0 +1,42 @@
+# Renovate Global Configuration
+# https://docs.renovatebot.com/self-hosted-configuration/
+#
+# NOTE: This file uses bash/shell format and is sourced via `. renovate.env`.
+# Values containing spaces or special characters must be quoted.
+
+# Author to use for git commits made by Renovate
+# https://docs.renovatebot.com/configuration-options/#gitauthor
+export RENOVATE_GIT_AUTHOR='.NET Renovate '
+
+# Disable rate limiting for PR creation (0 = unlimited)
+# https://docs.renovatebot.com/presets-default/#prhourlylimitnone
+# https://docs.renovatebot.com/presets-default/#prconcurrentlimitnone
+export RENOVATE_PR_HOURLY_LIMIT=0
+export RENOVATE_PR_CONCURRENT_LIMIT=0
+
+# Skip the onboarding PR that Renovate normally creates for new repos
+# https://docs.renovatebot.com/config-overview/#onboarding
+export RENOVATE_ONBOARDING=false
+
+# Any Renovate config file in the cloned repository is ignored. Only
+# the Renovate config file from the repo where the pipeline is running
+# is used (yes, those are the same repo but the sources may be different).
+# https://docs.renovatebot.com/self-hosted-configuration/#requireconfig
+export RENOVATE_REQUIRE_CONFIG=ignored
+
+# Customize the PR body content. This removes some of the default
+# sections that aren't relevant in a self-hosted config.
+# https://docs.renovatebot.com/configuration-options/#prheader
+# https://docs.renovatebot.com/configuration-options/#prbodynotes
+# https://docs.renovatebot.com/configuration-options/#prbodytemplate
+export RENOVATE_PR_HEADER='## Automated Dependency Update'
+export RENOVATE_PR_BODY_NOTES='["This PR has been created automatically by the [.NET Renovate Bot](https://github.com/dotnet/arcade/blob/main/Documentation/Renovate.md) to update one or more dependencies in your repo. Please review the changes and merge the PR if everything looks good."]'
+export RENOVATE_PR_BODY_TEMPLATE='{{{header}}}{{{table}}}{{{warnings}}}{{{notes}}}{{{changelogs}}}'
+
+# Extend the global config with additional presets
+# https://docs.renovatebot.com/self-hosted-configuration/#globalextends
+# Disable the Dependency Dashboard issue that tracks all updates
+export RENOVATE_GLOBAL_EXTENDS='[":disableDependencyDashboard"]'
+
+# Allow all commands for post-upgrade commands.
+export RENOVATE_ALLOWED_COMMANDS='[".*"]'
diff --git a/eng/common/sdk-task.ps1 b/eng/common/sdk-task.ps1
index ab2b13f63..8d72d803d 100644
--- a/eng/common/sdk-task.ps1
+++ b/eng/common/sdk-task.ps1
@@ -4,22 +4,39 @@ Param(
[string] $task,
[string] $verbosity = 'minimal',
[string] $msbuildEngine = $null,
- [switch] $restore,
+ # Restore defaults to on; -restore is retained only so existing consumers that pass it don't break. Use -norestore to opt out.
+ [switch] $restore = $true,
+ [switch] $norestore,
[switch] $prepareMachine,
+ [switch][Alias('nobl')]$excludeCIBinaryLog,
+ [switch]$noWarnAsError,
[switch] $help,
+ [string] $runtimeSourceFeed = '',
+ [string] $runtimeSourceFeedKey = '',
[Parameter(ValueFromRemainingArguments=$true)][String[]]$properties
)
$ci = $true
-$binaryLog = $true
-$warnAsError = $true
+$binaryLog = if ($excludeCIBinaryLog) { $false } else { $true }
+$warnAsError = if ($noWarnAsError) { $false } else { $true }
+
+# Reconcile the restore state before importing tools.ps1: it reads $restore at import time to
+# decide whether toolset/SDK acquisition installs. -norestore must win so that skipping restore
+# also skips toolset initialization, not just the explicit Restore build below.
+if ($norestore) { $restore = $false }
+
+# sdk-task runs a standalone Arcade SDK task and does not need repo-specific toolset setup.
+# Skip importing configure-toolset.ps1 so its side effects (e.g. a repo's configure-toolset.ps1
+# calling exit) don't terminate this script before the task runs.
+$disableConfigureToolsetImport = $true
. $PSScriptRoot\tools.ps1
function Print-Usage() {
Write-Host "Common settings:"
- Write-Host " -task Name of Arcade task (name of a project in SdkTasks directory of the Arcade SDK package)"
- Write-Host " -restore Restore dependencies"
+ Write-Host " -task Name of Arcade task (name of a project in toolset directory of the Arcade SDK package)"
+ Write-Host " -restore (Legacy) Restore runs by default; retained for backward compatibility. Use -norestore to skip"
+ Write-Host " -norestore Skip restoring dependencies"
Write-Host " -verbosity Msbuild verbosity: q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic]"
Write-Host " -help Print help and exit"
Write-Host ""
@@ -27,6 +44,7 @@ function Print-Usage() {
Write-Host "Advanced settings:"
Write-Host " -prepareMachine Prepare machine for CI run"
Write-Host " -msbuildEngine Msbuild engine to use to run build ('dotnet', 'vs', or unspecified)."
+ Write-Host " -excludeCIBinaryLog When running on CI, allow no binary log (short: -nobl)"
Write-Host ""
Write-Host "Command line arguments not listed above are passed thru to msbuild."
}
@@ -34,10 +52,11 @@ function Print-Usage() {
function Build([string]$target) {
$logSuffix = if ($target -eq 'Execute') { '' } else { ".$target" }
$log = Join-Path $LogDir "$task$logSuffix.binlog"
+ $binaryLogArg = if ($binaryLog) { "/bl:$log" } else { "" }
$outputPath = Join-Path $ToolsetDir "$task\"
MSBuild $taskProject `
- /bl:$log `
+ $binaryLogArg `
/t:$target `
/p:Configuration=$configuration `
/p:RepoRoot=$RepoRoot `
@@ -60,20 +79,7 @@ try {
if( $msbuildEngine -eq "vs") {
# Ensure desktop MSBuild is available for sdk tasks.
- if( -not ($GlobalJson.tools.PSObject.Properties.Name -contains "vs" )) {
- $GlobalJson.tools | Add-Member -Name "vs" -Value (ConvertFrom-Json "{ `"version`": `"16.5`" }") -MemberType NoteProperty
- }
- if( -not ($GlobalJson.tools.PSObject.Properties.Name -match "xcopy-msbuild" )) {
- $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "17.13.0" -MemberType NoteProperty
- }
- if ($GlobalJson.tools."xcopy-msbuild".Trim() -ine "none") {
- $xcopyMSBuildToolsFolder = InitializeXCopyMSBuild $GlobalJson.tools."xcopy-msbuild" -install $true
- }
- if ($xcopyMSBuildToolsFolder -eq $null) {
- throw 'Unable to get xcopy downloadable version of msbuild'
- }
-
- $global:_MSBuildExe = "$($xcopyMSBuildToolsFolder)\MSBuild\Current\Bin\MSBuild.exe"
+ $global:_MSBuildExe = InitializeVisualStudioMSBuild
}
$taskProject = GetSdkTaskProject $task
diff --git a/eng/common/sdk-task.sh b/eng/common/sdk-task.sh
index b9b9e58db..a7f1ba060 100644
--- a/eng/common/sdk-task.sh
+++ b/eng/common/sdk-task.sh
@@ -2,11 +2,17 @@
show_usage() {
echo "Common settings:"
- echo " --task Name of Arcade task (name of a project in SdkTasks directory of the Arcade SDK package)"
- echo " --restore Restore dependencies"
+ echo " --task Name of Arcade task (name of a project in toolset directory of the Arcade SDK package)"
+ echo " --restore (Legacy) Restore runs by default; retained for backward compatibility. Use --norestore to skip"
+ echo " --norestore Skip restoring dependencies"
echo " --verbosity Msbuild verbosity: q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic]"
echo " --help Print help and exit"
echo ""
+
+ echo "Advanced settings:"
+ echo " --excludeCIBinarylog Don't output binary log (short: -nobl)"
+ echo " --noWarnAsError Do not warn as error"
+ echo ""
echo "Command line arguments not listed above are passed thru to msbuild."
}
@@ -27,10 +33,12 @@ Build() {
local log_suffix=""
[[ "$target" != "Execute" ]] && log_suffix=".$target"
local log="$log_dir/$task$log_suffix.binlog"
+ local binaryLogArg=""
+ [[ $binary_log == true ]] && binaryLogArg="/bl:$log"
local output_path="$toolset_dir/$task/"
MSBuild "$taskProject" \
- /bl:"$log" \
+ $binaryLogArg \
/t:"$target" \
/p:Configuration="$configuration" \
/p:RepoRoot="$repo_root" \
@@ -39,11 +47,15 @@ Build() {
$properties
}
+binary_log=true
configuration="Debug"
verbosity="minimal"
-restore=false
+exclude_ci_binary_log=false
+# restore defaults to on; --restore is retained only so existing consumers that pass it don't break. Use --norestore to opt out.
+restore=true
help=false
properties=''
+warn_as_error=true
while (($# > 0)); do
lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")"
@@ -53,13 +65,25 @@ while (($# > 0)); do
shift 2
;;
--restore)
- restore=true
+ shift 1
+ ;;
+ --norestore)
+ restore=false
shift 1
;;
--verbosity)
verbosity=$2
shift 2
;;
+ --excludecibinarylog|--nobl)
+ binary_log=false
+ exclude_ci_binary_log=true
+ shift 1
+ ;;
+ --nowarnaserror)
+ warn_as_error=false
+ shift 1
+ ;;
--help)
help=true
shift 1
@@ -72,14 +96,17 @@ while (($# > 0)); do
done
ci=true
-binaryLog=true
-warnAsError=true
if $help; then
show_usage
exit 0
fi
+# sdk-task runs a standalone Arcade SDK task and does not need repo-specific toolset setup.
+# Skip importing configure-toolset.sh so its side effects (e.g. a repo's configure-toolset.sh
+# calling exit) don't terminate this script before the task runs.
+disable_configure_toolset_import=1
+
. "$scriptroot/tools.sh"
InitializeToolset
diff --git a/eng/common/sdl/NuGet.config b/eng/common/sdl/NuGet.config
deleted file mode 100644
index 3849bdb3c..000000000
--- a/eng/common/sdl/NuGet.config
+++ /dev/null
@@ -1,18 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
diff --git a/eng/common/sdl/configure-sdl-tool.ps1 b/eng/common/sdl/configure-sdl-tool.ps1
deleted file mode 100644
index 27f5a4115..000000000
--- a/eng/common/sdl/configure-sdl-tool.ps1
+++ /dev/null
@@ -1,130 +0,0 @@
-Param(
- [string] $GuardianCliLocation,
- [string] $WorkingDirectory,
- [string] $TargetDirectory,
- [string] $GdnFolder,
- # The list of Guardian tools to configure. For each object in the array:
- # - If the item is a [hashtable], it must contain these entries:
- # - Name = The tool name as Guardian knows it.
- # - Scenario = (Optional) Scenario-specific name for this configuration entry. It must be unique
- # among all tool entries with the same Name.
- # - Args = (Optional) Array of Guardian tool configuration args, like '@("Target > C:\temp")'
- # - If the item is a [string] $v, it is treated as '@{ Name="$v" }'
- [object[]] $ToolsList,
- [string] $GuardianLoggerLevel='Standard',
- # Optional: Additional params to add to any tool using CredScan.
- [string[]] $CrScanAdditionalRunConfigParams,
- # Optional: Additional params to add to any tool using PoliCheck.
- [string[]] $PoliCheckAdditionalRunConfigParams,
- # Optional: Additional params to add to any tool using CodeQL/Semmle.
- [string[]] $CodeQLAdditionalRunConfigParams,
- # Optional: Additional params to add to any tool using Binskim.
- [string[]] $BinskimAdditionalRunConfigParams
-)
-
-$ErrorActionPreference = 'Stop'
-Set-StrictMode -Version 2.0
-$disableConfigureToolsetImport = $true
-$global:LASTEXITCODE = 0
-
-try {
- # `tools.ps1` checks $ci to perform some actions. Since the SDL
- # scripts don't necessarily execute in the same agent that run the
- # build.ps1/sh script this variable isn't automatically set.
- $ci = $true
- . $PSScriptRoot\..\tools.ps1
-
- # Normalize tools list: all in [hashtable] form with defined values for each key.
- $ToolsList = $ToolsList |
- ForEach-Object {
- if ($_ -is [string]) {
- $_ = @{ Name = $_ }
- }
-
- if (-not ($_['Scenario'])) { $_.Scenario = "" }
- if (-not ($_['Args'])) { $_.Args = @() }
- $_
- }
-
- Write-Host "List of tools to configure:"
- $ToolsList | ForEach-Object { $_ | Out-String | Write-Host }
-
- # We store config files in the r directory of .gdn
- $gdnConfigPath = Join-Path $GdnFolder 'r'
- $ValidPath = Test-Path $GuardianCliLocation
-
- if ($ValidPath -eq $False)
- {
- Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Invalid Guardian CLI Location."
- ExitWithExitCode 1
- }
-
- foreach ($tool in $ToolsList) {
- # Put together the name and scenario to make a unique key.
- $toolConfigName = $tool.Name
- if ($tool.Scenario) {
- $toolConfigName += "_" + $tool.Scenario
- }
-
- Write-Host "=== Configuring $toolConfigName..."
-
- $gdnConfigFile = Join-Path $gdnConfigPath "$toolConfigName-configure.gdnconfig"
-
- # For some tools, add default and automatic args.
- switch -Exact ($tool.Name) {
- 'credscan' {
- if ($targetDirectory) {
- $tool.Args += "`"TargetDirectory < $TargetDirectory`""
- }
- $tool.Args += "`"OutputType < pre`""
- $tool.Args += $CrScanAdditionalRunConfigParams
- }
- 'policheck' {
- if ($targetDirectory) {
- $tool.Args += "`"Target < $TargetDirectory`""
- }
- $tool.Args += $PoliCheckAdditionalRunConfigParams
- }
- {$_ -in 'semmle', 'codeql'} {
- if ($targetDirectory) {
- $tool.Args += "`"SourceCodeDirectory < $TargetDirectory`""
- }
- $tool.Args += $CodeQLAdditionalRunConfigParams
- }
- 'binskim' {
- if ($targetDirectory) {
- # Binskim crashes due to specific PDBs. GitHub issue: https://github.com/microsoft/binskim/issues/924.
- # We are excluding all `_.pdb` files from the scan.
- $tool.Args += "`"Target < $TargetDirectory\**;-:file|$TargetDirectory\**\_.pdb`""
- }
- $tool.Args += $BinskimAdditionalRunConfigParams
- }
- }
-
- # Create variable pointing to the args array directly so we can use splat syntax later.
- $toolArgs = $tool.Args
-
- # Configure the tool. If args array is provided or the current tool has some default arguments
- # defined, add "--args" and splat each element on the end. Arg format is "{Arg id} < {Value}",
- # one per parameter. Doc page for "guardian configure":
- # https://dev.azure.com/securitytools/SecurityIntegration/_wiki/wikis/Guardian/1395/configure
- Exec-BlockVerbosely {
- & $GuardianCliLocation configure `
- --working-directory $WorkingDirectory `
- --tool $tool.Name `
- --output-path $gdnConfigFile `
- --logger-level $GuardianLoggerLevel `
- --noninteractive `
- --force `
- $(if ($toolArgs) { "--args" }) @toolArgs
- Exit-IfNZEC "Sdl"
- }
-
- Write-Host "Created '$toolConfigName' configuration file: $gdnConfigFile"
- }
-}
-catch {
- Write-Host $_.ScriptStackTrace
- Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_
- ExitWithExitCode 1
-}
diff --git a/eng/common/sdl/execute-all-sdl-tools.ps1 b/eng/common/sdl/execute-all-sdl-tools.ps1
deleted file mode 100644
index 4715d75e9..000000000
--- a/eng/common/sdl/execute-all-sdl-tools.ps1
+++ /dev/null
@@ -1,167 +0,0 @@
-Param(
- [string] $GuardianPackageName, # Required: the name of guardian CLI package (not needed if GuardianCliLocation is specified)
- [string] $NugetPackageDirectory, # Required: directory where NuGet packages are installed (not needed if GuardianCliLocation is specified)
- [string] $GuardianCliLocation, # Optional: Direct location of Guardian CLI executable if GuardianPackageName & NugetPackageDirectory are not specified
- [string] $Repository=$env:BUILD_REPOSITORY_NAME, # Required: the name of the repository (e.g. dotnet/arcade)
- [string] $BranchName=$env:BUILD_SOURCEBRANCH, # Optional: name of branch or version of gdn settings; defaults to master
- [string] $SourceDirectory=$env:BUILD_SOURCESDIRECTORY, # Required: the directory where source files are located
- [string] $ArtifactsDirectory = (Join-Path $env:BUILD_ARTIFACTSTAGINGDIRECTORY ('artifacts')), # Required: the directory where build artifacts are located
- [string] $AzureDevOpsAccessToken, # Required: access token for dnceng; should be provided via KeyVault
-
- # Optional: list of SDL tools to run on source code. See 'configure-sdl-tool.ps1' for tools list
- # format.
- [object[]] $SourceToolsList,
- # Optional: list of SDL tools to run on built artifacts. See 'configure-sdl-tool.ps1' for tools
- # list format.
- [object[]] $ArtifactToolsList,
- # Optional: list of SDL tools to run without automatically specifying a target directory. See
- # 'configure-sdl-tool.ps1' for tools list format.
- [object[]] $CustomToolsList,
-
- [bool] $TsaPublish=$False, # Optional: true will publish results to TSA; only set to true after onboarding to TSA; TSA is the automated framework used to upload test results as bugs.
- [string] $TsaBranchName=$env:BUILD_SOURCEBRANCH, # Optional: required for TSA publish; defaults to $(Build.SourceBranchName); TSA is the automated framework used to upload test results as bugs.
- [string] $TsaRepositoryName=$env:BUILD_REPOSITORY_NAME, # Optional: TSA repository name; will be generated automatically if not submitted; TSA is the automated framework used to upload test results as bugs.
- [string] $BuildNumber=$env:BUILD_BUILDNUMBER, # Optional: required for TSA publish; defaults to $(Build.BuildNumber)
- [bool] $UpdateBaseline=$False, # Optional: if true, will update the baseline in the repository; should only be run after fixing any issues which need to be fixed
- [bool] $TsaOnboard=$False, # Optional: if true, will onboard the repository to TSA; should only be run once; TSA is the automated framework used to upload test results as bugs.
- [string] $TsaInstanceUrl, # Optional: only needed if TsaOnboard or TsaPublish is true; the instance-url registered with TSA; TSA is the automated framework used to upload test results as bugs.
- [string] $TsaCodebaseName, # Optional: only needed if TsaOnboard or TsaPublish is true; the name of the codebase registered with TSA; TSA is the automated framework used to upload test results as bugs.
- [string] $TsaProjectName, # Optional: only needed if TsaOnboard or TsaPublish is true; the name of the project registered with TSA; TSA is the automated framework used to upload test results as bugs.
- [string] $TsaNotificationEmail, # Optional: only needed if TsaOnboard is true; the email(s) which will receive notifications of TSA bug filings (e.g. alias@microsoft.com); TSA is the automated framework used to upload test results as bugs.
- [string] $TsaCodebaseAdmin, # Optional: only needed if TsaOnboard is true; the aliases which are admins of the TSA codebase (e.g. DOMAIN\alias); TSA is the automated framework used to upload test results as bugs.
- [string] $TsaBugAreaPath, # Optional: only needed if TsaOnboard is true; the area path where TSA will file bugs in AzDO; TSA is the automated framework used to upload test results as bugs.
- [string] $TsaIterationPath, # Optional: only needed if TsaOnboard is true; the iteration path where TSA will file bugs in AzDO; TSA is the automated framework used to upload test results as bugs.
- [string] $GuardianLoggerLevel='Standard', # Optional: the logger level for the Guardian CLI; options are Trace, Verbose, Standard, Warning, and Error
- [string[]] $CrScanAdditionalRunConfigParams, # Optional: Additional Params to custom build a CredScan run config in the format @("xyz:abc","sdf:1")
- [string[]] $PoliCheckAdditionalRunConfigParams, # Optional: Additional Params to custom build a Policheck run config in the format @("xyz:abc","sdf:1")
- [string[]] $CodeQLAdditionalRunConfigParams, # Optional: Additional Params to custom build a Semmle/CodeQL run config in the format @("xyz < abc","sdf < 1")
- [string[]] $BinskimAdditionalRunConfigParams, # Optional: Additional Params to custom build a Binskim run config in the format @("xyz < abc","sdf < 1")
- [bool] $BreakOnFailure=$False # Optional: Fail the build if there were errors during the run
-)
-
-try {
- $ErrorActionPreference = 'Stop'
- Set-StrictMode -Version 2.0
- $disableConfigureToolsetImport = $true
- $global:LASTEXITCODE = 0
-
- # `tools.ps1` checks $ci to perform some actions. Since the SDL
- # scripts don't necessarily execute in the same agent that run the
- # build.ps1/sh script this variable isn't automatically set.
- $ci = $true
- . $PSScriptRoot\..\tools.ps1
-
- #Replace repo names to the format of org/repo
- if (!($Repository.contains('/'))) {
- $RepoName = $Repository -replace '(.*?)-(.*)', '$1/$2';
- }
- else{
- $RepoName = $Repository;
- }
-
- if ($GuardianPackageName) {
- $guardianCliLocation = Join-Path $NugetPackageDirectory (Join-Path $GuardianPackageName (Join-Path 'tools' 'guardian.cmd'))
- } else {
- $guardianCliLocation = $GuardianCliLocation
- }
-
- $workingDirectory = (Split-Path $SourceDirectory -Parent)
- $ValidPath = Test-Path $guardianCliLocation
-
- if ($ValidPath -eq $False)
- {
- Write-PipelineTelemetryError -Force -Category 'Sdl' -Message 'Invalid Guardian CLI Location.'
- ExitWithExitCode 1
- }
-
- Exec-BlockVerbosely {
- & $(Join-Path $PSScriptRoot 'init-sdl.ps1') -GuardianCliLocation $guardianCliLocation -Repository $RepoName -BranchName $BranchName -WorkingDirectory $workingDirectory -AzureDevOpsAccessToken $AzureDevOpsAccessToken -GuardianLoggerLevel $GuardianLoggerLevel
- }
- $gdnFolder = Join-Path $workingDirectory '.gdn'
-
- if ($TsaOnboard) {
- if ($TsaCodebaseName -and $TsaNotificationEmail -and $TsaCodebaseAdmin -and $TsaBugAreaPath) {
- Exec-BlockVerbosely {
- & $guardianCliLocation tsa-onboard --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel
- }
- if ($LASTEXITCODE -ne 0) {
- Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Guardian tsa-onboard failed with exit code $LASTEXITCODE."
- ExitWithExitCode $LASTEXITCODE
- }
- } else {
- Write-PipelineTelemetryError -Force -Category 'Sdl' -Message 'Could not onboard to TSA -- not all required values ($TsaCodebaseName, $TsaNotificationEmail, $TsaCodebaseAdmin, $TsaBugAreaPath) were specified.'
- ExitWithExitCode 1
- }
- }
-
- # Configure a list of tools with a default target directory. Populates the ".gdn/r" directory.
- function Configure-ToolsList([object[]] $tools, [string] $targetDirectory) {
- if ($tools -and $tools.Count -gt 0) {
- Exec-BlockVerbosely {
- & $(Join-Path $PSScriptRoot 'configure-sdl-tool.ps1') `
- -GuardianCliLocation $guardianCliLocation `
- -WorkingDirectory $workingDirectory `
- -TargetDirectory $targetDirectory `
- -GdnFolder $gdnFolder `
- -ToolsList $tools `
- -AzureDevOpsAccessToken $AzureDevOpsAccessToken `
- -GuardianLoggerLevel $GuardianLoggerLevel `
- -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams `
- -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams `
- -CodeQLAdditionalRunConfigParams $CodeQLAdditionalRunConfigParams `
- -BinskimAdditionalRunConfigParams $BinskimAdditionalRunConfigParams
- if ($BreakOnFailure) {
- Exit-IfNZEC "Sdl"
- }
- }
- }
- }
-
- # Configure Artifact and Source tools with default Target directories.
- Configure-ToolsList $ArtifactToolsList $ArtifactsDirectory
- Configure-ToolsList $SourceToolsList $SourceDirectory
- # Configure custom tools with no default Target directory.
- Configure-ToolsList $CustomToolsList $null
-
- # At this point, all tools are configured in the ".gdn" directory. Run them all in a single call.
- # (If we used "run" multiple times, each run would overwrite data from earlier runs.)
- Exec-BlockVerbosely {
- & $(Join-Path $PSScriptRoot 'run-sdl.ps1') `
- -GuardianCliLocation $guardianCliLocation `
- -WorkingDirectory $SourceDirectory `
- -UpdateBaseline $UpdateBaseline `
- -GdnFolder $gdnFolder
- }
-
- if ($TsaPublish) {
- if ($TsaBranchName -and $BuildNumber) {
- if (-not $TsaRepositoryName) {
- $TsaRepositoryName = "$($Repository)-$($BranchName)"
- }
- Exec-BlockVerbosely {
- & $guardianCliLocation tsa-publish --all-tools --repository-name "$TsaRepositoryName" --branch-name "$TsaBranchName" --build-number "$BuildNumber" --onboard $True --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel
- }
- if ($LASTEXITCODE -ne 0) {
- Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Guardian tsa-publish failed with exit code $LASTEXITCODE."
- ExitWithExitCode $LASTEXITCODE
- }
- } else {
- Write-PipelineTelemetryError -Force -Category 'Sdl' -Message 'Could not publish to TSA -- not all required values ($TsaBranchName, $BuildNumber) were specified.'
- ExitWithExitCode 1
- }
- }
-
- if ($BreakOnFailure) {
- Write-Host "Failing the build in case of breaking results..."
- Exec-BlockVerbosely {
- & $guardianCliLocation break --working-directory $workingDirectory --logger-level $GuardianLoggerLevel
- }
- } else {
- Write-Host "Letting the build pass even if there were breaking results..."
- }
-}
-catch {
- Write-Host $_.ScriptStackTrace
- Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_
- exit 1
-}
diff --git a/eng/common/sdl/extract-artifact-archives.ps1 b/eng/common/sdl/extract-artifact-archives.ps1
deleted file mode 100644
index 68da4fbf2..000000000
--- a/eng/common/sdl/extract-artifact-archives.ps1
+++ /dev/null
@@ -1,63 +0,0 @@
-# This script looks for each archive file in a directory and extracts it into the target directory.
-# For example, the file "$InputPath/bin.tar.gz" extracts to "$ExtractPath/bin.tar.gz.extracted/**".
-# Uses the "tar" utility added to Windows 10 / Windows 2019 that supports tar.gz and zip.
-param(
- # Full path to directory where archives are stored.
- [Parameter(Mandatory=$true)][string] $InputPath,
- # Full path to directory to extract archives into. May be the same as $InputPath.
- [Parameter(Mandatory=$true)][string] $ExtractPath
-)
-
-$ErrorActionPreference = 'Stop'
-Set-StrictMode -Version 2.0
-
-$disableConfigureToolsetImport = $true
-
-try {
- # `tools.ps1` checks $ci to perform some actions. Since the SDL
- # scripts don't necessarily execute in the same agent that run the
- # build.ps1/sh script this variable isn't automatically set.
- $ci = $true
- . $PSScriptRoot\..\tools.ps1
-
- Measure-Command {
- $jobs = @()
-
- # Find archive files for non-Windows and Windows builds.
- $archiveFiles = @(
- Get-ChildItem (Join-Path $InputPath "*.tar.gz")
- Get-ChildItem (Join-Path $InputPath "*.zip")
- )
-
- foreach ($targzFile in $archiveFiles) {
- $jobs += Start-Job -ScriptBlock {
- $file = $using:targzFile
- $fileName = [System.IO.Path]::GetFileName($file)
- $extractDir = Join-Path $using:ExtractPath "$fileName.extracted"
-
- New-Item $extractDir -ItemType Directory -Force | Out-Null
-
- Write-Host "Extracting '$file' to '$extractDir'..."
-
- # Pipe errors to stdout to prevent PowerShell detecting them and quitting the job early.
- # This type of quit skips the catch, so we wouldn't be able to tell which file triggered the
- # error. Save output so it can be stored in the exception string along with context.
- $output = tar -xf $file -C $extractDir 2>&1
- # Handle NZEC manually rather than using Exit-IfNZEC: we are in a background job, so we
- # don't have access to the outer scope.
- if ($LASTEXITCODE -ne 0) {
- throw "Error extracting '$file': non-zero exit code ($LASTEXITCODE). Output: '$output'"
- }
-
- Write-Host "Extracted to $extractDir"
- }
- }
-
- Receive-Job $jobs -Wait
- }
-}
-catch {
- Write-Host $_
- Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_
- ExitWithExitCode 1
-}
diff --git a/eng/common/sdl/extract-artifact-packages.ps1 b/eng/common/sdl/extract-artifact-packages.ps1
deleted file mode 100644
index f031ed5b2..000000000
--- a/eng/common/sdl/extract-artifact-packages.ps1
+++ /dev/null
@@ -1,82 +0,0 @@
-param(
- [Parameter(Mandatory=$true)][string] $InputPath, # Full path to directory where artifact packages are stored
- [Parameter(Mandatory=$true)][string] $ExtractPath # Full path to directory where the packages will be extracted
-)
-
-$ErrorActionPreference = 'Stop'
-Set-StrictMode -Version 2.0
-
-$disableConfigureToolsetImport = $true
-
-function ExtractArtifacts {
- if (!(Test-Path $InputPath)) {
- Write-Host "Input Path does not exist: $InputPath"
- ExitWithExitCode 0
- }
- $Jobs = @()
- Get-ChildItem "$InputPath\*.nupkg" |
- ForEach-Object {
- $Jobs += Start-Job -ScriptBlock $ExtractPackage -ArgumentList $_.FullName
- }
-
- foreach ($Job in $Jobs) {
- Wait-Job -Id $Job.Id | Receive-Job
- }
-}
-
-try {
- # `tools.ps1` checks $ci to perform some actions. Since the SDL
- # scripts don't necessarily execute in the same agent that run the
- # build.ps1/sh script this variable isn't automatically set.
- $ci = $true
- . $PSScriptRoot\..\tools.ps1
-
- $ExtractPackage = {
- param(
- [string] $PackagePath # Full path to a NuGet package
- )
-
- if (!(Test-Path $PackagePath)) {
- Write-PipelineTelemetryError -Category 'Build' -Message "Input file does not exist: $PackagePath"
- ExitWithExitCode 1
- }
-
- $RelevantExtensions = @('.dll', '.exe', '.pdb')
- Write-Host -NoNewLine 'Extracting ' ([System.IO.Path]::GetFileName($PackagePath)) '...'
-
- $PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath)
- $ExtractPath = Join-Path -Path $using:ExtractPath -ChildPath $PackageId
-
- Add-Type -AssemblyName System.IO.Compression.FileSystem
-
- [System.IO.Directory]::CreateDirectory($ExtractPath);
-
- try {
- $zip = [System.IO.Compression.ZipFile]::OpenRead($PackagePath)
-
- $zip.Entries |
- Where-Object {$RelevantExtensions -contains [System.IO.Path]::GetExtension($_.Name)} |
- ForEach-Object {
- $TargetPath = Join-Path -Path $ExtractPath -ChildPath (Split-Path -Path $_.FullName)
- [System.IO.Directory]::CreateDirectory($TargetPath);
-
- $TargetFile = Join-Path -Path $ExtractPath -ChildPath $_.FullName
- [System.IO.Compression.ZipFileExtensions]::ExtractToFile($_, $TargetFile)
- }
- }
- catch {
- Write-Host $_
- Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_
- ExitWithExitCode 1
- }
- finally {
- $zip.Dispose()
- }
- }
- Measure-Command { ExtractArtifacts }
-}
-catch {
- Write-Host $_
- Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_
- ExitWithExitCode 1
-}
diff --git a/eng/common/sdl/init-sdl.ps1 b/eng/common/sdl/init-sdl.ps1
deleted file mode 100644
index 3ac1d92b3..000000000
--- a/eng/common/sdl/init-sdl.ps1
+++ /dev/null
@@ -1,55 +0,0 @@
-Param(
- [string] $GuardianCliLocation,
- [string] $Repository,
- [string] $BranchName='master',
- [string] $WorkingDirectory,
- [string] $AzureDevOpsAccessToken,
- [string] $GuardianLoggerLevel='Standard'
-)
-
-$ErrorActionPreference = 'Stop'
-Set-StrictMode -Version 2.0
-$disableConfigureToolsetImport = $true
-$global:LASTEXITCODE = 0
-
-# `tools.ps1` checks $ci to perform some actions. Since the SDL
-# scripts don't necessarily execute in the same agent that run the
-# build.ps1/sh script this variable isn't automatically set.
-$ci = $true
-. $PSScriptRoot\..\tools.ps1
-
-# Don't display the console progress UI - it's a huge perf hit
-$ProgressPreference = 'SilentlyContinue'
-
-# Construct basic auth from AzDO access token; construct URI to the repository's gdn folder stored in that repository; construct location of zip file
-$encodedPat = [Convert]::ToBase64String([System.Text.Encoding]::ASCII.GetBytes(":$AzureDevOpsAccessToken"))
-$escapedRepository = [Uri]::EscapeDataString("/$Repository/$BranchName/.gdn")
-$uri = "https://dev.azure.com/dnceng/internal/_apis/git/repositories/sdl-tool-cfg/Items?path=$escapedRepository&versionDescriptor[versionOptions]=0&`$format=zip&api-version=5.0"
-$zipFile = "$WorkingDirectory/gdn.zip"
-
-Add-Type -AssemblyName System.IO.Compression.FileSystem
-$gdnFolder = (Join-Path $WorkingDirectory '.gdn')
-
-try {
- # if the folder does not exist, we'll do a guardian init and push it to the remote repository
- Write-Host 'Initializing Guardian...'
- Write-Host "$GuardianCliLocation init --working-directory $WorkingDirectory --logger-level $GuardianLoggerLevel"
- & $GuardianCliLocation init --working-directory $WorkingDirectory --logger-level $GuardianLoggerLevel
- if ($LASTEXITCODE -ne 0) {
- Write-PipelineTelemetryError -Force -Category 'Build' -Message "Guardian init failed with exit code $LASTEXITCODE."
- ExitWithExitCode $LASTEXITCODE
- }
- # We create the mainbaseline so it can be edited later
- Write-Host "$GuardianCliLocation baseline --working-directory $WorkingDirectory --name mainbaseline"
- & $GuardianCliLocation baseline --working-directory $WorkingDirectory --name mainbaseline
- if ($LASTEXITCODE -ne 0) {
- Write-PipelineTelemetryError -Force -Category 'Build' -Message "Guardian baseline failed with exit code $LASTEXITCODE."
- ExitWithExitCode $LASTEXITCODE
- }
- ExitWithExitCode 0
-}
-catch {
- Write-Host $_.ScriptStackTrace
- Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_
- ExitWithExitCode 1
-}
diff --git a/eng/common/sdl/packages.config b/eng/common/sdl/packages.config
deleted file mode 100644
index 4585cfd6b..000000000
--- a/eng/common/sdl/packages.config
+++ /dev/null
@@ -1,4 +0,0 @@
-
-
-
-
diff --git a/eng/common/sdl/run-sdl.ps1 b/eng/common/sdl/run-sdl.ps1
deleted file mode 100644
index 2eac8c78f..000000000
--- a/eng/common/sdl/run-sdl.ps1
+++ /dev/null
@@ -1,49 +0,0 @@
-Param(
- [string] $GuardianCliLocation,
- [string] $WorkingDirectory,
- [string] $GdnFolder,
- [string] $UpdateBaseline,
- [string] $GuardianLoggerLevel='Standard'
-)
-
-$ErrorActionPreference = 'Stop'
-Set-StrictMode -Version 2.0
-$disableConfigureToolsetImport = $true
-$global:LASTEXITCODE = 0
-
-try {
- # `tools.ps1` checks $ci to perform some actions. Since the SDL
- # scripts don't necessarily execute in the same agent that run the
- # build.ps1/sh script this variable isn't automatically set.
- $ci = $true
- . $PSScriptRoot\..\tools.ps1
-
- # We store config files in the r directory of .gdn
- $gdnConfigPath = Join-Path $GdnFolder 'r'
- $ValidPath = Test-Path $GuardianCliLocation
-
- if ($ValidPath -eq $False)
- {
- Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Invalid Guardian CLI Location."
- ExitWithExitCode 1
- }
-
- $gdnConfigFiles = Get-ChildItem $gdnConfigPath -Recurse -Include '*.gdnconfig'
- Write-Host "Discovered Guardian config files:"
- $gdnConfigFiles | Out-String | Write-Host
-
- Exec-BlockVerbosely {
- & $GuardianCliLocation run `
- --working-directory $WorkingDirectory `
- --baseline mainbaseline `
- --update-baseline $UpdateBaseline `
- --logger-level $GuardianLoggerLevel `
- --config @gdnConfigFiles
- Exit-IfNZEC "Sdl"
- }
-}
-catch {
- Write-Host $_.ScriptStackTrace
- Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_
- ExitWithExitCode 1
-}
diff --git a/eng/common/sdl/sdl.ps1 b/eng/common/sdl/sdl.ps1
deleted file mode 100644
index 648c5068d..000000000
--- a/eng/common/sdl/sdl.ps1
+++ /dev/null
@@ -1,38 +0,0 @@
-
-function Install-Gdn {
- param(
- [Parameter(Mandatory=$true)]
- [string]$Path,
-
- # If omitted, install the latest version of Guardian, otherwise install that specific version.
- [string]$Version
- )
-
- $ErrorActionPreference = 'Stop'
- Set-StrictMode -Version 2.0
- $disableConfigureToolsetImport = $true
- $global:LASTEXITCODE = 0
-
- # `tools.ps1` checks $ci to perform some actions. Since the SDL
- # scripts don't necessarily execute in the same agent that run the
- # build.ps1/sh script this variable isn't automatically set.
- $ci = $true
- . $PSScriptRoot\..\tools.ps1
-
- $argumentList = @("install", "Microsoft.Guardian.Cli", "-Source https://securitytools.pkgs.visualstudio.com/_packaging/Guardian/nuget/v3/index.json", "-OutputDirectory $Path", "-NonInteractive", "-NoCache")
-
- if ($Version) {
- $argumentList += "-Version $Version"
- }
-
- Start-Process nuget -Verbose -ArgumentList $argumentList -NoNewWindow -Wait
-
- $gdnCliPath = Get-ChildItem -Filter guardian.cmd -Recurse -Path $Path
-
- if (!$gdnCliPath)
- {
- Write-PipelineTelemetryError -Category 'Sdl' -Message 'Failure installing Guardian'
- }
-
- return $gdnCliPath.FullName
-}
\ No newline at end of file
diff --git a/eng/common/sdl/trim-assets-version.ps1 b/eng/common/sdl/trim-assets-version.ps1
deleted file mode 100644
index 0daa2a9e9..000000000
--- a/eng/common/sdl/trim-assets-version.ps1
+++ /dev/null
@@ -1,75 +0,0 @@
-<#
-.SYNOPSIS
-Install and run the 'Microsoft.DotNet.VersionTools.Cli' tool with the 'trim-artifacts-version' command to trim the version from the NuGet assets file name.
-
-.PARAMETER InputPath
-Full path to directory where artifact packages are stored
-
-.PARAMETER Recursive
-Search for NuGet packages recursively
-
-#>
-
-Param(
- [string] $InputPath,
- [bool] $Recursive = $true
-)
-
-$CliToolName = "Microsoft.DotNet.VersionTools.Cli"
-
-function Install-VersionTools-Cli {
- param(
- [Parameter(Mandatory=$true)][string]$Version
- )
-
- Write-Host "Installing the package '$CliToolName' with a version of '$version' ..."
- $feed = "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json"
-
- $argumentList = @("tool", "install", "--local", "$CliToolName", "--add-source $feed", "--no-cache", "--version $Version", "--create-manifest-if-needed")
- Start-Process "$dotnet" -Verbose -ArgumentList $argumentList -NoNewWindow -Wait
-}
-
-# -------------------------------------------------------------------
-
-if (!(Test-Path $InputPath)) {
- Write-Host "Input Path '$InputPath' does not exist"
- ExitWithExitCode 1
-}
-
-$ErrorActionPreference = 'Stop'
-Set-StrictMode -Version 2.0
-
-$disableConfigureToolsetImport = $true
-$global:LASTEXITCODE = 0
-
-# `tools.ps1` checks $ci to perform some actions. Since the SDL
-# scripts don't necessarily execute in the same agent that run the
-# build.ps1/sh script this variable isn't automatically set.
-$ci = $true
-. $PSScriptRoot\..\tools.ps1
-
-try {
- $dotnetRoot = InitializeDotNetCli -install:$true
- $dotnet = "$dotnetRoot\dotnet.exe"
-
- $toolsetVersion = Read-ArcadeSdkVersion
- Install-VersionTools-Cli -Version $toolsetVersion
-
- $cliToolFound = (& "$dotnet" tool list --local | Where-Object {$_.Split(' ')[0] -eq $CliToolName})
- if ($null -eq $cliToolFound) {
- Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "The '$CliToolName' tool is not installed."
- ExitWithExitCode 1
- }
-
- Exec-BlockVerbosely {
- & "$dotnet" $CliToolName trim-assets-version `
- --assets-path $InputPath `
- --recursive $Recursive
- Exit-IfNZEC "Sdl"
- }
-}
-catch {
- Write-Host $_
- Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_
- ExitWithExitCode 1
-}
diff --git a/eng/common/template-guidance.md b/eng/common/template-guidance.md
index 98bbc1ded..f772aa3d7 100644
--- a/eng/common/template-guidance.md
+++ b/eng/common/template-guidance.md
@@ -50,7 +50,7 @@ extends:
- task: CopyFiles@2
displayName: Gather build output
inputs:
- SourceFolder: '$(Build.SourcesDirectory)/artifacts/marvel'
+ SourceFolder: '$(System.DefaultWorkingDirectory)/artifacts/marvel'
Contents: '**'
TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/marvel'
```
@@ -71,7 +71,6 @@ eng\common\
source-build.yml (shim)
source-index-stage1.yml (shim)
jobs\
- codeql-build.yml (shim)
jobs.yml (shim)
source-build.yml (shim)
post-build\
@@ -82,14 +81,12 @@ eng\common\
publish-build-artifacts.yml (logic)
publish-pipeline-artifacts.yml (logic)
component-governance.yml (shim)
- generate-sbom.yml (shim)
publish-logs.yml (shim)
retain-build.yml (shim)
send-to-helix.yml (shim)
source-build.yml (shim)
variables\
pool-providers.yml (logic + redirect) # templates/variables/pool-providers.yml will redirect to templates-official/variables/pool-providers.yml if you are running in the internal project
- sdl-variables.yml (logic)
core-templates\
job\
job.yml (logic)
@@ -98,7 +95,6 @@ eng\common\
source-build.yml (logic)
source-index-stage1.yml (logic)
jobs\
- codeql-build.yml (logic)
jobs.yml (logic)
source-build.yml (logic)
post-build\
@@ -107,7 +103,6 @@ eng\common\
setup-maestro-vars.yml (logic)
steps\
component-governance.yml (logic)
- generate-sbom.yml (logic)
publish-build-artifacts.yml (redirect)
publish-logs.yml (logic)
publish-pipeline-artifacts.yml (redirect)
diff --git a/eng/common/templates-official/job/job.yml b/eng/common/templates-official/job/job.yml
index 605692d2f..d68e9fbc2 100644
--- a/eng/common/templates-official/job/job.yml
+++ b/eng/common/templates-official/job/job.yml
@@ -1,23 +1,15 @@
parameters:
-# Sbom related params
- enableSbom: true
runAsPublic: false
- PackageVersion: 9.0.0
- BuildDropPath: '$(Build.SourcesDirectory)/artifacts'
+# Sbom related params, unused now and can eventually be removed
+ enableSbom: unused
+ PackageVersion: unused
+ BuildDropPath: unused
jobs:
- template: /eng/common/core-templates/job/job.yml
parameters:
is1ESPipeline: true
- componentGovernanceSteps:
- - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.enableSbom, 'true')) }}:
- - template: /eng/common/templates/steps/generate-sbom.yml
- parameters:
- PackageVersion: ${{ parameters.packageVersion }}
- BuildDropPath: ${{ parameters.buildDropPath }}
- publishArtifacts: false
-
# publish artifacts
# for 1ES managed templates, use the templateContext.output to handle multiple outputs.
templateContext:
@@ -25,11 +17,19 @@ jobs:
outputs:
- ${{ if ne(parameters.artifacts.publish, '') }}:
- ${{ if and(ne(parameters.artifacts.publish.artifacts, 'false'), ne(parameters.artifacts.publish.artifacts, '')) }}:
- - output: buildArtifacts
+ - output: pipelineArtifact
displayName: Publish pipeline artifacts
- PathtoPublish: '$(Build.ArtifactStagingDirectory)/artifacts'
- ArtifactName: ${{ coalesce(parameters.artifacts.publish.artifacts.name , 'Artifacts_$(Agent.Os)_$(_BuildConfig)') }}
- condition: always()
+ targetPath: '$(Build.ArtifactStagingDirectory)/artifacts'
+ artifactName: ${{ coalesce(parameters.artifacts.publish.artifacts.name , 'Artifacts_$(Agent.Os)_$(_BuildConfig)') }}
+ condition: succeeded()
+ retryCountOnTaskFailure: 10 # for any files being locked
+ continueOnError: true
+ - output: pipelineArtifact
+ displayName: Publish pipeline artifacts
+ targetPath: '$(Build.ArtifactStagingDirectory)/artifacts'
+ artifactName: ${{ coalesce(parameters.artifacts.publish.artifacts.name , 'Artifacts_$(Agent.Os)_$(_BuildConfig)') }}_Attempt$(System.JobAttempt)
+ condition: not(succeeded())
+ retryCountOnTaskFailure: 10 # for any files being locked
continueOnError: true
- ${{ if and(ne(parameters.artifacts.publish.logs, 'false'), ne(parameters.artifacts.publish.logs, '')) }}:
- output: pipelineArtifact
@@ -38,17 +38,18 @@ jobs:
displayName: 'Publish logs'
continueOnError: true
condition: always()
- sbomEnabled: false # we don't need SBOM for logs
+ retryCountOnTaskFailure: 10 # for any files being locked
+ isProduction: false # logs are non-production artifacts
- ${{ if eq(parameters.enablePublishBuildArtifacts, true) }}:
- - output: buildArtifacts
+ - output: pipelineArtifact
displayName: Publish Logs
- PathtoPublish: '$(Build.ArtifactStagingDirectory)/artifacts/log/$(_BuildConfig)'
- publishLocation: Container
- ArtifactName: ${{ coalesce(parameters.enablePublishBuildArtifacts.artifactName, '$(Agent.Os)_$(Agent.JobName)' ) }}
+ targetPath: '$(Build.ArtifactStagingDirectory)/artifacts/log/$(_BuildConfig)'
+ artifactName: ${{ coalesce(parameters.enablePublishBuildArtifacts.artifactName, '$(Agent.Os)_$(Agent.JobName)_Attempt$(System.JobAttempt)' ) }}
continueOnError: true
condition: always()
- sbomEnabled: false # we don't need SBOM for logs
+ retryCountOnTaskFailure: 10 # for any files being locked
+ isProduction: false # logs are non-production artifacts
- ${{ if eq(parameters.enableBuildRetry, 'true') }}:
- output: pipelineArtifact
@@ -56,14 +57,20 @@ jobs:
artifactName: 'BuildConfiguration'
displayName: 'Publish build retry configuration'
continueOnError: true
- sbomEnabled: false # we don't need SBOM for BuildConfiguration
+ retryCountOnTaskFailure: 10 # for any files being locked
+ isProduction: false # BuildConfiguration is a non-production artifact
- - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.enableSbom, 'true')) }}:
+ # V4 publishing: automatically publish staged artifacts as a pipeline artifact.
+ # The artifact name matches the SDK's FutureArtifactName ($(System.PhaseName)_Artifacts),
+ # which is encoded in the asset manifest for downstream publishing to discover.
+ # Jobs can opt in by setting enablePublishing: true.
+ - ${{ if and(eq(parameters.publishingVersion, 4), eq(parameters.enablePublishing, 'true')) }}:
- output: pipelineArtifact
- displayName: Publish SBOM manifest
+ displayName: 'Publish V4 pipeline artifacts'
+ targetPath: '$(Build.ArtifactStagingDirectory)/artifacts'
+ artifactName: '$(System.PhaseName)_Artifacts'
continueOnError: true
- targetPath: $(Build.ArtifactStagingDirectory)/sbom
- artifactName: $(ARTIFACT_NAME)
+ retryCountOnTaskFailure: 10 # for any files being locked
# add any outputs provided via root yaml
- ${{ if ne(parameters.templateContext.outputs, '') }}:
diff --git a/eng/common/templates-official/jobs/codeql-build.yml b/eng/common/templates-official/jobs/codeql-build.yml
deleted file mode 100644
index a726322ec..000000000
--- a/eng/common/templates-official/jobs/codeql-build.yml
+++ /dev/null
@@ -1,7 +0,0 @@
-jobs:
-- template: /eng/common/core-templates/jobs/codeql-build.yml
- parameters:
- is1ESPipeline: true
-
- ${{ each parameter in parameters }}:
- ${{ parameter.key }}: ${{ parameter.value }}
diff --git a/eng/common/templates-official/steps/component-governance.yml b/eng/common/templates-official/steps/get-github-app-token.yml
similarity index 66%
rename from eng/common/templates-official/steps/component-governance.yml
rename to eng/common/templates-official/steps/get-github-app-token.yml
index 30bb3985c..c89f3641a 100644
--- a/eng/common/templates-official/steps/component-governance.yml
+++ b/eng/common/templates-official/steps/get-github-app-token.yml
@@ -1,5 +1,5 @@
steps:
-- template: /eng/common/core-templates/steps/component-governance.yml
+- template: /eng/common/core-templates/steps/get-github-app-token.yml
parameters:
is1ESPipeline: true
diff --git a/eng/common/templates-official/steps/publish-build-artifacts.yml b/eng/common/templates-official/steps/publish-build-artifacts.yml
index 100a3fc98..fcf6637b2 100644
--- a/eng/common/templates-official/steps/publish-build-artifacts.yml
+++ b/eng/common/templates-official/steps/publish-build-artifacts.yml
@@ -24,6 +24,10 @@ parameters:
- name: is1ESPipeline
type: boolean
default: true
+
+- name: retryCountOnTaskFailure
+ type: string
+ default: 10
steps:
- ${{ if ne(parameters.is1ESPipeline, true) }}:
@@ -38,4 +42,5 @@ steps:
PathtoPublish: ${{ parameters.pathToPublish }}
${{ if parameters.artifactName }}:
ArtifactName: ${{ parameters.artifactName }}
-
+ ${{ if parameters.retryCountOnTaskFailure }}:
+ retryCountOnTaskFailure: ${{ parameters.retryCountOnTaskFailure }}
diff --git a/eng/common/templates-official/steps/publish-pipeline-artifacts.yml b/eng/common/templates-official/steps/publish-pipeline-artifacts.yml
index 172f9f0fd..9e5981365 100644
--- a/eng/common/templates-official/steps/publish-pipeline-artifacts.yml
+++ b/eng/common/templates-official/steps/publish-pipeline-artifacts.yml
@@ -24,5 +24,7 @@ steps:
artifactName: ${{ parameters.args.artifactName }}
${{ if parameters.args.properties }}:
properties: ${{ parameters.args.properties }}
- ${{ if parameters.args.sbomEnabled }}:
+ ${{ if ne(parameters.args.sbomEnabled, '') }}:
sbomEnabled: ${{ parameters.args.sbomEnabled }}
+ ${{ if ne(parameters.args.isProduction, '') }}:
+ isProduction: ${{ parameters.args.isProduction }}
diff --git a/eng/common/templates-official/variables/pool-providers.yml b/eng/common/templates-official/variables/pool-providers.yml
index 1f308b24e..2cc3ae305 100644
--- a/eng/common/templates-official/variables/pool-providers.yml
+++ b/eng/common/templates-official/variables/pool-providers.yml
@@ -23,7 +23,7 @@
#
# pool:
# name: $(DncEngInternalBuildPool)
-# image: 1es-windows-2022
+# image: windows.vs2026.amd64
variables:
# Coalesce the target and source branches so we know when a PR targets a release branch
diff --git a/eng/common/templates-official/variables/sdl-variables.yml b/eng/common/templates-official/variables/sdl-variables.yml
deleted file mode 100644
index dbdd66d4a..000000000
--- a/eng/common/templates-official/variables/sdl-variables.yml
+++ /dev/null
@@ -1,7 +0,0 @@
-variables:
-# The Guardian version specified in 'eng/common/sdl/packages.config'. This value must be kept in
-# sync with the packages.config file.
-- name: DefaultGuardianVersion
- value: 0.109.0
-- name: GuardianPackagesConfigFile
- value: $(Build.SourcesDirectory)\eng\common\sdl\packages.config
\ No newline at end of file
diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml
index d1aeb92fc..85501406a 100644
--- a/eng/common/templates/job/job.yml
+++ b/eng/common/templates/job/job.yml
@@ -1,12 +1,12 @@
parameters:
enablePublishBuildArtifacts: false
- disableComponentGovernance: ''
- componentGovernanceIgnoreDirectories: ''
-# Sbom related params
- enableSbom: true
runAsPublic: false
- PackageVersion: 9.0.0
- BuildDropPath: '$(Build.SourcesDirectory)/artifacts'
+# CG related params, unused now and can eventually be removed
+ disableComponentGovernance: unused
+# Sbom related params, unused now and can eventually be removed
+ enableSbom: unused
+ PackageVersion: unused
+ BuildDropPath: unused
jobs:
- template: /eng/common/core-templates/job/job.yml
@@ -21,31 +21,29 @@ jobs:
- ${{ each step in parameters.steps }}:
- ${{ step }}
- componentGovernanceSteps:
- - template: /eng/common/templates/steps/component-governance.yml
- parameters:
- ${{ if eq(parameters.disableComponentGovernance, '') }}:
- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.runAsPublic, 'false'), or(startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/dotnet/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/microsoft/'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))) }}:
- disableComponentGovernance: false
- ${{ else }}:
- disableComponentGovernance: true
- ${{ else }}:
- disableComponentGovernance: ${{ parameters.disableComponentGovernance }}
- componentGovernanceIgnoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }}
-
artifactPublishSteps:
- ${{ if ne(parameters.artifacts.publish, '') }}:
- ${{ if and(ne(parameters.artifacts.publish.artifacts, 'false'), ne(parameters.artifacts.publish.artifacts, '')) }}:
- - template: /eng/common/core-templates/steps/publish-build-artifacts.yml
+ - template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml
parameters:
is1ESPipeline: false
args:
displayName: Publish pipeline artifacts
- pathToPublish: '$(Build.ArtifactStagingDirectory)/artifacts'
- publishLocation: Container
+ targetPath: '$(Build.ArtifactStagingDirectory)/artifacts'
artifactName: ${{ coalesce(parameters.artifacts.publish.artifacts.name , 'Artifacts_$(Agent.Os)_$(_BuildConfig)') }}
continueOnError: true
- condition: always()
+ condition: succeeded()
+ retryCountOnTaskFailure: 10 # for any files being locked
+ - template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml
+ parameters:
+ is1ESPipeline: false
+ args:
+ displayName: Publish pipeline artifacts
+ targetPath: '$(Build.ArtifactStagingDirectory)/artifacts'
+ artifactName: ${{ coalesce(parameters.artifacts.publish.artifacts.name , 'Artifacts_$(Agent.Os)_$(_BuildConfig)') }}_Attempt$(System.JobAttempt)
+ continueOnError: true
+ condition: not(succeeded())
+ retryCountOnTaskFailure: 10 # for any files being locked
- ${{ if and(ne(parameters.artifacts.publish.logs, 'false'), ne(parameters.artifacts.publish.logs, '')) }}:
- template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml
parameters:
@@ -56,27 +54,27 @@ jobs:
displayName: 'Publish logs'
continueOnError: true
condition: always()
- sbomEnabled: false # we don't need SBOM for logs
+ retryCountOnTaskFailure: 10 # for any files being locked
- ${{ if ne(parameters.enablePublishBuildArtifacts, 'false') }}:
- - template: /eng/common/core-templates/steps/publish-build-artifacts.yml
+ - template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml
parameters:
is1ESPipeline: false
args:
displayName: Publish Logs
- pathToPublish: '$(Build.ArtifactStagingDirectory)/artifacts/log/$(_BuildConfig)'
- publishLocation: Container
- artifactName: ${{ coalesce(parameters.enablePublishBuildArtifacts.artifactName, '$(Agent.Os)_$(Agent.JobName)' ) }}
+ targetPath: '$(Build.ArtifactStagingDirectory)/artifacts/log/$(_BuildConfig)'
+ artifactName: ${{ coalesce(parameters.enablePublishBuildArtifacts.artifactName, '$(Agent.Os)_$(Agent.JobName)_Attempt$(System.JobAttempt)' ) }}
continueOnError: true
condition: always()
+ retryCountOnTaskFailure: 10 # for any files being locked
- ${{ if eq(parameters.enableBuildRetry, 'true') }}:
- template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml
parameters:
is1ESPipeline: false
args:
- targetPath: '$(Build.SourcesDirectory)\eng\common\BuildConfiguration'
+ targetPath: '$(System.DefaultWorkingDirectory)\eng\common\BuildConfiguration'
artifactName: 'BuildConfiguration'
displayName: 'Publish build retry configuration'
continueOnError: true
- sbomEnabled: false # we don't need SBOM for BuildConfiguration
+ retryCountOnTaskFailure: 10 # for any files being locked
diff --git a/eng/common/templates/jobs/codeql-build.yml b/eng/common/templates/jobs/codeql-build.yml
deleted file mode 100644
index 517f24d6a..000000000
--- a/eng/common/templates/jobs/codeql-build.yml
+++ /dev/null
@@ -1,7 +0,0 @@
-jobs:
-- template: /eng/common/core-templates/jobs/codeql-build.yml
- parameters:
- is1ESPipeline: false
-
- ${{ each parameter in parameters }}:
- ${{ parameter.key }}: ${{ parameter.value }}
diff --git a/eng/common/templates/steps/component-governance.yml b/eng/common/templates/steps/get-github-app-token.yml
similarity index 66%
rename from eng/common/templates/steps/component-governance.yml
rename to eng/common/templates/steps/get-github-app-token.yml
index c12a5f8d2..79e182c64 100644
--- a/eng/common/templates/steps/component-governance.yml
+++ b/eng/common/templates/steps/get-github-app-token.yml
@@ -1,5 +1,5 @@
steps:
-- template: /eng/common/core-templates/steps/component-governance.yml
+- template: /eng/common/core-templates/steps/get-github-app-token.yml
parameters:
is1ESPipeline: false
diff --git a/eng/common/templates/steps/publish-build-artifacts.yml b/eng/common/templates/steps/publish-build-artifacts.yml
index 6428a98df..605e602e9 100644
--- a/eng/common/templates/steps/publish-build-artifacts.yml
+++ b/eng/common/templates/steps/publish-build-artifacts.yml
@@ -25,6 +25,10 @@ parameters:
type: string
default: 'Container'
+- name: retryCountOnTaskFailure
+ type: string
+ default: 10
+
steps:
- ${{ if eq(parameters.is1ESPipeline, true) }}:
- 'eng/common/templates cannot be referenced from a 1ES managed template': error
@@ -37,4 +41,6 @@ steps:
PublishLocation: ${{ parameters.publishLocation }}
PathtoPublish: ${{ parameters.pathToPublish }}
${{ if parameters.artifactName }}:
- ArtifactName: ${{ parameters.artifactName }}
\ No newline at end of file
+ ArtifactName: ${{ parameters.artifactName }}
+ ${{ if parameters.retryCountOnTaskFailure }}:
+ retryCountOnTaskFailure: ${{ parameters.retryCountOnTaskFailure }}
diff --git a/eng/common/templates/steps/vmr-sync.yml b/eng/common/templates/steps/vmr-sync.yml
new file mode 100644
index 000000000..cdc6a28ff
--- /dev/null
+++ b/eng/common/templates/steps/vmr-sync.yml
@@ -0,0 +1,186 @@
+### These steps synchronize new code from product repositories into the VMR (https://github.com/dotnet/dotnet).
+### They initialize the darc CLI and pull the new updates.
+### Changes are applied locally onto the already cloned VMR (located in $vmrPath).
+
+parameters:
+- name: targetRef
+ displayName: Target revision in dotnet/ to synchronize
+ type: string
+ default: $(Build.SourceVersion)
+
+- name: vmrPath
+ displayName: Path where the dotnet/dotnet is checked out to
+ type: string
+ default: $(Agent.BuildDirectory)/vmr
+
+- name: additionalSyncs
+ displayName: Optional list of package names whose repo's source will also be synchronized in the local VMR, e.g. NuGet.Protocol
+ type: object
+ default: []
+
+steps:
+- checkout: vmr
+ displayName: Clone dotnet/dotnet
+ path: vmr
+ clean: true
+
+- checkout: self
+ displayName: Clone $(Build.Repository.Name)
+ path: repo
+ fetchDepth: 0
+
+# This step is needed so that when we get a detached HEAD / shallow clone,
+# we still pull the commit into the temporary repo clone to use it during the sync.
+# Also unshallow the clone so that forwardflow command would work.
+- script: |
+ git branch repo-head
+ git rev-parse HEAD
+ displayName: Label PR commit
+ workingDirectory: $(Agent.BuildDirectory)/repo
+
+- script: |
+ git config --global user.name "dotnet-maestro[bot]"
+ git config --global user.email "dotnet-maestro[bot]@users.noreply.github.com"
+ displayName: Set git author to dotnet-maestro[bot]
+ workingDirectory: ${{ parameters.vmrPath }}
+
+- script: |
+ ./eng/common/vmr-sync.sh \
+ --vmr ${{ parameters.vmrPath }} \
+ --tmp $(Agent.TempDirectory) \
+ --azdev-pat '$(AzdoToken)' \
+ --ci \
+ --debug
+
+ if [ "$?" -ne 0 ]; then
+ echo "##vso[task.logissue type=error]Failed to synchronize the VMR"
+ exit 1
+ fi
+ displayName: Sync repo into VMR (Unix)
+ condition: ne(variables['Agent.OS'], 'Windows_NT')
+ workingDirectory: $(Agent.BuildDirectory)/repo
+
+- script: |
+ git config --global diff.astextplain.textconv echo
+ git config --system core.longpaths true
+ displayName: Configure Windows git (longpaths, astextplain)
+ condition: eq(variables['Agent.OS'], 'Windows_NT')
+
+- powershell: |
+ ./eng/common/vmr-sync.ps1 `
+ -vmr ${{ parameters.vmrPath }} `
+ -tmp $(Agent.TempDirectory) `
+ -azdevPat '$(AzdoToken)' `
+ -ci `
+ -debugOutput
+
+ if ($LASTEXITCODE -ne 0) {
+ echo "##vso[task.logissue type=error]Failed to synchronize the VMR"
+ exit 1
+ }
+ displayName: Sync repo into VMR (Windows)
+ condition: eq(variables['Agent.OS'], 'Windows_NT')
+ workingDirectory: $(Agent.BuildDirectory)/repo
+
+- ${{ if eq(variables['Build.Reason'], 'PullRequest') }}:
+ - task: CopyFiles@2
+ displayName: Collect failed patches
+ condition: failed()
+ inputs:
+ SourceFolder: '$(Agent.TempDirectory)'
+ Contents: '*.patch'
+ TargetFolder: '$(Build.ArtifactStagingDirectory)/FailedPatches'
+
+ - publish: '$(Build.ArtifactStagingDirectory)/FailedPatches'
+ artifact: $(System.JobDisplayName)_FailedPatches
+ displayName: Upload failed patches
+ condition: failed()
+
+- ${{ each assetName in parameters.additionalSyncs }}:
+ # The vmr-sync script ends up staging files in the local VMR so we have to commit those
+ - script:
+ git commit --allow-empty -am "Forward-flow $(Build.Repository.Name)"
+ displayName: Commit local VMR changes
+ workingDirectory: ${{ parameters.vmrPath }}
+
+ - script: |
+ set -ex
+
+ echo "Searching for details of asset ${{ assetName }}..."
+
+ # Use darc to get dependencies information
+ dependencies=$(./.dotnet/dotnet darc get-dependencies --name '${{ assetName }}' --ci)
+
+ # Extract repository URL and commit hash
+ repository=$(echo "$dependencies" | grep 'Repo:' | sed 's/Repo:[[:space:]]*//' | head -1)
+
+ if [ -z "$repository" ]; then
+ echo "##vso[task.logissue type=error]Asset ${{ assetName }} not found in the dependency list"
+ exit 1
+ fi
+
+ commit=$(echo "$dependencies" | grep 'Commit:' | sed 's/Commit:[[:space:]]*//' | head -1)
+
+ echo "Updating the VMR from $repository / $commit..."
+ cd ..
+ git clone $repository ${{ assetName }}
+ cd ${{ assetName }}
+ git checkout $commit
+ git branch "sync/$commit"
+
+ ./eng/common/vmr-sync.sh \
+ --vmr ${{ parameters.vmrPath }} \
+ --tmp $(Agent.TempDirectory) \
+ --azdev-pat '$(dn-bot-all-orgs-code-r)' \
+ --ci \
+ --debug
+
+ if [ "$?" -ne 0 ]; then
+ echo "##vso[task.logissue type=error]Failed to synchronize the VMR"
+ exit 1
+ fi
+ displayName: Sync ${{ assetName }} into (Unix)
+ condition: ne(variables['Agent.OS'], 'Windows_NT')
+ workingDirectory: $(Agent.BuildDirectory)/repo
+
+ - powershell: |
+ $ErrorActionPreference = 'Stop'
+
+ Write-Host "Searching for details of asset ${{ assetName }}..."
+
+ $dependencies = .\.dotnet\dotnet darc get-dependencies --name '${{ assetName }}' --ci
+
+ $repository = $dependencies | Select-String -Pattern 'Repo:\s+([^\s]+)' | Select-Object -First 1
+ $repository -match 'Repo:\s+([^\s]+)' | Out-Null
+ $repository = $matches[1]
+
+ if ($repository -eq $null) {
+ Write-Error "Asset ${{ assetName }} not found in the dependency list"
+ exit 1
+ }
+
+ $commit = $dependencies | Select-String -Pattern 'Commit:\s+([^\s]+)' | Select-Object -First 1
+ $commit -match 'Commit:\s+([^\s]+)' | Out-Null
+ $commit = $matches[1]
+
+ Write-Host "Updating the VMR from $repository / $commit..."
+ cd ..
+ git clone $repository ${{ assetName }}
+ cd ${{ assetName }}
+ git checkout $commit
+ git branch "sync/$commit"
+
+ .\eng\common\vmr-sync.ps1 `
+ -vmr ${{ parameters.vmrPath }} `
+ -tmp $(Agent.TempDirectory) `
+ -azdevPat '$(dn-bot-all-orgs-code-r)' `
+ -ci `
+ -debugOutput
+
+ if ($LASTEXITCODE -ne 0) {
+ echo "##vso[task.logissue type=error]Failed to synchronize the VMR"
+ exit 1
+ }
+ displayName: Sync ${{ assetName }} into (Windows)
+ condition: ne(variables['Agent.OS'], 'Windows_NT')
+ workingDirectory: $(Agent.BuildDirectory)/repo
diff --git a/eng/common/templates/variables/pool-providers.yml b/eng/common/templates/variables/pool-providers.yml
index e0b19c14a..587770f0a 100644
--- a/eng/common/templates/variables/pool-providers.yml
+++ b/eng/common/templates/variables/pool-providers.yml
@@ -23,7 +23,7 @@
#
# pool:
# name: $(DncEngInternalBuildPool)
-# demands: ImageOverride -equals windows.vs2019.amd64
+# demands: ImageOverride -equals windows.vs2026.amd64
variables:
- ${{ if eq(variables['System.TeamProject'], 'internal') }}:
- template: /eng/common/templates-official/variables/pool-providers.yml
diff --git a/eng/common/templates/vmr-build-pr.yml b/eng/common/templates/vmr-build-pr.yml
new file mode 100644
index 000000000..d24de9352
--- /dev/null
+++ b/eng/common/templates/vmr-build-pr.yml
@@ -0,0 +1,43 @@
+# This pipeline is used for running the VMR verification of the PR changes in repo-level PRs.
+#
+# It will run a full set of verification jobs defined in:
+# https://github.com/dotnet/dotnet/blob/10060d128e3f470e77265f8490f5e4f72dae738e/eng/pipelines/templates/stages/vmr-build.yml#L27-L38
+#
+# For repos that do not need to run the full set, you would do the following:
+#
+# 1. Copy this YML file to a repo-specific location, i.e. outside of eng/common.
+#
+# 2. Add `verifications` parameter to VMR template reference
+#
+# Examples:
+# - For source-build stage 1 verification, add the following:
+# verifications: [ "source-build-stage1" ]
+#
+# - For Windows only verifications, add the following:
+# verifications: [ "unified-build-windows-x64", "unified-build-windows-x86" ]
+
+trigger: none
+pr: none
+
+variables:
+- template: /eng/common/templates/variables/pool-providers.yml@self
+
+- name: skipComponentGovernanceDetection # we run CG on internal builds only
+ value: true
+
+- name: Codeql.Enabled # we run CodeQL on internal builds only
+ value: false
+
+resources:
+ repositories:
+ - repository: vmr
+ type: github
+ name: dotnet/dotnet
+ endpoint: public
+ ref: refs/heads/main # Set to whatever VMR branch the PR build should insert into
+
+stages:
+- template: /eng/pipelines/templates/stages/vmr-build.yml@vmr
+ parameters:
+ isBuiltFromVmr: false
+ scope: lite
diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1
index 95ccdf82e..e84033dad 100644
--- a/eng/common/tools.ps1
+++ b/eng/common/tools.ps1
@@ -15,7 +15,7 @@
# Set to true to use the pipelines logger which will enable Azure logging output.
# https://github.com/Microsoft/azure-pipelines-tasks/blob/master/docs/authoring/commands.md
-# This flag is meant as a temporary opt-opt for the feature while validate it across
+# This flag is meant as a temporary opt-in for the feature while validating it across
# our consumers. It will be deleted in the future.
[bool]$pipelinesLog = if (Test-Path variable:pipelinesLog) { $pipelinesLog } else { $ci }
@@ -31,9 +31,17 @@
# Set to true to reuse msbuild nodes. Recommended to not reuse on CI.
[bool]$nodeReuse = if (Test-Path variable:nodeReuse) { $nodeReuse } else { !$ci }
+# Set to true to build with MSBuild's multi-threaded mode (-mt). Opt-in for now, so off unless it was
+# explicitly requested. It's intended to become the default for local builds once it has proven out.
+[bool]$msbuildMultiThreaded = if (Test-Path variable:msbuildMultiThreaded) { $msbuildMultiThreaded } else { $false }
+
# Configures warning treatment in msbuild.
[bool]$warnAsError = if (Test-Path variable:warnAsError) { $warnAsError } else { $true }
+# Specifies semi-colon delimited list of warning codes that should not be treated as errors.
+# Defaults to NuGet Audit warning codes NU1901-NU1904.
+[string]$warnNotAsError = if ((Test-Path variable:warnNotAsError) -and $warnNotAsError) { $warnNotAsError } else { 'NU1901;NU1902;NU1903;NU1904' }
+
# Specifies which msbuild engine to use for build: 'vs', 'dotnet' or unspecified (determined based on presence of tools.vs in global.json).
[string]$msbuildEngine = if (Test-Path variable:msbuildEngine) { $msbuildEngine } else { $null }
@@ -65,10 +73,10 @@ $ErrorActionPreference = 'Stop'
# Base-64 encoded SAS token that has permission to storage container described by $runtimeSourceFeed
[string]$runtimeSourceFeedKey = if (Test-Path variable:runtimeSourceFeedKey) { $runtimeSourceFeedKey } else { $null }
-# True if the build is a product build
-[bool]$productBuild = if (Test-Path variable:productBuild) { $productBuild } else { $false }
+# True when the build is running within the VMR.
+[bool]$fromVMR = if (Test-Path variable:fromVMR) { $fromVMR } else { $false }
-[String[]]$properties = if (Test-Path variable:properties) { $properties } else { @() }
+[bool]$disablePipelineSetResult = if (Test-Path variable:disablePipelineSetResult) { $disablePipelineSetResult } else { $false }
function Create-Directory ([string[]] $path) {
New-Item -Path $path -Force -ItemType 'Directory' | Out-Null
@@ -159,9 +167,6 @@ function InitializeDotNetCli([bool]$install, [bool]$createSdkLocationFile) {
return $global:_DotNetInstallDir
}
- # Don't resolve runtime, shared framework, or SDK from other locations to ensure build determinism
- $env:DOTNET_MULTILEVEL_LOOKUP=0
-
# Disable first run since we do not need all ASP.NET packages restored.
$env:DOTNET_NOLOGO=1
@@ -187,7 +192,11 @@ function InitializeDotNetCli([bool]$install, [bool]$createSdkLocationFile) {
if ((-not $globalJsonHasRuntimes) -and (-not [string]::IsNullOrEmpty($env:DOTNET_INSTALL_DIR)) -and (Test-Path(Join-Path $env:DOTNET_INSTALL_DIR "sdk\$dotnetSdkVersion"))) {
$dotnetRoot = $env:DOTNET_INSTALL_DIR
} else {
- $dotnetRoot = Join-Path $RepoRoot '.dotnet'
+ if (-not [string]::IsNullOrEmpty($env:DOTNET_GLOBAL_INSTALL_DIR)) {
+ $dotnetRoot = $env:DOTNET_GLOBAL_INSTALL_DIR
+ } else {
+ $dotnetRoot = Join-Path $RepoRoot '.dotnet'
+ }
if (-not (Test-Path(Join-Path $dotnetRoot "sdk\$dotnetSdkVersion"))) {
if ($install) {
@@ -227,7 +236,6 @@ function InitializeDotNetCli([bool]$install, [bool]$createSdkLocationFile) {
# Make Sure that our bootstrapped dotnet cli is available in future steps of the Azure Pipelines build
Write-PipelinePrependPath -Path $dotnetRoot
- Write-PipelineSetVariable -Name 'DOTNET_MULTILEVEL_LOOKUP' -Value '0'
Write-PipelineSetVariable -Name 'DOTNET_NOLOGO' -Value '1'
return $global:_DotNetInstallDir = $dotnetRoot
@@ -259,14 +267,27 @@ function Retry($downloadBlock, $maxRetries = 5) {
function GetDotNetInstallScript([string] $dotnetRoot) {
$installScript = Join-Path $dotnetRoot 'dotnet-install.ps1'
+ $shouldDownload = $false
+
if (!(Test-Path $installScript)) {
+ $shouldDownload = $true
+ } else {
+ # Check if the script is older than 30 days
+ $fileAge = (Get-Date) - (Get-Item $installScript).LastWriteTime
+ if ($fileAge.Days -gt 30) {
+ Write-Host "Existing install script is too old, re-downloading..."
+ $shouldDownload = $true
+ }
+ }
+
+ if ($shouldDownload) {
Create-Directory $dotnetRoot
$ProgressPreference = 'SilentlyContinue' # Don't display the console progress UI - it's a huge perf hit
- $uri = "https://builds.dotnet.microsoft.com/dotnet/scripts/v1/dotnet-install.ps1"
+ $uri = "https://builds.dotnet.microsoft.com/dotnet/scripts/$dotnetInstallScriptVersion/dotnet-install.ps1"
Retry({
Write-Host "GET $uri"
- Invoke-WebRequest $uri -OutFile $installScript
+ Invoke-WebRequest $uri -UseBasicParsing -OutFile $installScript
})
}
@@ -288,6 +309,8 @@ function InstallDotNet([string] $dotnetRoot,
$dotnetVersionLabel = "'sdk v$version'"
+ # For performance this check is duplicated in src/Microsoft.DotNet.Arcade.Sdk/src/InstallDotNetCore.cs
+ # if you are making changes here, consider if you need to make changes there as well.
if ($runtime -ne '' -and $runtime -ne 'sdk') {
$runtimePath = $dotnetRoot
$runtimePath = $runtimePath + "\shared"
@@ -363,12 +386,11 @@ function InstallDotNet([string] $dotnetRoot,
#
# 1. MSBuild from an active VS command prompt
# 2. MSBuild from a compatible VS installation
-# 3. MSBuild from the xcopy tool package
#
# Returns full path to msbuild.exe.
# Throws on failure.
#
-function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = $null) {
+function InitializeVisualStudioMSBuild([object]$vsRequirements = $null) {
if (-not (IsWindowsPlatform)) {
throw "Cannot initialize Visual Studio on non-Windows"
}
@@ -378,13 +400,7 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements =
}
# Minimum VS version to require.
- $vsMinVersionReqdStr = '17.7'
- $vsMinVersionReqd = [Version]::new($vsMinVersionReqdStr)
-
- # If the version of msbuild is going to be xcopied,
- # use this version. Version matches a package here:
- # https://dev.azure.com/dnceng/public/_artifacts/feed/dotnet-eng/NuGet/Microsoft.DotNet.Arcade.MSBuild.Xcopy/versions/17.13.0
- $defaultXCopyMSBuildVersion = '17.13.0'
+ $vsMinVersionReqdStr = '18.0'
if (!$vsRequirements) {
if (Get-Member -InputObject $GlobalJson.tools -Name 'vs') {
@@ -414,7 +430,7 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements =
}
}
- # Locate Visual Studio installation or download x-copy msbuild.
+ # Locate Visual Studio installation.
$vsInfo = LocateVisualStudio $vsRequirements
if ($vsInfo -ne $null) {
# Ensure vsInstallDir has a trailing slash
@@ -423,47 +439,37 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements =
InitializeVisualStudioEnvironmentVariables $vsInstallDir $vsMajorVersion
} else {
- if (Get-Member -InputObject $GlobalJson.tools -Name 'xcopy-msbuild') {
- $xcopyMSBuildVersion = $GlobalJson.tools.'xcopy-msbuild'
- $vsMajorVersion = $xcopyMSBuildVersion.Split('.')[0]
- } else {
- #if vs version provided in global.json is incompatible (too low) then use the default version for xcopy msbuild download
- if($vsMinVersion -lt $vsMinVersionReqd){
- Write-Host "Using xcopy-msbuild version of $defaultXCopyMSBuildVersion since VS version $vsMinVersionStr provided in global.json is not compatible"
- $xcopyMSBuildVersion = $defaultXCopyMSBuildVersion
- $vsMajorVersion = $xcopyMSBuildVersion.Split('.')[0]
- }
- else{
- # If the VS version IS compatible, look for an xcopy msbuild package
- # with a version matching VS.
- # Note: If this version does not exist, then an explicit version of xcopy msbuild
- # can be specified in global.json. This will be required for pre-release versions of msbuild.
- $vsMajorVersion = $vsMinVersion.Major
- $vsMinorVersion = $vsMinVersion.Minor
- $xcopyMSBuildVersion = "$vsMajorVersion.$vsMinorVersion.0"
- }
- }
-
- $vsInstallDir = $null
- if ($xcopyMSBuildVersion.Trim() -ine "none") {
- $vsInstallDir = InitializeXCopyMSBuild $xcopyMSBuildVersion $install
- if ($vsInstallDir -eq $null) {
- throw "Could not xcopy msbuild. Please check that package 'Microsoft.DotNet.Arcade.MSBuild.Xcopy @ $xcopyMSBuildVersion' exists on feed 'dotnet-eng'."
- }
- }
- if ($vsInstallDir -eq $null) {
- throw 'Unable to find Visual Studio that has required version and components installed'
- }
+ throw 'Unable to find Visual Studio that has required version and components installed'
}
$msbuildVersionDir = if ([int]$vsMajorVersion -lt 16) { "$vsMajorVersion.0" } else { "Current" }
$local:BinFolder = Join-Path $vsInstallDir "MSBuild\$msbuildVersionDir\Bin"
- $local:Prefer64bit = if (Get-Member -InputObject $vsRequirements -Name 'Prefer64bit') { $vsRequirements.Prefer64bit } else { $false }
- if ($local:Prefer64bit -and (Test-Path(Join-Path $local:BinFolder "amd64"))) {
- $global:_MSBuildExe = Join-Path $local:BinFolder "amd64\msbuild.exe"
- } else {
- $global:_MSBuildExe = Join-Path $local:BinFolder "msbuild.exe"
+
+ # Use the MSBuild matching the host's process architecture (e.g. amd64 or arm64),
+ # falling back to the 32-bit MSBuild in the root Bin folder when no matching subfolder exists.
+
+ # Determine the architecture of the current process, accounting for a 32-bit process
+ # running on a 64-bit OS (PROCESSOR_ARCHITEW6432 holds the real machine architecture).
+ $local:ProcessArchitecture = $env:PROCESSOR_ARCHITECTURE
+ if (($local:ProcessArchitecture -eq 'x86') -and ($env:PROCESSOR_ARCHITEW6432)) {
+ $local:ProcessArchitecture = $env:PROCESSOR_ARCHITEW6432
+ }
+
+ # Map the architecture to the corresponding MSBuild subfolder. The 32-bit MSBuild lives in the
+ # root Bin folder, so x86 maps to an empty subfolder.
+ $local:MSBuildArchSubFolder = switch ($local:ProcessArchitecture) {
+ 'AMD64' { 'amd64' }
+ 'ARM64' { 'arm64' }
+ default { '' }
+ }
+
+ $global:_MSBuildExe = Join-Path $local:BinFolder "msbuild.exe"
+ if ($local:MSBuildArchSubFolder) {
+ $local:ArchMSBuildExe = Join-Path $local:BinFolder (Join-Path $local:MSBuildArchSubFolder "msbuild.exe")
+ if (Test-Path $local:ArchMSBuildExe) {
+ $global:_MSBuildExe = $local:ArchMSBuildExe
+ }
}
return $global:_MSBuildExe
@@ -480,38 +486,6 @@ function InitializeVisualStudioEnvironmentVariables([string] $vsInstallDir, [str
}
}
-function InstallXCopyMSBuild([string]$packageVersion) {
- return InitializeXCopyMSBuild $packageVersion -install $true
-}
-
-function InitializeXCopyMSBuild([string]$packageVersion, [bool]$install) {
- $packageName = 'Microsoft.DotNet.Arcade.MSBuild.Xcopy'
- $packageDir = Join-Path $ToolsDir "msbuild\$packageVersion"
- $packagePath = Join-Path $packageDir "$packageName.$packageVersion.nupkg"
-
- if (!(Test-Path $packageDir)) {
- if (!$install) {
- return $null
- }
-
- Create-Directory $packageDir
-
- Write-Host "Downloading $packageName $packageVersion"
- $ProgressPreference = 'SilentlyContinue' # Don't display the console progress UI - it's a huge perf hit
- Retry({
- Invoke-WebRequest "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/flat2/$packageName/$packageVersion/$packageName.$packageVersion.nupkg" -OutFile $packagePath
- })
-
- if (!(Test-Path $packagePath)) {
- Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "See https://dev.azure.com/dnceng/internal/_wiki/wikis/DNCEng%20Services%20Wiki/1074/Updating-Microsoft.DotNet.Arcade.MSBuild.Xcopy-WAS-RoslynTools.MSBuild-(xcopy-msbuild)-generation?anchor=troubleshooting for help troubleshooting issues with XCopy MSBuild"
- throw
- }
- Unzip $packagePath $packageDir
- }
-
- return Join-Path $packageDir 'tools'
-}
-
#
# Locates Visual Studio instance that meets the minimal requirements specified by tools.vs object in global.json.
#
@@ -533,7 +507,7 @@ function LocateVisualStudio([object]$vsRequirements = $null){
if (Get-Member -InputObject $GlobalJson.tools -Name 'vswhere') {
$vswhereVersion = $GlobalJson.tools.vswhere
} else {
- $vswhereVersion = '2.5.2'
+ $vswhereVersion = '3.1.7'
}
$vsWhereDir = Join-Path $ToolsDir "vswhere\$vswhereVersion"
@@ -541,25 +515,33 @@ function LocateVisualStudio([object]$vsRequirements = $null){
if (!(Test-Path $vsWhereExe)) {
Create-Directory $vsWhereDir
- Write-Host 'Downloading vswhere'
+ Write-Host "Downloading vswhere $vswhereVersion"
+ $ProgressPreference = 'SilentlyContinue' # Don't display the console progress UI - it's a huge perf hit
Retry({
- Invoke-WebRequest "https://netcorenativeassets.blob.core.windows.net/resource-packages/external/windows/vswhere/$vswhereVersion/vswhere.exe" -OutFile $vswhereExe
+ Invoke-WebRequest "https://netcorenativeassets.blob.core.windows.net/resource-packages/external/windows/vswhere/$vswhereVersion/vswhere.exe" -UseBasicParsing -OutFile $vswhereExe
})
}
- if (!$vsRequirements) { $vsRequirements = $GlobalJson.tools.vs }
+ if (!$vsRequirements) {
+ if (Get-Member -InputObject $GlobalJson.tools -Name 'vs' -ErrorAction SilentlyContinue) {
+ $vsRequirements = $GlobalJson.tools.vs
+ } else {
+ $vsRequirements = $null
+ }
+ }
+
$args = @('-latest', '-format', 'json', '-requires', 'Microsoft.Component.MSBuild', '-products', '*')
if (!$excludePrereleaseVS) {
$args += '-prerelease'
}
- if (Get-Member -InputObject $vsRequirements -Name 'version') {
+ if ($vsRequirements -and (Get-Member -InputObject $vsRequirements -Name 'version' -ErrorAction SilentlyContinue)) {
$args += '-version'
$args += $vsRequirements.version
}
- if (Get-Member -InputObject $vsRequirements -Name 'components') {
+ if ($vsRequirements -and (Get-Member -InputObject $vsRequirements -Name 'components' -ErrorAction SilentlyContinue)) {
foreach ($component in $vsRequirements.components) {
$args += '-requires'
$args += $component
@@ -572,11 +554,26 @@ function LocateVisualStudio([object]$vsRequirements = $null){
return $null
}
+ if ($null -eq $vsInfo -or $vsInfo.Count -eq 0) {
+ throw "No instance of Visual Studio meeting the requirements specified was found. Requirements: $($args -join ' ')"
+ return $null
+ }
+
# use first matching instance
return $vsInfo[0]
}
function InitializeBuildTool() {
+ # Allow a caller (e.g. a bootstrap script running out-of-proc) to inject the build tool via
+ # environment variables instead of the in-proc $global:_BuildTool variable. Only Path and
+ # Command are consumed by the MSBuild function below, so those are all that's needed.
+ if ($env:_BuildToolPath) {
+ return $global:_BuildTool = @{
+ Path = $env:_BuildToolPath
+ Command = $env:_BuildToolCommand
+ }
+ }
+
if (Test-Path variable:global:_BuildTool) {
# If the requested msbuild parameters do not match, clear the cached variables.
if($global:_BuildTool.Contains('ExcludePrereleaseVS') -and $global:_BuildTool.ExcludePrereleaseVS -ne $excludePrereleaseVS) {
@@ -604,16 +601,16 @@ function InitializeBuildTool() {
}
$dotnetPath = Join-Path $dotnetRoot (GetExecutableFileName 'dotnet')
- $buildTool = @{ Path = $dotnetPath; Command = 'msbuild'; Tool = 'dotnet'; Framework = 'net' }
+ $buildTool = @{ Path = $dotnetPath; Command = 'msbuild' }
} elseif ($msbuildEngine -eq "vs") {
try {
- $msbuildPath = InitializeVisualStudioMSBuild -install:$restore
+ $msbuildPath = InitializeVisualStudioMSBuild
} catch {
Write-PipelineTelemetryError -Category 'InitializeToolset' -Message $_
ExitWithExitCode 1
}
- $buildTool = @{ Path = $msbuildPath; Command = ""; Tool = "vs"; Framework = "netframework"; ExcludePrereleaseVS = $excludePrereleaseVS }
+ $buildTool = @{ Path = $msbuildPath; Command = ""; ExcludePrereleaseVS = $excludePrereleaseVS }
} else {
Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Unexpected value of -msbuildEngine: '$msbuildEngine'."
ExitWithExitCode 1
@@ -636,17 +633,16 @@ function GetDefaultMSBuildEngine() {
ExitWithExitCode 1
}
-function GetNuGetPackageCachePath() {
+function InitializeNuGetPackageCachePath() {
if ($env:NUGET_PACKAGES -eq $null) {
# Use local cache on CI to ensure deterministic build.
- # Avoid using the http cache as workaround for https://github.com/NuGet/Home/issues/3116
# use global cache in dev builds to avoid cost of downloading packages.
# For directory normalization, see also: https://github.com/NuGet/Home/issues/7968
if ($useGlobalNuGetCache) {
- $env:NUGET_PACKAGES = Join-Path $env:UserProfile '.nuget\packages\'
+ $userProfile = if (IsWindowsPlatform) { $env:UserProfile } else { $env:HOME }
+ $env:NUGET_PACKAGES = [IO.Path]::Combine($userProfile, '.nuget', 'packages') + [IO.Path]::DirectorySeparatorChar
} else {
- $env:NUGET_PACKAGES = Join-Path $RepoRoot '.packages\'
- $env:RESTORENOHTTPCACHE = $true
+ $env:NUGET_PACKAGES = [IO.Path]::Combine($RepoRoot, '.packages') + [IO.Path]::DirectorySeparatorChar
}
}
@@ -655,7 +651,13 @@ function GetNuGetPackageCachePath() {
# Returns a full path to an Arcade SDK task project file.
function GetSdkTaskProject([string]$taskName) {
- return Join-Path (Split-Path (InitializeToolset) -Parent) "SdkTasks\$taskName.proj"
+ $toolsetDir = Split-Path (InitializeToolset) -Parent
+ $proj = Join-Path $toolsetDir "$taskName.proj"
+ if (Test-Path $proj) {
+ return $proj
+ }
+
+ throw "Unable to find $taskName.proj in toolset at: $toolsetDir"
}
function InitializeNativeTools() {
@@ -689,16 +691,19 @@ function InitializeToolset() {
return $global:_InitializeToolset
}
- $nugetCache = GetNuGetPackageCachePath
-
$toolsetVersion = Read-ArcadeSdkVersion
- $toolsetLocationFile = Join-Path $ToolsetDir "$toolsetVersion.txt"
+ $toolsetToolsDir = Join-Path $ToolsetDir $toolsetVersion
- if (Test-Path $toolsetLocationFile) {
- $path = Get-Content $toolsetLocationFile -TotalCount 1
- if (Test-Path $path) {
- return $global:_InitializeToolset = $path
- }
+ # Check if the toolset has already been extracted
+ $toolsetBuildProj = $null
+ $buildProjPath = Join-Path $toolsetToolsDir 'Build.proj'
+
+ if (Test-Path $buildProjPath) {
+ $toolsetBuildProj = $buildProjPath
+ }
+
+ if ($toolsetBuildProj -ne $null) {
+ return $global:_InitializeToolset = $toolsetBuildProj
}
if (-not $restore) {
@@ -706,25 +711,55 @@ function InitializeToolset() {
ExitWithExitCode 1
}
- $buildTool = InitializeBuildTool
+ $downloadArgs = @("package", "download", "Microsoft.DotNet.Arcade.Sdk@$toolsetVersion", "--verbosity", "minimal", "--prerelease", "--output", "$nugetPackageCachePath")
+ $nugetConfig = $env:NUGET_CONFIG
+ if (-not $nugetConfig) {
+ # Search for any variation of nuget.config in the RepoRoot
+ $configFile = Get-ChildItem -Path $RepoRoot -File | Where-Object { $_.Name -ieq "nuget.config" } | Select-Object -First 1
+
+ if ($configFile) {
+ $nugetConfig = $configFile.FullName
+ }
+ }
+
+ if ($nugetConfig) {
+ $downloadArgs += "--configfile"
+ $downloadArgs += $nugetConfig
+ }
- $proj = Join-Path $ToolsetDir 'restore.proj'
- $bl = if ($binaryLog) { '/bl:' + (Join-Path $LogDir 'ToolsetRestore.binlog') } else { '' }
+ # 'dotnet package download' fails outright if any source in the repo's NuGet.config is
+ # unavailable (for example a transport feed that was decommissioned after a release). The
+ # Arcade SDK is always published to the public dotnet-eng feed, so if the config-driven
+ # download fails, retry once against that feed directly (which ignores the other sources)
+ # before giving up, so a single dead source doesn't block the build.
+ $downloadExitCode = DotNet -ignoreFailure @downloadArgs
+ if ($downloadExitCode) {
+ Write-Host "Restoring the Arcade SDK from the configured sources failed; retrying from the public dotnet-eng feed."
+ DotNet @downloadArgs --source "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json"
+ }
+
+ $packageDir = Join-Path $nugetPackageCachePath (Join-Path 'microsoft.dotnet.arcade.sdk' $toolsetVersion)
+ $packageToolsetDir = Join-Path $packageDir 'toolset'
- '' | Set-Content $proj
+ if (!(Test-Path $packageToolsetDir)) {
+ Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Arcade SDK package does not contain a toolset or tools folder: $packageDir"
+ ExitWithExitCode 3
+ }
- MSBuild-Core $proj $bl /t:__WriteToolsetLocation /clp:ErrorsOnly`;NoSummary /p:__ToolsetLocationOutputFile=$toolsetLocationFile
+ New-Item -ItemType Directory -Path $toolsetToolsDir -Force | Out-Null
+ Copy-Item -Path "$packageToolsetDir\*" -Destination $toolsetToolsDir -Recurse -Force
- $path = Get-Content $toolsetLocationFile -Encoding UTF8 -TotalCount 1
- if (!(Test-Path $path)) {
- throw "Invalid toolset path: $path"
+ if (Test-Path $buildProjPath) {
+ $toolsetBuildProj = $buildProjPath
+ } else {
+ throw "Unable to find Build.proj in toolset at: $toolsetToolsDir"
}
- return $global:_InitializeToolset = $path
+ return $global:_InitializeToolset = $toolsetBuildProj
}
function ExitWithExitCode([int] $exitCode) {
- if ($ci -and $prepareMachine) {
+ if ($prepareMachine) {
Stop-Processes
}
exit $exitCode
@@ -754,72 +789,35 @@ function Stop-Processes() {
# Terminates the script if the build fails.
#
function MSBuild() {
- if ($pipelinesLog) {
- $buildTool = InitializeBuildTool
-
- if ($ci -and $buildTool.Tool -eq 'dotnet') {
- $env:NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS = 20
- $env:NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS = 20
- Write-PipelineSetVariable -Name 'NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS' -Value '20'
- Write-PipelineSetVariable -Name 'NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS' -Value '20'
- }
-
- Enable-Nuget-EnhancedRetry
-
- $toolsetBuildProject = InitializeToolset
- $basePath = Split-Path -parent $toolsetBuildProject
- $possiblePaths = @(
- # new scripts need to work with old packages, so we need to look for the old names/versions
- (Join-Path $basePath (Join-Path $buildTool.Framework 'Microsoft.DotNet.ArcadeLogging.dll')),
- (Join-Path $basePath (Join-Path $buildTool.Framework 'Microsoft.DotNet.Arcade.Sdk.dll')),
-
- # This list doesn't need to be updated anymore and can eventually be removed.
- (Join-Path $basePath (Join-Path net9.0 'Microsoft.DotNet.ArcadeLogging.dll')),
- (Join-Path $basePath (Join-Path net9.0 'Microsoft.DotNet.Arcade.Sdk.dll')),
- (Join-Path $basePath (Join-Path net8.0 'Microsoft.DotNet.ArcadeLogging.dll')),
- (Join-Path $basePath (Join-Path net8.0 'Microsoft.DotNet.Arcade.Sdk.dll'))
- )
- $selectedPath = $null
- foreach ($path in $possiblePaths) {
- if (Test-Path $path -PathType Leaf) {
- $selectedPath = $path
- break
- }
- }
- if (-not $selectedPath) {
- Write-PipelineTelemetryError -Category 'Build' -Message 'Unable to find arcade sdk logger assembly.'
- ExitWithExitCode 1
- }
- $args += "/logger:$selectedPath"
- }
-
- MSBuild-Core @args
-}
-
-#
-# Executes msbuild (or 'dotnet msbuild') with arguments passed to the function.
-# The arguments are automatically quoted.
-# Terminates the script if the build fails.
-#
-function MSBuild-Core() {
if ($ci) {
if (!$binaryLog -and !$excludeCIBinarylog) {
Write-PipelineTelemetryError -Category 'Build' -Message 'Binary log must be enabled in CI build, or explicitly opted-out from with the -excludeCIBinarylog switch.'
ExitWithExitCode 1
}
-
- if ($nodeReuse) {
- Write-PipelineTelemetryError -Category 'Build' -Message 'Node reuse must be disabled in CI build.'
- ExitWithExitCode 1
- }
}
- Enable-Nuget-EnhancedRetry
-
$buildTool = InitializeBuildTool
+ if ($pipelinesLog) {
+ $toolsetBuildProject = InitializeToolset
+ $basePath = Split-Path -parent $toolsetBuildProject
+ $selectedPath = Join-Path $basePath (Join-Path 'net' 'Microsoft.DotNet.ArcadeLogging.dll')
+
+ # Only inject the logger when it's present. A last-known-good Arcade used to bootstrap
+ # the build may not ship the logger yet, so its absence must not be a hard error.
+ # Specify the logger type explicitly so loading is deterministic.
+ if (Test-Path $selectedPath) {
+ $args += "/logger:Microsoft.DotNet.ArcadeLogging.PipelinesLogger,$selectedPath"
+ }
+ }
+
$cmdArgs = "$($buildTool.Command) /m /nologo /clp:Summary /v:$verbosity /nr:$nodeReuse /p:ContinuousIntegrationBuild=$ci"
+ # Build with MSBuild's multi-threaded mode.
+ if ($msbuildMultiThreaded) {
+ $cmdArgs += ' -mt'
+ }
+
if ($warnAsError) {
$cmdArgs += ' /warnaserror /p:TreatWarningsAsErrors=true'
}
@@ -827,6 +825,11 @@ function MSBuild-Core() {
$cmdArgs += ' /p:TreatWarningsAsErrors=false'
}
+ if ($warnAsError -and $warnNotAsError) {
+ $escapedWarnNotAsError = $warnNotAsError -replace ';', '%3B'
+ $cmdArgs += " /warnnotaserror:$warnNotAsError /p:AdditionalWarningsNotAsErrors=$escapedWarnNotAsError"
+ }
+
foreach ($arg in $args) {
if ($null -ne $arg -and $arg.Trim() -ne "") {
if ($arg.EndsWith('\')) {
@@ -846,14 +849,9 @@ function MSBuild-Core() {
# The build already logged an error, that's the reason it failed. Producing an error here only adds noise.
Write-Host "Build failed with exit code $exitCode. Check errors above." -ForegroundColor Red
- $buildLog = GetMSBuildBinaryLogCommandLineArgument $args
- if ($null -ne $buildLog) {
- Write-Host "See log: $buildLog" -ForegroundColor DarkGray
- }
-
# When running on Azure Pipelines, override the returned exit code to avoid double logging.
- # Skip this when the build is a child of the VMR orchestrator build.
- if ($ci -and $env:SYSTEM_TEAMPROJECT -ne $null -and !$productBuild -and -not($properties -like "*DotNetBuildRepo=true*")) {
+ # Skip this when the build is a child of the VMR build, or when -disablePipelineSetResult is set so the real exit code propagates.
+ if ($ci -and $env:SYSTEM_TEAMPROJECT -ne $null -and !$fromVMR -and !$disablePipelineSetResult) {
Write-PipelineSetResult -Result "Failed" -Message "msbuild execution failed."
# Exiting with an exit code causes the azure pipelines task to log yet another "noise" error
# The above Write-PipelineSetResult will cause the task to be marked as failure without adding yet another error
@@ -864,21 +862,44 @@ function MSBuild-Core() {
}
}
-function GetMSBuildBinaryLogCommandLineArgument($arguments) {
- foreach ($argument in $arguments) {
- if ($argument -ne $null) {
- $arg = $argument.Trim()
- if ($arg.StartsWith('/bl:', "OrdinalIgnoreCase")) {
- return $arg.Substring('/bl:'.Length)
- }
+#
+# Executes a dotnet command with arguments passed to the function.
+# Terminates the script if the command fails.
+#
+function DotNet([switch]$ignoreFailure) {
+ $dotnetRoot = InitializeDotNetCli -install:$restore
+ $dotnetPath = Join-Path $dotnetRoot (GetExecutableFileName 'dotnet')
- if ($arg.StartsWith('/binaryLogger:', 'OrdinalIgnoreCase')) {
- return $arg.Substring('/binaryLogger:'.Length)
+ $cmdArgs = ""
+ foreach ($arg in $args) {
+ if ($null -ne $arg -and $arg.Trim() -ne "") {
+ if ($arg.EndsWith('\')) {
+ $arg = $arg + "\"
}
+ $cmdArgs += " `"$arg`""
}
}
- return $null
+ $env:ARCADE_BUILD_TOOL_COMMAND = "`"$dotnetPath`" $cmdArgs"
+
+ $exitCode = Exec-Process $dotnetPath $cmdArgs
+
+ if ($exitCode -ne 0) {
+ # When -ignoreFailure is set, return the exit code to the caller so it can implement
+ # its own fallback logic instead of terminating the script.
+ if ($ignoreFailure) {
+ return $exitCode
+ }
+
+ Write-Host "dotnet command failed with exit code $exitCode. Check errors above." -ForegroundColor Red
+
+ if ($ci -and $env:SYSTEM_TEAMPROJECT -ne $null -and !$fromVMR -and !$disablePipelineSetResult) {
+ Write-PipelineSetResult -Result "Failed" -Message "dotnet command execution failed."
+ ExitWithExitCode 0
+ } else {
+ ExitWithExitCode $exitCode
+ }
+ }
}
function GetExecutableFileName($baseName) {
@@ -921,6 +942,12 @@ Create-Directory $ToolsetDir
Create-Directory $TempDir
Create-Directory $LogDir
+# Direct MSBuild crash diagnostics (MSB4166 failure.txt files) to a known location
+# under artifacts/log so they are captured as build artifacts in CI.
+if (-not $env:MSBUILDDEBUGPATH) {
+ $env:MSBUILDDEBUGPATH = Join-Path $LogDir 'MsbuildDebugLogs'
+}
+
Write-PipelineSetVariable -Name 'Artifacts' -Value $ArtifactsDir
Write-PipelineSetVariable -Name 'Artifacts.Toolset' -Value $ToolsetDir
Write-PipelineSetVariable -Name 'Artifacts.Log' -Value $LogDir
@@ -942,19 +969,5 @@ if (!$disableConfigureToolsetImport) {
}
}
-#
-# If $ci flag is set, turn on (and log that we did) special environment variables for improved Nuget client retry logic.
-#
-function Enable-Nuget-EnhancedRetry() {
- if ($ci) {
- Write-Host "Setting NUGET enhanced retry environment variables"
- $env:NUGET_ENABLE_ENHANCED_HTTP_RETRY = 'true'
- $env:NUGET_ENHANCED_MAX_NETWORK_TRY_COUNT = 6
- $env:NUGET_ENHANCED_NETWORK_RETRY_DELAY_MILLISECONDS = 1000
- $env:NUGET_RETRY_HTTP_429 = 'true'
- Write-PipelineSetVariable -Name 'NUGET_ENABLE_ENHANCED_HTTP_RETRY' -Value 'true'
- Write-PipelineSetVariable -Name 'NUGET_ENHANCED_MAX_NETWORK_TRY_COUNT' -Value '6'
- Write-PipelineSetVariable -Name 'NUGET_ENHANCED_NETWORK_RETRY_DELAY_MILLISECONDS' -Value '1000'
- Write-PipelineSetVariable -Name 'NUGET_RETRY_HTTP_429' -Value 'true'
- }
-}
+# Initialize the nuget package cache vars
+$nugetPackageCachePath = InitializeNuGetPackageCachePath
diff --git a/eng/common/tools.sh b/eng/common/tools.sh
index 4a5fa9947..4d14b100b 100644
--- a/eng/common/tools.sh
+++ b/eng/common/tools.sh
@@ -1,13 +1,26 @@
#!/usr/bin/env bash
+# Normalizes the value of a boolean build argument. Accepts 1/0 in addition to true/false so that
+# the same value works with the PowerShell scripts, whose [bool] parameters only bind 1/0.
+function NormalizeBoolArg {
+ case "${1:-}" in
+ 1) echo true ;;
+ 0) echo false ;;
+ *) echo "${1:-}" ;;
+ esac
+}
+
# Initialize variables if they aren't already defined.
# CI mode - set to true on CI server for PR validation build or official build.
ci=${ci:-false}
+# Build mode
+source_build=${source_build:-false}
+
# Set to true to use the pipelines logger which will enable Azure logging output.
# https://github.com/Microsoft/azure-pipelines-tasks/blob/master/docs/authoring/commands.md
-# This flag is meant as a temporary opt-opt for the feature while validate it across
+# This flag is meant as a temporary opt-in for the feature while validating it across
# our consumers. It will be deleted in the future.
if [[ "$ci" == true ]]; then
pipelines_log=${pipelines_log:-true}
@@ -40,15 +53,26 @@ restore=${restore:-true}
verbosity=${verbosity:-'minimal'}
# Set to true to reuse msbuild nodes. Recommended to not reuse on CI.
+node_reuse=$(NormalizeBoolArg "${node_reuse:-}")
if [[ "$ci" == true ]]; then
node_reuse=${node_reuse:-false}
else
node_reuse=${node_reuse:-true}
fi
+# Set to true to build with MSBuild's multi-threaded mode (-mt). Opt-in for now, so off unless it was
+# explicitly requested. It's intended to become the default for local builds once it has proven out.
+msbuild_multi_threaded=$(NormalizeBoolArg "${msbuild_multi_threaded:-}")
+msbuild_multi_threaded=${msbuild_multi_threaded:-false}
+
# Configures warning treatment in msbuild.
+warn_as_error=$(NormalizeBoolArg "${warn_as_error:-}")
warn_as_error=${warn_as_error:-true}
+# Specifies semi-colon delimited list of warning codes that should not be treated as errors.
+# Defaults to NuGet Audit warning codes NU1901-NU1904.
+warn_not_as_error="${warn_not_as_error:-NU1901;NU1902;NU1903;NU1904}"
+
# True to attempt using .NET Core already that meets requirements specified in global.json
# installed on the machine instead of downloading one.
use_installed_dotnet_cli=${use_installed_dotnet_cli:-true}
@@ -58,7 +82,8 @@ use_installed_dotnet_cli=${use_installed_dotnet_cli:-true}
dotnetInstallScriptVersion=${dotnetInstallScriptVersion:-'v1'}
# True to use global NuGet cache instead of restoring packages to repository-local directory.
-if [[ "$ci" == true ]]; then
+# Keep in sync with NuGetPackageroot in Arcade SDK's RepositoryLayout.props.
+if [[ "$ci" == true || "$source_build" == true ]]; then
use_global_nuget_cache=${use_global_nuget_cache:-false}
else
use_global_nuget_cache=${use_global_nuget_cache:-true}
@@ -68,8 +93,10 @@ fi
runtime_source_feed=${runtime_source_feed:-''}
runtime_source_feed_key=${runtime_source_feed_key:-''}
-# True if the build is a product build
-product_build=${product_build:-false}
+# True when the build is running within the VMR.
+from_vmr=${from_vmr:-false}
+
+disable_pipeline_set_result=${disable_pipeline_set_result:-false}
# Resolve any symlinks in the given path.
function ResolvePath {
@@ -111,9 +138,6 @@ function InitializeDotNetCli {
local install=$1
- # Don't resolve runtime, shared framework, or SDK from other locations to ensure build determinism
- export DOTNET_MULTILEVEL_LOOKUP=0
-
# Disable first run since we want to control all package sources
export DOTNET_NOLOGO=1
@@ -144,7 +168,11 @@ function InitializeDotNetCli {
if [[ $global_json_has_runtimes == false && -n "${DOTNET_INSTALL_DIR:-}" && -d "$DOTNET_INSTALL_DIR/sdk/$dotnet_sdk_version" ]]; then
dotnet_root="$DOTNET_INSTALL_DIR"
else
- dotnet_root="${repo_root}.dotnet"
+ if [[ -n "${DOTNET_GLOBAL_INSTALL_DIR:-}" ]]; then
+ dotnet_root="$DOTNET_GLOBAL_INSTALL_DIR"
+ else
+ dotnet_root="${repo_root}.dotnet"
+ fi
export DOTNET_INSTALL_DIR="$dotnet_root"
@@ -162,7 +190,6 @@ function InitializeDotNetCli {
# build steps from using anything other than what we've downloaded.
Write-PipelinePrependPath -path "$dotnet_root"
- Write-PipelineSetVariable -name "DOTNET_MULTILEVEL_LOOKUP" -value "0"
Write-PipelineSetVariable -name "DOTNET_NOLOGO" -value "1"
# return value
@@ -184,6 +211,8 @@ function InstallDotNet {
local version=$2
local runtime=$4
+ # For performance this check is duplicated in src/Microsoft.DotNet.Arcade.Sdk/src/InstallDotNetCore.cs
+ # if you are making changes here, consider if you need to make changes there as well.
local dotnetVersionLabel="'$runtime v$version'"
if [[ -n "${4:-}" ]] && [ "$4" != 'sdk' ]; then
runtimePath="$root"
@@ -295,9 +324,30 @@ function with_retries {
function GetDotNetInstallScript {
local root=$1
local install_script="$root/dotnet-install.sh"
- local install_script_url="https://builds.dotnet.microsoft.com/dotnet/scripts/v1/dotnet-install.sh"
+ local install_script_url="https://builds.dotnet.microsoft.com/dotnet/scripts/$dotnetInstallScriptVersion/dotnet-install.sh"
+ local timestamp_file="$root/.dotnet-install.timestamp"
+ local should_download=false
if [[ ! -a "$install_script" ]]; then
+ should_download=true
+ elif [[ -f "$timestamp_file" ]]; then
+ # Check if the script is older than 30 days using timestamp file
+ local download_time=$(cat "$timestamp_file" 2>/dev/null || echo "0")
+ local current_time=$(date +%s)
+ local age_seconds=$((current_time - download_time))
+
+ # 30 days = 30 * 24 * 60 * 60 = 2592000 seconds
+ if [[ $age_seconds -gt 2592000 ]]; then
+ echo "Existing install script is too old, re-downloading..."
+ should_download=true
+ fi
+ else
+ # No timestamp file exists, assume script is old and re-download
+ echo "No timestamp found for existing install script, re-downloading..."
+ should_download=true
+ fi
+
+ if [[ "$should_download" == true ]]; then
mkdir -p "$root"
echo "Downloading '$install_script_url'"
@@ -324,12 +374,24 @@ function GetDotNetInstallScript {
ExitWithExitCode $exit_code
}
fi
+
+ # Create timestamp file to track download time in seconds from epoch
+ date +%s > "$timestamp_file"
fi
# return value
_GetDotNetInstallScript="$install_script"
}
function InitializeBuildTool {
+ # Allow a caller (e.g. a bootstrap script running out-of-proc) to inject the build tool via
+ # environment variables instead of the in-proc _InitializeBuildTool variable. Only the tool path and
+ # command are consumed by the MSBuild function below, so those are all that's needed.
+ if [[ -n "${_BuildToolPath:-}" ]]; then
+ _InitializeBuildTool="$_BuildToolPath"
+ _InitializeBuildToolCommand="$_BuildToolCommand"
+ return
+ fi
+
if [[ -n "${_InitializeBuildTool:-}" ]]; then
return
fi
@@ -341,19 +403,17 @@ function InitializeBuildTool {
_InitializeBuildToolCommand="msbuild"
}
-# Set RestoreNoHttpCache as a workaround for https://github.com/NuGet/Home/issues/3116
-function GetNuGetPackageCachePath {
+function InitializeNuGetPackageCachePath {
if [[ -z ${NUGET_PACKAGES:-} ]]; then
if [[ "$use_global_nuget_cache" == true ]]; then
export NUGET_PACKAGES="$HOME/.nuget/packages/"
else
export NUGET_PACKAGES="$repo_root/.packages/"
- export RESTORENOHTTPCACHE=true
fi
fi
# return value
- _GetNuGetPackageCachePath=$NUGET_PACKAGES
+ _InitializeNuGetPackageCachePath=$NUGET_PACKAGES
}
function InitializeNativeTools() {
@@ -375,20 +435,21 @@ function InitializeToolset {
return
fi
- GetNuGetPackageCachePath
-
ReadGlobalVersion "Microsoft.DotNet.Arcade.Sdk"
local toolset_version=$_ReadGlobalVersion
- local toolset_location_file="$toolset_dir/$toolset_version.txt"
+ local toolset_tools_dir="$toolset_dir/$toolset_version"
- if [[ -a "$toolset_location_file" ]]; then
- local path=`cat "$toolset_location_file"`
- if [[ -a "$path" ]]; then
- # return value
- _InitializeToolset="$path"
- return
- fi
+ # Check if the toolset has already been extracted
+ local toolset_build_proj=""
+ if [[ -a "$toolset_tools_dir/Build.proj" ]]; then
+ toolset_build_proj="$toolset_tools_dir/Build.proj"
+ fi
+
+ if [[ -n "$toolset_build_proj" ]]; then
+ # return value
+ _InitializeToolset="$toolset_build_proj"
+ return
fi
if [[ "$restore" != true ]]; then
@@ -396,20 +457,46 @@ function InitializeToolset {
ExitWithExitCode 2
fi
- local proj="$toolset_dir/restore.proj"
+ local download_args=("package" "download" "Microsoft.DotNet.Arcade.Sdk@$toolset_version" "--verbosity" "minimal" "--prerelease" "--output" "$_InitializeNuGetPackageCachePath")
+ local nuget_config="${NUGET_CONFIG:-}"
+ if [[ -z "$nuget_config" ]]; then
+ # Search for any variation of nuget.config in the RepoRoot
+ local found_config
+ found_config=$(find "$repo_root" -maxdepth 1 -type f -iname nuget.config | head -n 1)
+
+ if [[ -n "$found_config" ]]; then
+ nuget_config="$found_config"
+ fi
+ fi
+
+ if [[ -n "$nuget_config" ]]; then
+ download_args+=("--configfile" "$nuget_config")
+ fi
- local bl=""
- if [[ "$binary_log" == true ]]; then
- bl="/bl:$log_dir/ToolsetRestore.binlog"
+ # 'dotnet package download' fails outright if any source in the repo's NuGet.config is
+ # unavailable (for example a transport feed that was decommissioned after a release). The
+ # Arcade SDK is always published to the public dotnet-eng feed, so if the config-driven
+ # download fails, retry once against that feed directly (which ignores the other sources)
+ # before giving up, so a single dead source doesn't block the build.
+ if ! DotNet true "${download_args[@]}"; then
+ echo "Restoring the Arcade SDK from the configured sources failed; retrying from the public dotnet-eng feed."
+ DotNet "${download_args[@]}" --source "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json"
fi
- echo '' > "$proj"
- MSBuild-Core "$proj" $bl /t:__WriteToolsetLocation /clp:ErrorsOnly\;NoSummary /p:__ToolsetLocationOutputFile="$toolset_location_file"
+ local package_dir="$_InitializeNuGetPackageCachePath/microsoft.dotnet.arcade.sdk/$toolset_version"
- local toolset_build_proj=`cat "$toolset_location_file"`
+ if [[ ! -d "$package_dir/toolset" ]]; then
+ Write-PipelineTelemetryError -category 'InitializeToolset' "Arcade SDK package does not contain a toolset folder: $package_dir"
+ ExitWithExitCode 3
+ fi
- if [[ ! -a "$toolset_build_proj" ]]; then
- Write-PipelineTelemetryError -category 'Build' "Invalid toolset path: $toolset_build_proj"
+ mkdir -p "$toolset_tools_dir"
+ cp -r "$package_dir/toolset/." "$toolset_tools_dir"
+
+ if [[ -a "$toolset_tools_dir/Build.proj" ]]; then
+ toolset_build_proj="$toolset_tools_dir/Build.proj"
+ else
+ Write-PipelineTelemetryError -category 'Build' "Unable to find Build.proj in toolset at: $toolset_tools_dir"
ExitWithExitCode 3
fi
@@ -418,7 +505,7 @@ function InitializeToolset {
}
function ExitWithExitCode {
- if [[ "$ci" == true && "$prepare_machine" == true ]]; then
+ if [[ "$prepare_machine" == true ]]; then
StopProcesses
fi
exit $1
@@ -427,66 +514,70 @@ function ExitWithExitCode {
function StopProcesses {
echo "Killing running build processes..."
pkill -9 "dotnet" || true
- pkill -9 "vbcscompiler" || true
+ pkill -9 -i -x VBCSCompiler || true
+ pkill -9 -i -x MSBuild || true
return 0
}
-function MSBuild {
- local args=( "$@" )
- if [[ "$pipelines_log" == true ]]; then
- InitializeBuildTool
- InitializeToolset
+function DotNet {
+ # When the first argument is 'true' or 'false' it controls the exit behavior on failure:
+ # 'true' returns the dotnet exit code to the caller (so it can implement its own fallback),
+ # while the default terminates the script. Any other first argument is treated as a dotnet argument.
+ local ignore_failure=false
+ if [[ "$1" == 'true' || "$1" == 'false' ]]; then
+ ignore_failure="$1"
+ shift
+ fi
- if [[ "$ci" == true ]]; then
- export NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS=20
- export NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS=20
- Write-PipelineSetVariable -name "NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS" -value "20"
- Write-PipelineSetVariable -name "NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS" -value "20"
- fi
+ InitializeDotNetCli $restore
- local toolset_dir="${_InitializeToolset%/*}"
- # new scripts need to work with old packages, so we need to look for the old names/versions
- local selectedPath=
- local possiblePaths=()
- possiblePaths+=( "$toolset_dir/net/Microsoft.DotNet.ArcadeLogging.dll" )
- possiblePaths+=( "$toolset_dir/net/Microsoft.DotNet.Arcade.Sdk.dll" )
-
- # This list doesn't need to be updated anymore and can eventually be removed.
- possiblePaths+=( "$toolset_dir/net9.0/Microsoft.DotNet.ArcadeLogging.dll" )
- possiblePaths+=( "$toolset_dir/net9.0/Microsoft.DotNet.Arcade.Sdk.dll" )
- possiblePaths+=( "$toolset_dir/net8.0/Microsoft.DotNet.ArcadeLogging.dll" )
- possiblePaths+=( "$toolset_dir/net8.0/Microsoft.DotNet.Arcade.Sdk.dll" )
- for path in "${possiblePaths[@]}"; do
- if [[ -f $path ]]; then
- selectedPath=$path
- break
- fi
- done
- if [[ -z "$selectedPath" ]]; then
- Write-PipelineTelemetryError -category 'Build' "Unable to find arcade sdk logger assembly."
- ExitWithExitCode 1
+ local dotnet_path="$_InitializeDotNetCli/dotnet"
+
+ export ARCADE_BUILD_TOOL_COMMAND="$dotnet_path $@"
+
+ "$dotnet_path" "$@" || {
+ local exit_code=$?
+
+ if [[ "$ignore_failure" == true ]]; then
+ return $exit_code
fi
- args+=( "-logger:$selectedPath" )
- fi
- MSBuild-Core "${args[@]}"
+ echo "dotnet command failed with exit code $exit_code. Check errors above."
+
+ if [[ "$ci" == true && -n ${SYSTEM_TEAMPROJECT:-} && "$from_vmr" != true && "$disable_pipeline_set_result" != true ]]; then
+ Write-PipelineSetResult -result "Failed" -message "dotnet command execution failed."
+ ExitWithExitCode 0
+ else
+ ExitWithExitCode $exit_code
+ fi
+ }
}
-function MSBuild-Core {
+function MSBuild {
if [[ "$ci" == true ]]; then
if [[ "$binary_log" != true && "$exclude_ci_binary_log" != true ]]; then
- Write-PipelineTelemetryError -category 'Build' "Binary log must be enabled in CI build, or explicitly opted-out from with the -noBinaryLog switch."
- ExitWithExitCode 1
- fi
-
- if [[ "$node_reuse" == true ]]; then
- Write-PipelineTelemetryError -category 'Build' "Node reuse must be disabled in CI build."
+ Write-PipelineTelemetryError -category 'Build' "Binary log must be enabled in CI build, or explicitly opted-out from with the --excludeCIBinarylog switch."
ExitWithExitCode 1
fi
fi
InitializeBuildTool
+ local logger_switch=()
+ if [[ "$pipelines_log" == true ]]; then
+ InitializeToolset
+
+ local toolset_dir="${_InitializeToolset%/*}"
+ local selectedPath="$toolset_dir/net/Microsoft.DotNet.ArcadeLogging.dll"
+
+ # Only inject the logger when it's present. A last-known-good Arcade used to bootstrap
+ # the build may not ship the logger yet, so its absence must not be a hard error.
+ # Specify the logger type explicitly so loading is deterministic.
+ if [[ -f "$selectedPath" ]]; then
+ logger_switch=("-logger:Microsoft.DotNet.ArcadeLogging.PipelinesLogger,$selectedPath")
+ fi
+ fi
+
local warnaserror_switch=""
if [[ $warn_as_error == true ]]; then
warnaserror_switch="/warnaserror"
@@ -502,8 +593,8 @@ function MSBuild-Core {
echo "Build failed with exit code $exit_code. Check errors above."
# When running on Azure Pipelines, override the returned exit code to avoid double logging.
- # Skip this when the build is a child of the VMR orchestrator build.
- if [[ "$ci" == true && -n ${SYSTEM_TEAMPROJECT:-} && "$product_build" != true && "$properties" != *"DotNetBuildRepo=true"* ]]; then
+ # Skip this when the build is a child of the VMR build, or when -disablePipelineSetResult is set so the real exit code propagates.
+ if [[ "$ci" == true && -n ${SYSTEM_TEAMPROJECT:-} && "$from_vmr" != true && "$disable_pipeline_set_result" != true ]]; then
Write-PipelineSetResult -result "Failed" -message "msbuild execution failed."
# Exiting with an exit code causes the azure pipelines task to log yet another "noise" error
# The above Write-PipelineSetResult will cause the task to be marked as failure without adding yet another error
@@ -514,7 +605,18 @@ function MSBuild-Core {
}
}
- RunBuildTool "$_InitializeBuildToolCommand" /m /nologo /clp:Summary /v:$verbosity /nr:$node_reuse $warnaserror_switch /p:TreatWarningsAsErrors=$warn_as_error /p:ContinuousIntegrationBuild=$ci "$@"
+ # Build with MSBuild's multi-threaded mode.
+ local mt_switch=""
+ if [[ "$msbuild_multi_threaded" == true ]]; then
+ mt_switch="-mt"
+ fi
+
+ local warnnotaserror_switch=""
+ if [[ -n "$warn_not_as_error" && "$warn_as_error" == true ]]; then
+ warnnotaserror_switch="/warnnotaserror:$warn_not_as_error /p:AdditionalWarningsNotAsErrors=${warn_not_as_error//;/%3B}"
+ fi
+
+ RunBuildTool "$_InitializeBuildToolCommand" /m /nologo /clp:Summary /v:$verbosity /nr:$node_reuse $warnaserror_switch $mt_switch $warnnotaserror_switch ${logger_switch[@]+"${logger_switch[@]}"} /p:TreatWarningsAsErrors=$warn_as_error /p:ContinuousIntegrationBuild=$ci "$@"
}
function GetDarc {
@@ -526,12 +628,22 @@ function GetDarc {
fi
"$eng_root/common/darc-init.sh" --toolpath "$darc_path" $version
+ darc_tool="$darc_path/darc"
}
# Returns a full path to an Arcade SDK task project file.
function GetSdkTaskProject {
- taskName=$1
- echo "$(dirname $_InitializeToolset)/SdkTasks/$taskName.proj"
+ local taskName=$1
+ local toolsetDir
+ toolsetDir="$(dirname "$_InitializeToolset")"
+ local proj="$toolsetDir/$taskName.proj"
+ if [[ -a "$proj" ]]; then
+ echo "$proj"
+ return
+ fi
+
+ Write-PipelineTelemetryError -category 'Build' "Unable to find $taskName.proj in toolset at: $toolsetDir"
+ ExitWithExitCode 3
}
ResolvePath "${BASH_SOURCE[0]}"
@@ -569,6 +681,12 @@ mkdir -p "$toolset_dir"
mkdir -p "$temp_dir"
mkdir -p "$log_dir"
+# Direct MSBuild crash diagnostics (MSB4166 failure.txt files) to a known location
+# under artifacts/log so they are captured as build artifacts in CI.
+if [[ -z "${MSBUILDDEBUGPATH:-}" ]]; then
+ export MSBUILDDEBUGPATH="$log_dir/MsbuildDebugLogs"
+fi
+
Write-PipelineSetVariable -name "Artifacts" -value "$artifacts_dir"
Write-PipelineSetVariable -name "Artifacts.Toolset" -value "$toolset_dir"
Write-PipelineSetVariable -name "Artifacts.Log" -value "$log_dir"
@@ -589,3 +707,6 @@ fi
if [[ -n "${useInstalledDotNetCli:-}" ]]; then
use_installed_dotnet_cli="$useInstalledDotNetCli"
fi
+
+# Initialize the nuget package cache vars
+InitializeNuGetPackageCachePath
diff --git a/eng/common/vmr-sync.ps1 b/eng/common/vmr-sync.ps1
new file mode 100644
index 000000000..b37992d91
--- /dev/null
+++ b/eng/common/vmr-sync.ps1
@@ -0,0 +1,164 @@
+<#
+.SYNOPSIS
+
+This script is used for synchronizing the current repository into a local VMR.
+It pulls the current repository's code into the specified VMR directory for local testing or
+Source-Build validation.
+
+.DESCRIPTION
+
+The tooling used for synchronization will clone the VMR repository into a temporary folder if
+it does not already exist. These clones can be reused in future synchronizations, so it is
+recommended to dedicate a folder for this to speed up re-runs.
+
+.EXAMPLE
+ Synchronize current repository into a local VMR:
+ ./vmr-sync.ps1 -vmrDir "$HOME/repos/dotnet" -tmpDir "$HOME/repos/tmp"
+
+.PARAMETER tmpDir
+Required. Path to the temporary folder where repositories will be cloned
+
+.PARAMETER vmrBranch
+Optional. Branch of the 'dotnet/dotnet' repo to synchronize. The VMR will be checked out to this branch
+
+.PARAMETER azdevPat
+Optional. Azure DevOps PAT to use for cloning private repositories.
+
+.PARAMETER vmrDir
+Optional. Path to the dotnet/dotnet repository. When null, gets cloned to the temporary folder
+
+.PARAMETER debugOutput
+Optional. Enables debug logging in the darc vmr command.
+
+.PARAMETER ci
+Optional. Denotes that the script is running in a CI environment.
+#>
+param (
+ [Parameter(Mandatory=$true, HelpMessage="Path to the temporary folder where repositories will be cloned")]
+ [string][Alias('t', 'tmp')]$tmpDir,
+ [string][Alias('b', 'branch')]$vmrBranch,
+ [string]$remote,
+ [string]$azdevPat,
+ [string][Alias('v', 'vmr')]$vmrDir,
+ [switch]$ci,
+ [switch]$debugOutput
+)
+
+function Fail {
+ Write-Host "> $($args[0])" -ForegroundColor 'Red'
+}
+
+function Highlight {
+ Write-Host "> $($args[0])" -ForegroundColor 'Cyan'
+}
+
+$verbosity = 'verbose'
+if ($debugOutput) {
+ $verbosity = 'debug'
+}
+# Validation
+
+if (-not $tmpDir) {
+ Fail "Missing -tmpDir argument. Please specify the path to the temporary folder where the repositories will be cloned"
+ exit 1
+}
+
+# Sanitize the input
+
+if (-not $vmrDir) {
+ $vmrDir = Join-Path $tmpDir 'dotnet'
+}
+
+if (-not (Test-Path -Path $tmpDir -PathType Container)) {
+ New-Item -ItemType Directory -Path $tmpDir | Out-Null
+}
+
+# Prepare the VMR
+
+if (-not (Test-Path -Path $vmrDir -PathType Container)) {
+ Highlight "Cloning 'dotnet/dotnet' into $vmrDir.."
+ git clone https://github.com/dotnet/dotnet $vmrDir
+
+ if ($vmrBranch) {
+ git -C $vmrDir switch -c $vmrBranch
+ }
+}
+else {
+ if ((git -C $vmrDir diff --quiet) -eq $false) {
+ Fail "There are changes in the working tree of $vmrDir. Please commit or stash your changes"
+ exit 1
+ }
+
+ if ($vmrBranch) {
+ Highlight "Preparing $vmrDir"
+ git -C $vmrDir checkout $vmrBranch
+ git -C $vmrDir pull
+ }
+}
+
+Set-StrictMode -Version Latest
+
+# Prepare darc
+
+Highlight 'Installing .NET, preparing the tooling..'
+. .\eng\common\tools.ps1
+$dotnetRoot = InitializeDotNetCli -install:$true
+$env:DOTNET_ROOT = $dotnetRoot
+$darc = Get-Darc
+
+Highlight "Starting the synchronization of VMR.."
+
+# Synchronize the VMR
+$versionDetailsPath = Resolve-Path (Join-Path $PSScriptRoot '..\Version.Details.xml') | Select-Object -ExpandProperty Path
+[xml]$versionDetails = Get-Content -Path $versionDetailsPath
+$repoName = $versionDetails.SelectSingleNode('//Source').Mapping
+if (-not $repoName) {
+ Fail "Failed to resolve repo mapping from $versionDetailsPath"
+ exit 1
+}
+
+$darcArgs = (
+ "vmr", "forwardflow",
+ "--tmp", $tmpDir,
+ "--$verbosity",
+ $vmrDir
+)
+
+if ($ci) {
+ $darcArgs += ("--ci")
+}
+
+if ($azdevPat) {
+ $darcArgs += ("--azdev-pat", $azdevPat)
+}
+
+& "$darc" $darcArgs
+
+if ($LASTEXITCODE -eq 0) {
+ Highlight "Synchronization succeeded"
+}
+else {
+ Highlight "Failed to flow code into the local VMR. Falling back to resetting the VMR to match repo contents..."
+ git -C $vmrDir reset --hard
+
+ $resetArgs = (
+ "vmr", "reset",
+ "${repoName}:HEAD",
+ "--vmr", $vmrDir,
+ "--tmp", $tmpDir,
+ "--additional-remotes", "${repoName}:${repoRoot}"
+ )
+
+ & "$darc" $resetArgs
+
+ if ($LASTEXITCODE -eq 0) {
+ Highlight "Successfully reset the VMR using 'darc vmr reset'"
+ }
+ else {
+ Fail "Synchronization of repo to VMR failed!"
+ Fail "'$vmrDir' is left in its last state (re-run of this script will reset it)."
+ Fail "Please inspect the logs which contain path to the failing patch file (use -debugOutput to get all the details)."
+ Fail "Once you make changes to the conflicting VMR patch, commit it locally and re-run this script."
+ exit 1
+ }
+}
diff --git a/eng/common/vmr-sync.sh b/eng/common/vmr-sync.sh
new file mode 100644
index 000000000..198caec59
--- /dev/null
+++ b/eng/common/vmr-sync.sh
@@ -0,0 +1,227 @@
+#!/bin/bash
+
+### This script is used for synchronizing the current repository into a local VMR.
+### It pulls the current repository's code into the specified VMR directory for local testing or
+### Source-Build validation.
+###
+### The tooling used for synchronization will clone the VMR repository into a temporary folder if
+### it does not already exist. These clones can be reused in future synchronizations, so it is
+### recommended to dedicate a folder for this to speed up re-runs.
+###
+### USAGE:
+### Synchronize current repository into a local VMR:
+### ./vmr-sync.sh --tmp "$HOME/repos/tmp" "$HOME/repos/dotnet"
+###
+### Options:
+### -t, --tmp, --tmp-dir PATH
+### Required. Path to the temporary folder where repositories will be cloned
+###
+### -b, --branch, --vmr-branch BRANCH_NAME
+### Optional. Branch of the 'dotnet/dotnet' repo to synchronize. The VMR will be checked out to this branch
+###
+### --debug
+### Optional. Turns on the most verbose logging for the VMR tooling
+###
+### --remote name:URI
+### Optional. Additional remote to use during the synchronization
+### This can be used to synchronize to a commit from a fork of the repository
+### Example: 'runtime:https://github.com/yourfork/runtime'
+###
+### --azdev-pat
+### Optional. Azure DevOps PAT to use for cloning private repositories.
+###
+### -v, --vmr, --vmr-dir PATH
+### Optional. Path to the dotnet/dotnet repository. When null, gets cloned to the temporary folder
+
+source="${BASH_SOURCE[0]}"
+
+# resolve $source until the file is no longer a symlink
+while [[ -h "$source" ]]; do
+ scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
+ source="$(readlink "$source")"
+ # if $source was a relative symlink, we need to resolve it relative to the path where the
+ # symlink file was located
+ [[ $source != /* ]] && source="$scriptroot/$source"
+done
+scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
+
+function print_help () {
+ sed -n '/^### /,/^$/p' "$source" | cut -b 5-
+}
+
+COLOR_RED=$(tput setaf 1 2>/dev/null || true)
+COLOR_CYAN=$(tput setaf 6 2>/dev/null || true)
+COLOR_CLEAR=$(tput sgr0 2>/dev/null || true)
+COLOR_RESET=uniquesearchablestring
+FAILURE_PREFIX='> '
+
+function fail () {
+ echo "${COLOR_RED}$FAILURE_PREFIX${1//${COLOR_RESET}/${COLOR_RED}}${COLOR_CLEAR}" >&2
+}
+
+function highlight () {
+ echo "${COLOR_CYAN}$FAILURE_PREFIX${1//${COLOR_RESET}/${COLOR_CYAN}}${COLOR_CLEAR}"
+}
+
+tmp_dir=''
+vmr_dir=''
+vmr_branch=''
+additional_remotes=''
+verbosity=verbose
+azdev_pat=''
+ci=false
+
+while [[ $# -gt 0 ]]; do
+ opt="$(echo "$1" | tr "[:upper:]" "[:lower:]")"
+ case "$opt" in
+ -t|--tmp|--tmp-dir)
+ tmp_dir=$2
+ shift
+ ;;
+ -v|--vmr|--vmr-dir)
+ vmr_dir=$2
+ shift
+ ;;
+ -b|--branch|--vmr-branch)
+ vmr_branch=$2
+ shift
+ ;;
+ --remote)
+ additional_remotes="$additional_remotes $2"
+ shift
+ ;;
+ --azdev-pat)
+ azdev_pat=$2
+ shift
+ ;;
+ --ci)
+ ci=true
+ ;;
+ -d|--debug)
+ verbosity=debug
+ ;;
+ -h|--help)
+ print_help
+ exit 0
+ ;;
+ *)
+ fail "Invalid argument: $1"
+ print_help
+ exit 1
+ ;;
+ esac
+
+ shift
+done
+
+# Validation
+
+if [[ -z "$tmp_dir" ]]; then
+ fail "Missing --tmp-dir argument. Please specify the path to the temporary folder where the repositories will be cloned"
+ exit 1
+fi
+
+# Sanitize the input
+
+if [[ -z "$vmr_dir" ]]; then
+ vmr_dir="$tmp_dir/dotnet"
+fi
+
+if [[ ! -d "$tmp_dir" ]]; then
+ mkdir -p "$tmp_dir"
+fi
+
+if [[ "$verbosity" == "debug" ]]; then
+ set -x
+fi
+
+# Prepare the VMR
+
+if [[ ! -d "$vmr_dir" ]]; then
+ highlight "Cloning 'dotnet/dotnet' into $vmr_dir.."
+ git clone https://github.com/dotnet/dotnet "$vmr_dir"
+
+ if [[ -n "$vmr_branch" ]]; then
+ git -C "$vmr_dir" switch -c "$vmr_branch"
+ fi
+else
+ if ! git -C "$vmr_dir" diff --quiet; then
+ fail "There are changes in the working tree of $vmr_dir. Please commit or stash your changes"
+ exit 1
+ fi
+
+ if [[ -n "$vmr_branch" ]]; then
+ highlight "Preparing $vmr_dir"
+ git -C "$vmr_dir" checkout "$vmr_branch"
+ git -C "$vmr_dir" pull
+ fi
+fi
+
+set -e
+
+# Prepare darc
+
+highlight 'Installing .NET, preparing the tooling..'
+source "./eng/common/tools.sh"
+InitializeDotNetCli true
+GetDarc
+dotnetDir=$( cd ./.dotnet/; pwd -P )
+dotnet=$dotnetDir/dotnet
+
+highlight "Starting the synchronization of VMR.."
+set +e
+
+if [[ -n "$additional_remotes" ]]; then
+ additional_remotes="--additional-remotes $additional_remotes"
+fi
+
+if [[ -n "$azdev_pat" ]]; then
+ azdev_pat="--azdev-pat $azdev_pat"
+fi
+
+ci_arg=''
+if [[ "$ci" == "true" ]]; then
+ ci_arg="--ci"
+fi
+
+# Synchronize the VMR
+
+version_details_path=$(cd "$scriptroot/.."; pwd -P)/Version.Details.xml
+repo_name=$(grep -m 1 '