diff --git a/azure-pipelines.yml b/azure-pipelines.yml index a4a23ce22d..6d1b803abd 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -71,6 +71,9 @@ variables: value: true - name: VSO_DEDUP_REDIRECT_TIMEOUT_IN_SEC value: 5 + # Pinned prototype containing dotnet/msbuild#14824, #14825, and #14826. + - name: _PatchedMSBuildCommit + value: 16e184d749654ec43bf92c67ae4a9bcca2225401 # Microsoft.Testing.Platform report/dump arguments for the Windows Debug unit-test command that uses # --test-modules. That mode bypasses test/Directory.Build.targets, so the command must restore these options # explicitly. The full Release runs evaluate the test projects and receive the same options from that targets file. @@ -146,6 +149,8 @@ stages: - template: /eng/pipelines/steps/install-windows-prereqs.yml + - template: /eng/pipelines/steps/build-patched-msbuild.yml + # Seed from every trusted main merge. PR merge refs are read-only consumers, so the cache must be updated as # soon as main changes to avoid forcing the entire 204-node graph to miss. - pwsh: | @@ -168,6 +173,7 @@ stages: "-NoProfile", "-File", "./eng/common/msbuild.ps1", "-prepareMachine", + "-msbuildMultiThreaded:`$false", "-warnAsError:`$false", "./TestFx.slnx", "/restore", @@ -208,7 +214,10 @@ stages: Write-Host "##vso[task.logissue type=warning]Seeding MSBuildCache failed with exit code $exitCode; continuing with the regular Arcade build." exit 0 displayName: Seed $(_BuildConfig) project cache + condition: eq(variables['PatchedMSBuildAvailable'], 'true') env: + _BuildToolPath: $(PatchedMSBuildPath) + MSBUILD_EXE_PATH: $(PatchedMSBuildPath) SYSTEM_ACCESSTOKEN: $(System.AccessToken) # MSBuildCache reads NUGET_PACKAGES to build its path normalizer, and every restored package file is a # fingerprinted input. The seeding and the consuming builds must therefore agree on this folder or every @@ -451,23 +460,24 @@ stages: - template: /eng/pipelines/steps/install-windows-prereqs.yml + - template: /eng/pipelines/steps/build-patched-msbuild.yml + parameters: + condition: and(succeeded(), or(and(eq(variables['Build.Reason'], 'PullRequest'), ne(variables['System.PullRequest.IsFork'], 'True')), and(ne(variables['Build.Reason'], 'PullRequest'), eq(variables['Build.SourceBranch'], 'refs/heads/main')))) + # MSBuildCache requires a direct static graph build; Arcade's outer Build.proj discovers its projects too late - # for the project-cache plugin. Keep the regular Arcade build below as the correctness fallback during rollout. + # for the project-cache plugin. This experimental consumer uses MSBuild's in-process multithreaded mode but + # deliberately does not enable Detours file-access reporting: access attribution assumes one active project + # per process and is only safe on the process-based trusted seed above. Keep the regular Arcade build below as + # the correctness fallback while the required MSBuild fixes from dotnet/msbuild#14824 and #14826 are unreleased. # - # This step reports one of three outcomes, and the fallback below keys off it: + # This step reports one of two outcomes, and the fallback below keys off it: # succeeded - the cache supplied the outputs, the Arcade 'Build' step is skipped. - # succeeded with warning - the cache build broke for a reason the fallback can plausibly fix, so the - # Arcade 'Build' step runs. The step deliberately stays green: a step marked - # 'SucceededWithIssues' makes the job PartiallySucceeded, which Azure Repos - # build-validation policies treat as a failure and would block the merge on a - # run that ultimately builds fine (same reason as _MacOSNonBlockingTrailer). - # failed - the sources genuinely do not build, see the classification below. + # succeeded with warning - the experimental MT cache build failed, so the authoritative process-based + # Arcade 'Build' step runs. MT changes task routing, so even a source-shaped error + # can be specific to the experiment and must not suppress the fallback. - pwsh: | - # This step no longer sets continueOnError, so its result is meaningful and an *unexpected* - # terminating error (Tee-Object, log reading, the classification below) must not fail the job: - # that is an infrastructure failure, which belongs on the fallback path exactly like a crash of - # the build itself. `exit` is flow control rather than an error, so the deliberate `exit 1` for a - # classified source break further down still fails the step as intended. + # This step no longer sets continueOnError, so an unexpected terminating error must not fail the job: + # that is an infrastructure failure, which belongs on the fallback path exactly like a nonzero build. trap { Write-Host "##vso[task.setvariable variable=MSBuildCacheBuildSucceeded]false" Write-Host "##vso[task.logissue type=warning]The MSBuildCache step itself failed unexpectedly ($($_.Exception.Message)); continuing with the regular Arcade build." @@ -485,12 +495,12 @@ stages: "-NoProfile", "-File", "./eng/common/msbuild.ps1", "-prepareMachine", + "-msbuildMultiThreaded:`$true", "-warnAsError:`$false", "./TestFx.slnx", "/restore", "/graph", "/m", - "/reportfileaccesses", "/nr:false", "/t:Build", "/v:minimal", @@ -507,16 +517,10 @@ stages: "/p:MSBuildCacheLogDirectory=$(Agent.TempDirectory)\MSBuildCache" ) - # Tee MSBuild's console output (errors included) so the failure can be classified below without - # re-running anything. stderr is deliberately left unredirected, exactly as before, so PowerShell's - # native-command error handling in this step is unchanged. - $logPath = Join-Path "$(Agent.TempDirectory)" "MSBuildCacheGraphBuild.log" - $previousPSNativeCommandUseErrorActionPreference = $PSNativeCommandUseErrorActionPreference try { - # Unlike Start-Process -Wait, direct native invocation does not wait for detached telemetry descendants. $PSNativeCommandUseErrorActionPreference = $false - & $pwsh @arguments | Tee-Object -FilePath $logPath + & $pwsh @arguments $exitCode = $LASTEXITCODE } finally { @@ -530,68 +534,13 @@ stages: } Write-Host "##vso[task.setvariable variable=MSBuildCacheBuildSucceeded]false" - - # Decide whether the Arcade fallback can still reach a different result than this build did. - # - # A cache hit only materializes outputs that were stored earlier; it never runs the compiler. Every - # "error XXnnnn:" below was therefore emitted by a project that actually executed - exactly as the - # fallback would execute it - so running the whole solution through Arcade again only reproduces the - # same errors several minutes later. Fail here instead, with the errors surfaced on this step. - # - # Three classes of failure stay on the fallback path because the non-cached build can legitimately - # behave differently: anything MSBuildCache itself reports (plugin, cache or file-access problems), - # restore failures (NU*, which are often transient), and the engine/graph failures classified below - - # whether or not they reached MSBuild's error summary. - $log = if (Test-Path $logPath) { @(Get-Content -LiteralPath $logPath) } else { @() } - - $reportedErrorCount = -1 - # Matches MSBuild's English summary. A localized agent leaves the count at -1, which falls through - # to the fallback below - the safe direction, and the same outcome as a build that never reached a - # summary at all. - $errorSummary = @($log | Select-String -Pattern '(\d+)\s+Error\(s\)') | Select-Object -Last 1 - if ($errorSummary) { - $reportedErrorCount = [int]$errorSummary.Matches[0].Groups[1].Value - } - - $errorLines = @($log | Where-Object { $_ -match '(?i):\s*error(\s+[a-z]+\d+)?\s*:\s' } | Select-Object -Unique) - $fallbackMayHelp = '(?i)MSBuildCache|ProjectCache|project cache|cache plugin|CacheClient|file access|\bNU\d{4}\b' - $mayHelpLines = @($errorLines | Where-Object { $_ -match $fallbackMayHelp }) - - # Some failures do reach the error summary but still are not source breaks the fallback would - # reproduce, so they must not fail fast. These are matched against the whole log rather than against - # $errorLines, because their diagnostic detail usually continues on following lines that are not - # themselves formatted as errors: - # - engine crashes and OOM (MSB4166 "exited prematurely", MSB0001/MSB1025 internal errors, - # MSB4017 logger failure). The node died before the projects it owned could report their real - # result, so the Arcade invocation can legitimately succeed where this one did not. - # - MSB4260, a project reference that cannot be resolved with a static graph. That is a - # limitation of the /graph switch this step passes; the Arcade fallback does not build the - # graph statically, so it can resolve the same reference and succeed. - # - MSBuildCache's own duplicate-output diagnostic, worded "Node ... produced output ... which was - # already produced by another node ...". It carries no plugin or cache token of its own, so - # without this marker it reads as an ordinary build error and would defeat the intent that - # everything the plugin reports goes to the fallback. - # - transient file locks (MSB3021/MSB3027, "used by another process"). Copy contention is timing - # dependent, and copying is heavier here because the cache uses copy rather than hardlink - # semantics, so a differently scheduled build can legitimately succeed. - $fallbackCanDiffer = '(?i)\bMSB4166\b|\bMSB0001\b|\bMSB1025\b|\bMSB4017\b|\bMSB4260\b|\bMSB3021\b|\bMSB3027\b|exited prematurely|OutOfMemoryException|already produced by another node|being used by another process' - $fallbackCanDifferLines = @($log | Where-Object { $_ -match $fallbackCanDiffer }) - - if ($reportedErrorCount -gt 0 -and $errorLines.Count -gt 0 -and $mayHelpLines.Count -eq 0 -and $fallbackCanDifferLines.Count -eq 0) { - foreach ($line in @($errorLines | Select-Object -First 20)) { - Write-Host "##vso[task.logissue type=error]$line" - } - - Write-Host "The solution does not build. The Arcade 'Build' step would report the same $reportedErrorCount error(s), so the build is failed here instead of repeating it." - Write-Host "If you believe an error above is an artifact of a cached dependency rather than a source break, the cache diagnostics for this run are published under artifacts/log/$(_BuildConfig)/MSBuildCache." - exit 1 - } - - Write-Host "##vso[task.logissue type=warning]MSBuildCache build failed with exit code $exitCode without a build error the fallback would reproduce; continuing with the regular Arcade build." + Write-Host "##vso[task.logissue type=warning]The experimental multithreaded MSBuildCache build failed with exit code $exitCode; continuing with the authoritative process-based Arcade build." exit 0 displayName: Build solution graph with MSBuildCache - condition: and(succeeded(), or(and(eq(variables['Build.Reason'], 'PullRequest'), ne(variables['System.PullRequest.IsFork'], 'True')), and(ne(variables['Build.Reason'], 'PullRequest'), eq(variables['Build.SourceBranch'], 'refs/heads/main')))) + condition: and(succeeded(), eq(variables['PatchedMSBuildAvailable'], 'true'), or(and(eq(variables['Build.Reason'], 'PullRequest'), ne(variables['System.PullRequest.IsFork'], 'True')), and(ne(variables['Build.Reason'], 'PullRequest'), eq(variables['Build.SourceBranch'], 'refs/heads/main')))) env: + _BuildToolPath: $(PatchedMSBuildPath) + MSBUILD_EXE_PATH: $(PatchedMSBuildPath) SYSTEM_ACCESSTOKEN: $(System.AccessToken) # Must match the seeding stage exactly. MSBuildCache normalizes observed input paths against # NUGET_PACKAGES, and every restored package file is a fingerprinted input, so a different package @@ -735,8 +684,8 @@ stages: New-ItemProperty -Path "HKLM:\SOFTWARE\Microsoft\Windows\Windows Error Reporting\LocalDumps" -Name "DumpFolder" -Value "$(Build.SourcesDirectory)\artifacts\CrashDumps" -PropertyType ExpandString -Force New-ItemProperty -Path "HKLM:\SOFTWARE\Microsoft\Windows\Windows Error Reporting\LocalDumps" -Name "DumpCount" -Value 10 -PropertyType DWord -Force - # Fallback build. Skipped when the MSBuildCache steps above already produced, signed and packed the - # outputs; also skipped when they failed with build errors, because this step would only reproduce them. + # Authoritative fallback build. It is skipped only when the cache graph and any required sign/pack work + # succeed; every cache-path failure runs this process-based build. - script: eng\common\CIBuild.cmd -configuration $(_BuildConfig) -prepareMachine diff --git a/docs/dev-guide.md b/docs/dev-guide.md index d2da4148cd..6e23986773 100644 --- a/docs/dev-guide.md +++ b/docs/dev-guide.md @@ -86,16 +86,27 @@ For more information about all the different options available, supply the argum ### MSBuildCache -The Windows PR pipeline experimentally runs [MSBuildCache](https://github.com/microsoft/MSBuildCache). The cache-aware build uses Arcade's `eng/common/msbuild.ps1` launcher to invoke the solution directly because Arcade's outer `Build.proj` discovers projects dynamically and cannot expose the repository's static project graph to the cache plugin. PR builds consume the immutable Azure Pipeline cache read-only, and fork PRs skip this step because they do not receive the required token scope. +The Windows PR pipeline experimentally runs [MSBuildCache](https://github.com/microsoft/MSBuildCache). The cache-aware build uses Arcade's `eng/common/msbuild.ps1` launcher to invoke the solution directly because Arcade's outer `Build.proj` discovers projects dynamically and cannot expose the repository's static project graph to the cache plugin. Trusted `main` builds populate the immutable Azure Pipeline cache with process-based MSBuild and Detours file-access reporting. PR builds consume that cache read-only with MSBuild's experimental in-process multithreaded mode and without file-access reporting. Fork PRs skip the cache step because they do not receive the required token scope. -The cache steps have three outcomes: +The cache steps have two outcomes: - **The cache build succeeds.** The pipeline runs the remaining Arcade restore, sign, and pack phases without rebuilding, then uses the cached outputs for the test steps. The regular Arcade `Build` step is skipped. -- **The cache build fails with errors the regular build would only reproduce.** A cache hit materializes previously stored outputs and never runs the compiler, so an ordinary build error came from a project that genuinely executed, exactly as the fallback would execute it. Re-running the whole solution through Arcade would report the same errors several minutes later, so the pipeline surfaces them on the cache step and fails the job there instead. Should such a verdict ever be wrong, the cache diagnostics published under `artifacts\log\\MSBuildCache` show what the plugin did for that run. -- **The cache build or the preparation phase fails for any other reason.** The pipeline preserves the cache diagnostics, removes partial outputs, and runs the regular Arcade build as a fallback. This covers anything MSBuildCache itself reports (plugin, cache, or file-access problems, including its duplicate-output diagnostic), `NU*` restore failures, engine crashes and OOM (`MSB4166`, `MSB0001`, `MSB1025`, `MSB4017`), `MSB4260` project references that cannot be resolved with a static graph (a `/graph` limitation the fallback does not have), transient file locks (`MSB3021`, `MSB3027`), an unexpected failure of the wrapper script itself, and any failure that never reached MSBuild's error summary. The cache step stays green in this case: marking it `SucceededWithIssues` would make the job `PartiallySucceeded`, which Azure Repos build-validation policies treat as a failure even when the fallback build then succeeds. +- **The cache build or the preparation phase fails.** The pipeline preserves the cache diagnostics, removes partial outputs, and runs the regular process-based Arcade build as a fallback. Every nonzero experimental cache-build exit takes this path because multithreaded mode changes task routing, including moving unannotated tasks into sidecar task hosts, so even a source-shaped error can be specific to the experiment. The cache step stays green: marking it `SucceededWithIssues` would make the job `PartiallySucceeded`, which Azure Repos build-validation policies treat as a failure even when the authoritative fallback then succeeds. Every merge to `main` that touches product build inputs runs a dedicated, batched seed stage for both Debug and Release. This is required: the cache fingerprints project inputs, so entries become stale whenever shared build inputs such as `global.json`, `eng/Versions.props`, or Arcade change. This stage is the only remote cache publisher; PR, manual, and nightly canary builds consume the cache read-only so they cannot race to publish immutable entries. Debug and Release use separate cache universes because configuration-independent projects can otherwise race while the two configurations publish in parallel. +Cache population and multithreaded cache consumption intentionally use different command lines: + +```powershell +# Trusted main population: process nodes provide unambiguous Detours attribution. +eng\common\msbuild.ps1 -msbuildMultiThreaded:$false -warnAsError:$false TestFx.slnx /restore /graph /m /reportfileaccesses /t:Build /p:Configuration= /p:MSBuildCachePackageEnabled=true /p:MSBuildCacheEnabled=true /p:MSBuildCacheRemoteCacheIsReadOnly=false + +# PR consumption: hits materialize; misses execute under MT but cannot publish to the remote cache. +eng\common\msbuild.ps1 -msbuildMultiThreaded:$true -warnAsError:$false TestFx.slnx /restore /graph /m /t:Build /p:Configuration= /p:MSBuildCachePackageEnabled=true /p:MSBuildCacheEnabled=true /p:MSBuildCacheRemoteCacheIsReadOnly=true +``` + +The cache paths require MSBuild fixes that are not present in the repository's normally pinned toolchain. [dotnet/msbuild#14824](https://github.com/dotnet/msbuild/issues/14824) fixes `FileAccessData` struct deserialization from sidecar task hosts, and [dotnet/msbuild#14826](https://github.com/dotnet/msbuild/issues/14826) propagates and gates task-host reporting when `/reportfileaccesses` is absent. Each eligible Windows cache job therefore builds the x64 .NET Framework bootstrap from immutable `dotnet/msbuild` commit `16e184d749654ec43bf92c67ae4a9bcca2225401` and injects `artifacts\bin\bootstrap\net472\MSBuild\Current\Bin\amd64\MSBuild.exe` into its cache graph step. The source build never receives `System.AccessToken`; if it fails, the cache step is skipped and the regular Arcade build runs with the normal pinned toolchain. Using the same prototype for trusted seeds and read-only consumers also prevents engine differences from obscuring cache hit measurements. [dotnet/msbuild#14825](https://github.com/dotnet/msbuild/issues/14825) tracks the fundamental incompatibility between in-process multithreading and Detours reporting; the prototype replaces the internal scheduler failure with an actionable rejection, so population must remain process-based. + The cache's detached-process exclusions use fully rooted paths (for example, `$(WinDir)\**`). A drive-relative pattern such as `\Windows\**` does not match the absolute file-access paths reported by MSBuild and causes otherwise successful cache builds to fail after compilation. Cache builds also pass `-warnAsError:$false` to Arcade's launcher because MSBuildCache intentionally warns about allowlisted detached telemetry accesses. The fallback Arcade build still treats warnings as errors. Two settings exist purely to keep the cache usable in this pipeline, and both must stay in sync between the seed stage and the PR canary: diff --git a/eng/pipelines/steps/build-patched-msbuild.yml b/eng/pipelines/steps/build-patched-msbuild.yml new file mode 100644 index 0000000000..72e65b01a4 --- /dev/null +++ b/eng/pipelines/steps/build-patched-msbuild.yml @@ -0,0 +1,59 @@ +parameters: +- name: condition + type: string + default: succeeded() + +steps: +# Build from an immutable commit in dotnet/msbuild without mapping System.AccessToken into the source build. +- pwsh: | + $ErrorActionPreference = "Stop" + Write-Host "##vso[task.setvariable variable=PatchedMSBuildAvailable]false" + + $sourceDirectory = "$(Agent.TempDirectory)\patched-msbuild-source" + $binlogDirectory = "$(Build.SourcesDirectory)\artifacts\log\$(_BuildConfig)" + $msbuildPath = Join-Path $sourceDirectory "artifacts\bin\bootstrap\net472\MSBuild\Current\Bin\amd64\MSBuild.exe" + + try { + Remove-Item $sourceDirectory -Recurse -Force -ErrorAction SilentlyContinue + New-Item $sourceDirectory -ItemType Directory -Force | Out-Null + New-Item $binlogDirectory -ItemType Directory -Force | Out-Null + + & git -C $sourceDirectory init --quiet + if ($LASTEXITCODE -ne 0) { throw "git init failed with exit code $LASTEXITCODE." } + + & git -C $sourceDirectory remote add origin https://github.com/dotnet/msbuild.git + if ($LASTEXITCODE -ne 0) { throw "git remote add failed with exit code $LASTEXITCODE." } + + & git -C $sourceDirectory -c core.longpaths=true fetch --depth=1 origin $(_PatchedMSBuildCommit) + if ($LASTEXITCODE -ne 0) { throw "git fetch failed with exit code $LASTEXITCODE." } + + & git -C $sourceDirectory -c core.longpaths=true checkout --detach FETCH_HEAD + if ($LASTEXITCODE -ne 0) { throw "git checkout failed with exit code $LASTEXITCODE." } + + $actualCommit = (& git -C $sourceDirectory rev-parse HEAD).Trim() + if ($LASTEXITCODE -ne 0 -or $actualCommit -ne "$(_PatchedMSBuildCommit)") { + throw "Expected MSBuild commit '$(_PatchedMSBuildCommit)' but checked out '$actualCommit'." + } + + & "$sourceDirectory\build.cmd" ` + -configuration Release ` + -verbosity quiet ` + -binaryLogName "$binlogDirectory\PatchedMSBuild.binlog" ` + /p:LocalizedBuild=false + if ($LASTEXITCODE -ne 0) { throw "The patched MSBuild build failed with exit code $LASTEXITCODE." } + if (-not (Test-Path $msbuildPath -PathType Leaf)) { throw "The patched MSBuild executable was not produced at '$msbuildPath'." } + + & $msbuildPath -version -nologo + if ($LASTEXITCODE -ne 0) { throw "The patched MSBuild executable failed its version check with exit code $LASTEXITCODE." } + } + catch { + Write-Host "##vso[task.logissue type=warning]Patched MSBuild is unavailable: $($_.Exception.Message) The authoritative Arcade build will run." + exit 0 + } + + Write-Host "##vso[task.setvariable variable=PatchedMSBuildPath]$msbuildPath" + Write-Host "##vso[task.setvariable variable=PatchedMSBuildAvailable]true" + displayName: Build patched MSBuild for cache experiment + condition: ${{ parameters.condition }} + env: + DOTNET_CLI_TELEMETRY_OPTOUT: 1