Refactor bootstrapped build infrastructure

.github/workflows/copilot-setup-steps.yml

@@ -18,7 +18,7 @@ jobs:
       - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
 
       - name: Restore MSBuild
-        run: ./eng/common/build.sh --restore
+        run: ./build.sh --restore
 
       - name: Put dotnet on the path
         run: echo "PATH=$PWD/.dotnet:$PATH" >> $GITHUB_ENV

.vsts-dotnet-ci.yml

@@ -44,13 +44,11 @@ jobs:
       env:
         Token: $(dn-bot-dnceng-artifact-feeds-rw)
   - task: BatchScript@1
-    displayName: cibuild_bootstrapped_msbuild.cmd
+    displayName: Build (with -mt in stage2)
     inputs:
-      filename: 'eng/cibuild_bootstrapped_msbuild.cmd'
-      arguments: '-onlyDocChanged $(onlyDocChanged) -stage2Properties /mt'
+      filename: 'build.cmd'
+      arguments: '-ci -test -stage2Arguments /mt'
     condition: eq(variables.onlyDocChanged, 0)
-    env:
-      ForceUseXCopyMSBuild: 1
 # Task to collect code coverage on Windows. Disabled by default due to being unstable and sometimes it stucks forever
 #   - task: PowerShell@2
 #     displayName: Process coverage reports
@@ -157,10 +155,10 @@ jobs:
       env:
         Token: $(dn-bot-dnceng-artifact-feeds-rw)
   - task: BatchScript@1
-    displayName: cibuild_bootstrapped_msbuild.cmd
+    displayName: Build (with -mt in stage2)
     inputs:
-      filename: 'eng/cibuild_bootstrapped_msbuild.cmd'
-      arguments: '-msbuildEngine dotnet -onlyDocChanged $(onlyDocChanged) -stage2Properties /mt'
+      filename: 'build.cmd'
+      arguments: '-msbuildEngine dotnet -ci -test -stage2Arguments /mt'
     condition: eq(variables.onlyDocChanged, 0)
     env:
       MSBUILDUSESERVER: "1"
@@ -238,8 +236,9 @@ jobs:
         Token: $(dn-bot-dnceng-artifact-feeds-rw)
   - bash: sudo apt-get update
   - bash: sudo apt-get install -y libxml2
-  - bash: . 'eng/cibuild_bootstrapped_msbuild.sh' --onlyDocChanged $(onlyDocChanged) --skipTests --stage2Properties '/mt /p:BuildAnalyzer=true'
-    displayName: 'Build with bootstrapped MSBuild and -mt mode'
+  # Intentionally not running tests
+  - bash: . 'build.sh' --ci --stage2Arguments '/mt /p:BuildAnalyzer=true'
+    displayName: Build (with -mt in stage2)
     condition: eq(variables.onlyDocChanged, 0)
     env:
       MSBUILDUSESERVER: "1"
@@ -275,10 +274,10 @@ jobs:
       env:
         Token: $(dn-bot-dnceng-artifact-feeds-rw)
   - task: BatchScript@1
-    displayName: cibuild.cmd
+    displayName: Build
     inputs:
-      filename: 'eng/cibuild.cmd'
-      arguments: '-configuration Release -test'
+      filename: 'build.cmd'
+      arguments: '-pack -sign -publish -ci -configuration Release -test'
     condition: eq(variables.onlyDocChanged, 0)
 # Task to collect code coverage on Windows. Disabled by default due to being unstable and sometimes it stucks forever
 #   - task: PowerShell@2
@@ -354,8 +353,8 @@ jobs:
         Token: $(dn-bot-dnceng-artifact-feeds-rw)
   - bash: sudo apt-get update
   - bash: sudo apt-get install -y libxml2
-  - bash: . 'eng/cibuild_bootstrapped_msbuild.sh' --onlyDocChanged $(onlyDocChanged) --stage2Properties '/mt'
-    displayName: CI Build
+  - bash: . 'build.sh' --ci --test --stage2Arguments '/mt'
+    displayName: Build (with -mt in stage2)
     condition: eq(variables.onlyDocChanged, 0)
     env:
         MSBUILDUSESERVER: "1"
@@ -443,8 +442,8 @@ jobs:
         arguments: $(Build.SourcesDirectory)/NuGet.config $Token
       env:
         Token: $(dn-bot-dnceng-artifact-feeds-rw)
-  - bash: . 'eng/cibuild_bootstrapped_msbuild.sh' --onlyDocChanged $(onlyDocChanged) --stage2Properties '/mt'
-    displayName: CI Build
+  - bash: . 'build.sh' --ci --test --stage2Arguments '/mt'
+    displayName: Build (with -mt in stage2)
     condition: eq(variables.onlyDocChanged, 0)
     env:
         MSBUILDUSESERVER: "1"

.vsts-dotnet.yml

@@ -2,12 +2,6 @@ trigger:
 - main
 - vs*
 
-# If defined here, these values are not overrideable
-# Once they exist, we should define these as "runtime parameters"
-# https://github.com/Microsoft/azure-pipelines-yaml/pull/129
-# variables:
-#   SkipApplyOptimizationData: false
-
 parameters:
 - name: OptProfDropName
   displayName: Optional OptProfDrop Override
@@ -59,10 +53,10 @@ variables:
       value: ${{ parameters.OptProfDropName }}
     - name: SourceBranch
       value: ''
-  # Override SkipApplyOptimizationData to true when disabling OptProf data collection
+  # Set SkipApplyOptimizationDataArg when disabling OptProf data collection
   - ${{ if eq(parameters.enableOptProf, false) }}:
-    - name: SkipApplyOptimizationData
-      value: true
+    - name: SkipApplyOptimizationDataArg
+      value: '/p:EnableNgenOptimization=false'
   - name: EnableReleaseOneLocBuild
     value: false
   - ${{ if eq(variables['Build.SourceBranch'], 'refs/heads/main') }}:

azure-pipelines/.vsts-dotnet-build-jobs.yml

@@ -117,10 +117,11 @@ jobs:
     inputs:
       dropName: $(VisualStudio.DropName)
 
-  - script: eng/CIBuild.cmd
+  - script: build.cmd
+              -pack -sign -publish -ci
               -configuration $(BuildConfiguration)
               -officialBuildId $(Build.BuildNumber)
-              -officialSkipApplyOptimizationData $(SkipApplyOptimizationData)
+              $(SkipApplyOptimizationDataArg)
               /p:RepositoryName=$(Build.Repository.Name)
               /p:VisualStudioIbcSourceBranchName=$(SourceBranch)
               /p:VisualStudioDropAccessToken=$(System.AccessToken)
@@ -139,6 +140,14 @@ jobs:
   # Required by Microsoft policy
   - template: /eng/common/templates-official/steps/generate-sbom.yml@self
 
+  # Compute the .vsman manifest list (generated by the Build step above) consumed by MicroBuildBuildVSBootstrapper below.
+  - pwsh: |
+      $insertionDir = "$(Build.SourcesDirectory)\artifacts\VSSetup\$(BuildConfiguration)\Insertion"
+      $manifestList = (Get-ChildItem "$insertionDir\*.vsman" | Select-Object -ExpandProperty FullName) -join ','
+      Write-Host "##vso[task.setvariable variable=VisualStudio.SetupManifestList;]$manifestList"
+    displayName: 'OptProf - Set manifest list'
+    condition: and(succeeded(), ${{ parameters.enableOptProf }})
+
   # Build VS bootstrapper
   # Generates $(Build.StagingDirectory)\MicroBuild\Output\BootstrapperInfo.json
   - task: MicroBuildBuildVSBootstrapper@3

azure-pipelines/.vsts-dotnet-exp-perf.yml

@@ -2,12 +2,6 @@ trigger:
 - exp/*
 - perf/*
 
-# If defined here, these values are not overrideable
-# Once they exist, we should define these as "runtime parameters"
-# https://github.com/Microsoft/azure-pipelines-yaml/pull/129
-# variables:
-#   SkipApplyOptimizationData: false
-
 parameters:
 - name: OptProfDropName
   displayName: Optional OptProfDrop Override

azure-pipelines/cts-steps/apply-steps.yml

@@ -186,15 +186,15 @@ steps:
     # + the bundled SDK props). See collect-steps.yml.
     # UsingToolVisualStudioIbcTraining=false: see collect-steps.yml note.
     if ($IsWindows) {
-      & "$(Build.SourcesDirectory)/eng/common/build.ps1" `
-        -restore -build `
+      & "$(Build.SourcesDirectory)/build.cmd" `
+        -ci `
         -configuration $(BuildConfiguration) `
         -projects "$(Build.SourcesDirectory)/MSBuild.VSTest.slnx" `
         -verbosity minimal `
         /p:UsingToolVisualStudioIbcTraining=false
     } else {
-      bash "$(Build.SourcesDirectory)/eng/common/build.sh" `
-        --restore --build `
+      bash "$(Build.SourcesDirectory)/build.sh" `
+        --ci `
         --configuration $(BuildConfiguration) `
         --projects "$(Build.SourcesDirectory)/MSBuild.VSTest.slnx" `
         --verbosity minimal `

azure-pipelines/cts-steps/collect-steps.yml

@@ -109,15 +109,15 @@ steps:
     # Insertion outputs, so skip Arcade's AfterSigning IBC-inputs step
     # which otherwise errors on missing artifacts/VSSetup/Debug/Insertion.
     if ($IsWindows) {
-      & "$(Build.SourcesDirectory)/eng/common/build.ps1" `
-        -restore -build `
+      & "$(Build.SourcesDirectory)/build.cmd" `
+        -ci `
         -configuration $(BuildConfiguration) `
         -projects "$(Build.SourcesDirectory)/MSBuild.VSTest.slnx" `
         -verbosity minimal `
         /p:UsingToolVisualStudioIbcTraining=false
     } else {
-      bash "$(Build.SourcesDirectory)/eng/common/build.sh" `
-        --restore --build `
+      bash "$(Build.SourcesDirectory)/build.sh" `
+        --ci `
         --configuration $(BuildConfiguration) `
         --projects "$(Build.SourcesDirectory)/MSBuild.VSTest.slnx" `
         --verbosity minimal `

azure-pipelines/quarantine.yml

@@ -66,13 +66,11 @@ jobs:
   steps:
   - template: azure-pipelines/check-documentation-only-change.yml
   - task: BatchScript@1
-    displayName: cibuild_bootstrapped_msbuild.cmd (quarantined tests only)
+    displayName: Build (quarantined tests only)
     inputs:
-      filename: 'eng/cibuild_bootstrapped_msbuild.cmd'
-      arguments: "-onlyDocChanged 0 -stage2Properties '/mt /p:RunQuarantinedTests=true'"
+      filename: 'build.cmd'
+      arguments: "-ci -test -stage2Arguments '/p:RunQuarantinedTests=true'"
     condition: or(ne(variables['Build.Reason'], 'PullRequest'), eq(variables.onlyDocChanged, 0))
-    env:
-      ForceUseXCopyMSBuild: 1
   - task: PublishTestResults@2
     displayName: Publish .NET Framework Test Results
     inputs:
@@ -120,8 +118,8 @@ jobs:
   - template: azure-pipelines/check-documentation-only-change.yml
   - bash: sudo apt-get update
   - bash: sudo apt-get install -y libxml2
-  - bash: . 'eng/cibuild_bootstrapped_msbuild.sh' --onlyDocChanged 0 --stage2Properties '/mt /p:RunQuarantinedTests=true'
-    displayName: CI Build (quarantined tests only)
+  - bash: . 'build.sh' --ci --test --stage2Arguments '/p:RunQuarantinedTests=true'
+    displayName: Build (quarantined tests only)
     condition: or(ne(variables['Build.Reason'], 'PullRequest'), eq(variables.onlyDocChanged, 0))
     env:
       MSBUILDUSESERVER: "1"
@@ -159,8 +157,8 @@ jobs:
   timeoutInMinutes: 120
   steps:
   - template: azure-pipelines/check-documentation-only-change.yml
-  - bash: . 'eng/cibuild_bootstrapped_msbuild.sh' --onlyDocChanged 0 --stage2Properties '/mt /p:RunQuarantinedTests=true'
-    displayName: CI Build (quarantined tests only)
+  - bash: . 'build.sh' --ci --test --stage2Arguments '/p:RunQuarantinedTests=true'
+    displayName: Build (quarantined tests only)
     condition: or(ne(variables['Build.Reason'], 'PullRequest'), eq(variables.onlyDocChanged, 0))
     env:
       MSBUILDUSESERVER: "1"

build.cmd

@@ -1,10 +1,16 @@
 @echo off
 setlocal
+
+set _args=%*
+if "%~1"=="-?" set _args=-help
+if "%~1"=="/?" set _args=-help
+
 if defined MSBUILDDEBUGONSTART_HARD goto build
 if not defined MSBUILDDEBUGONSTART goto build
 if %MSBUILDDEBUGONSTART% == 0 goto build
 set MSBUILDDEBUGONSTART=
 echo To debug the build, define a value for MSBUILDDEBUGONSTART_HARD.
+
 :build
-powershell -NoLogo -NoProfile -ExecutionPolicy ByPass -Command "& """%~dp0eng\common\build.ps1""" -build -restore %*"
-exit /b %ErrorLevel%
+powershell -ExecutionPolicy ByPass -NoProfile -Command "& '%~dp0eng\build.ps1'" -restore -build %_args%
+exit /b %ERRORLEVEL%

build.sh

@@ -1,11 +1,16 @@
 #!/usr/bin/env bash
 
-SOURCE="${BASH_SOURCE[0]}"
-while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
-  ScriptRoot="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
-  SOURCE="$(readlink "$SOURCE")"
-  [[ $SOURCE != /* ]] && SOURCE="$ScriptRoot/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
+source="${BASH_SOURCE[0]}"
+
+# resolve $SOURCE until the file is no longer a symlink
+while [[ -h $source ]]; do
+  scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
+  source="$(readlink "$source")"
+
+  # if $source was a relative symlink, we need to resolve it relative to the path where the
+  # symlink file was located
+  [[ $source != /* ]] && source="$scriptroot/$source"
 done
-ScriptRoot="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
 
-. "$ScriptRoot/eng/common/build.sh" --build --restore "$@"
+scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
+"$scriptroot/eng/build.sh" --restore --build "$@"

documentation/wiki/Building-Testing-and-Debugging-on-Full-Framework-MSBuild.md

@@ -29,7 +29,7 @@ Some tests are creating symlinks to test associated functionality - in order for
 * Enable [Development Mode](https://learn.microsoft.com/en-us/windows/apps/get-started/enable-your-device-for-development) on your machine.
 * Or run those tests elevated
 
-To mimic our CI job use `eng\CIBuild.cmd`. Be aware that this command may delete your local NuGet cache.
+To mimic our CI job run `build.cmd -pack -sign -publish -ci`.
 
 The CI does two builds. In the second build, it uses the binaries from the first build to build the repository again.