diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 7b291576f323..38b55d3c9b9a 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -2,21 +2,21 @@ - + https://github.com/dotnet/arcade - 9467b1074927ee3fe98d539d3b72f5686c861958 + c444d7967c5b79d16f2150370504368591883268 - + https://github.com/dotnet/arcade - 9467b1074927ee3fe98d539d3b72f5686c861958 + c444d7967c5b79d16f2150370504368591883268 - + https://github.com/dotnet/arcade - 9467b1074927ee3fe98d539d3b72f5686c861958 + c444d7967c5b79d16f2150370504368591883268 - + https://github.com/dotnet/arcade - 9467b1074927ee3fe98d539d3b72f5686c861958 + c444d7967c5b79d16f2150370504368591883268 https://github.com/dotnet/corefx diff --git a/eng/Versions.props b/eng/Versions.props index b8bfe7625302..f0df8235d7fe 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -15,9 +15,9 @@ false - 2.2.0-beta.21161.1 - 1.0.0-beta.21161.1 - 2.0.0-beta.21161.1 + 2.2.0-beta.21376.2 + 1.0.0-beta.21376.2 + 2.0.0-beta.21376.2 2.5.1-beta.19278.1 3.3.0-beta2-19367-02 diff --git a/eng/common/generate-locproject.ps1 b/eng/common/generate-locproject.ps1 new file mode 100644 index 000000000000..2907f916d6f3 --- /dev/null +++ b/eng/common/generate-locproject.ps1 @@ -0,0 +1,110 @@ +Param( + [Parameter(Mandatory=$true)][string] $SourcesDirectory, # Directory where source files live; if using a Localize directory it should live in here + [string] $LanguageSet = 'VS_Main_Languages', # Language set to be used in the LocProject.json + [switch] $UseCheckedInLocProjectJson, # When set, generates a LocProject.json and compares it to one that already exists in the repo; otherwise just generates one + [switch] $CreateNeutralXlfs # Creates neutral xlf files. Only set to false when running locally +) + +# Generates LocProject.json files for the OneLocBuild task. OneLocBuildTask is described here: +# https://ceapex.visualstudio.com/CEINTL/_wiki/wikis/CEINTL.wiki/107/Localization-with-OneLocBuild-Task + +Set-StrictMode -Version 2.0 +$ErrorActionPreference = "Stop" +. $PSScriptRoot\tools.ps1 + +Import-Module -Name (Join-Path $PSScriptRoot 'native\CommonLibrary.psm1') + +$exclusionsFilePath = "$SourcesDirectory\Localize\LocExclusions.json" +$exclusions = @{ Exclusions = @() } +if (Test-Path -Path $exclusionsFilePath) +{ + $exclusions = Get-Content "$exclusionsFilePath" | ConvertFrom-Json +} + +Push-Location "$SourcesDirectory" # push location for Resolve-Path -Relative to work + +# Template files +$jsonFiles = @() +$jsonFiles += Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "\.template\.config\\localize\\en\..+\.json" } # .NET templating pattern +$jsonFiles += Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "en\\strings\.json" } # current winforms pattern + +$xlfFiles = @() + +$allXlfFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory\*\*.xlf" +$langXlfFiles = @() +if ($allXlfFiles) { + $null = $allXlfFiles[0].FullName -Match "\.([\w-]+)\.xlf" # matches '[langcode].xlf' + $firstLangCode = $Matches.1 + $langXlfFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory\*\*.$firstLangCode.xlf" +} +$langXlfFiles | ForEach-Object { + $null = $_.Name -Match "(.+)\.[\w-]+\.xlf" # matches '[filename].[langcode].xlf' + + $destinationFile = "$($_.Directory.FullName)\$($Matches.1).xlf" + $xlfFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru +} + +$locFiles = $jsonFiles + $xlfFiles + +$locJson = @{ + Projects = @( + @{ + LanguageSet = $LanguageSet + LocItems = @( + $locFiles | ForEach-Object { + $outputPath = "$(($_.DirectoryName | Resolve-Path -Relative) + "\")" + $continue = $true + foreach ($exclusion in $exclusions.Exclusions) { + if ($outputPath.Contains($exclusion)) + { + $continue = $false + } + } + $sourceFile = ($_.FullName | Resolve-Path -Relative) + if (!$CreateNeutralXlfs -and $_.Extension -eq '.xlf') { + Remove-Item -Path $sourceFile + } + if ($continue) + { + if ($_.Directory.Name -eq 'en' -and $_.Extension -eq '.json') { + return @{ + SourceFile = $sourceFile + CopyOption = "LangIDOnPath" + OutputPath = "$($_.Directory.Parent.FullName | Resolve-Path -Relative)\" + } + } + else { + return @{ + SourceFile = $sourceFile + CopyOption = "LangIDOnName" + OutputPath = $outputPath + } + } + } + } + ) + } + ) +} + +$json = ConvertTo-Json $locJson -Depth 5 +Write-Host "LocProject.json generated:`n`n$json`n`n" +Pop-Location + +if (!$UseCheckedInLocProjectJson) { + New-Item "$SourcesDirectory\Localize\LocProject.json" -Force # Need this to make sure the Localize directory is created + Set-Content "$SourcesDirectory\Localize\LocProject.json" $json +} +else { + New-Item "$SourcesDirectory\Localize\LocProject-generated.json" -Force # Need this to make sure the Localize directory is created + Set-Content "$SourcesDirectory\Localize\LocProject-generated.json" $json + + if ((Get-FileHash "$SourcesDirectory\Localize\LocProject-generated.json").Hash -ne (Get-FileHash "$SourcesDirectory\Localize\LocProject.json").Hash) { + Write-PipelineTelemetryError -Category "OneLocBuild" -Message "Existing LocProject.json differs from generated LocProject.json. Download LocProject-generated.json and compare them." + + exit 1 + } + else { + Write-Host "Generated LocProject.json and current LocProject.json are identical." + } +} \ No newline at end of file diff --git a/eng/common/sdl/execute-all-sdl-tools.ps1 b/eng/common/sdl/execute-all-sdl-tools.ps1 index 01799d63ff36..06589e3d7bb3 100644 --- a/eng/common/sdl/execute-all-sdl-tools.ps1 +++ b/eng/common/sdl/execute-all-sdl-tools.ps1 @@ -29,7 +29,7 @@ Param( $ErrorActionPreference = "Stop" Set-StrictMode -Version 2.0 -$LASTEXITCODE = 0 +$global:LASTEXITCODE = 0 #Replace repo names to the format of org/repo if (!($Repository.contains('/'))) { diff --git a/eng/common/sdl/init-sdl.ps1 b/eng/common/sdl/init-sdl.ps1 index d34d25287546..6572a10c7713 100644 --- a/eng/common/sdl/init-sdl.ps1 +++ b/eng/common/sdl/init-sdl.ps1 @@ -9,7 +9,7 @@ Param( $ErrorActionPreference = "Stop" Set-StrictMode -Version 2.0 -$LASTEXITCODE = 0 +$global:LASTEXITCODE = 0 # `tools.ps1` checks $ci to perform some actions. Since the SDL # scripts don't necessarily execute in the same agent that run the diff --git a/eng/common/sdl/push-gdn.ps1 b/eng/common/sdl/push-gdn.ps1 index 79c707d6d8a0..43f00eae3fb5 100644 --- a/eng/common/sdl/push-gdn.ps1 +++ b/eng/common/sdl/push-gdn.ps1 @@ -8,7 +8,7 @@ Param( $ErrorActionPreference = "Stop" Set-StrictMode -Version 2.0 -$LASTEXITCODE = 0 +$global:LASTEXITCODE = 0 # We create the temp directory where we'll store the sdl-config repository $sdlDir = Join-Path $env:TEMP "sdl" @@ -36,16 +36,26 @@ git add . if ($LASTEXITCODE -ne 0) { Write-Error "Git add failed with exit code $LASTEXITCODE." } -Write-Host "git -c user.email=`"dn-bot@microsoft.com`" -c user.name=`"Dotnet Bot`" commit -m `"$PushReason for $Repository/$BranchName`"" -git -c user.email="dn-bot@microsoft.com" -c user.name="Dotnet Bot" commit -m "$PushReason for $Repository/$BranchName" +# check if there are any staged changes (0 = no changes, 1 = changes) +# if we don't do this and there's nothing to commit `git commit` will return +# exit code 1 and we will fail +Write-Host "git diff --cached --exit-code" +git diff --cached --exit-code +Write-Host "git diff exit code: $LASTEXITCODE" if ($LASTEXITCODE -ne 0) { - Write-Error "Git commit failed with exit code $LASTEXITCODE." + Write-Host "git -c user.email=`"dn-bot@microsoft.com`" -c user.name=`"Dotnet Bot`" commit -m `"$PushReason for $Repository/$BranchName`"" + git -c user.email="dn-bot@microsoft.com" -c user.name="Dotnet Bot" commit -m "$PushReason for $Repository/$BranchName" + if ($LASTEXITCODE -ne 0) { + Write-Error "Git commit failed with exit code $LASTEXITCODE." + } + Write-Host "git push" + git push + if ($LASTEXITCODE -ne 0) { + Write-Error "Git push failed with exit code $LASTEXITCODE." + } } -Write-Host "git push" -git push -if ($LASTEXITCODE -ne 0) { - Write-Error "Git push failed with exit code $LASTEXITCODE." +else { + Write-Host "Skipping commit and push because there is nothing to commit" } - # Return to the original directory Pop-Location \ No newline at end of file diff --git a/eng/common/sdl/run-sdl.ps1 b/eng/common/sdl/run-sdl.ps1 index 9bc25314ae21..cd3423a4ad64 100644 --- a/eng/common/sdl/run-sdl.ps1 +++ b/eng/common/sdl/run-sdl.ps1 @@ -12,7 +12,7 @@ Param( $ErrorActionPreference = "Stop" Set-StrictMode -Version 2.0 -$LASTEXITCODE = 0 +$global:LASTEXITCODE = 0 # We store config files in the r directory of .gdn Write-Host $ToolsList @@ -21,7 +21,7 @@ $ValidPath = Test-Path $GuardianCliLocation if ($ValidPath -eq $False) { - Write-Host "Invalid Guardian CLI Location." + Write-Error "Invalid Guardian CLI Location." exit 1 } @@ -35,7 +35,7 @@ foreach ($tool in $ToolsList) { Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" TargetDirectory < $TargetDirectory `" `" OutputType < pre `" $(If ($CrScanAdditionalRunConfigParams) {$CrScanAdditionalRunConfigParams})" & $GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args " TargetDirectory < $TargetDirectory " "OutputType < pre" $(If ($CrScanAdditionalRunConfigParams) {$CrScanAdditionalRunConfigParams}) if ($LASTEXITCODE -ne 0) { - Write-Host "Guardian configure for $tool failed with exit code $LASTEXITCODE." + Write-Error "Guardian configure for $tool failed with exit code $LASTEXITCODE." exit $LASTEXITCODE } } @@ -43,7 +43,7 @@ foreach ($tool in $ToolsList) { Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" Target < $TargetDirectory `" $(If ($PoliCheckAdditionalRunConfigParams) {$PoliCheckAdditionalRunConfigParams})" & $GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args " Target < $TargetDirectory " $(If ($PoliCheckAdditionalRunConfigParams) {$PoliCheckAdditionalRunConfigParams}) if ($LASTEXITCODE -ne 0) { - Write-Host "Guardian configure for $tool failed with exit code $LASTEXITCODE." + Write-Error "Guardian configure for $tool failed with exit code $LASTEXITCODE." exit $LASTEXITCODE } } @@ -54,6 +54,6 @@ foreach ($tool in $ToolsList) { Write-Host "$GuardianCliLocation run --working-directory $WorkingDirectory --baseline mainbaseline --update-baseline $UpdateBaseline --logger-level $GuardianLoggerLevel $configParam" & $GuardianCliLocation run --working-directory $WorkingDirectory --tool $tool --baseline mainbaseline --update-baseline $UpdateBaseline --logger-level $GuardianLoggerLevel $configParam if ($LASTEXITCODE -ne 0) { - Write-Host "Guardian run for $ToolsList using $configParam failed with exit code $LASTEXITCODE." + Write-Error "Guardian run for $ToolsList using $configParam failed with exit code $LASTEXITCODE." exit $LASTEXITCODE } diff --git a/eng/common/templates/job/onelocbuild.yml b/eng/common/templates/job/onelocbuild.yml new file mode 100644 index 000000000000..b27d6faf3035 --- /dev/null +++ b/eng/common/templates/job/onelocbuild.yml @@ -0,0 +1,83 @@ +parameters: + # Optional: dependencies of the job + dependsOn: '' + + # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool + pool: + vmImage: vs2017-win2016 + + CeapexPat: $(dn-bot-ceapex-package-r) # PAT for the loc AzDO instance https://dev.azure.com/ceapex + GithubPat: $(BotAccount-dotnet-bot-repo-PAT) + + SourcesDirectory: $(Build.SourcesDirectory) + CreatePr: true + AutoCompletePr: false + UseCheckedInLocProjectJson: false + LanguageSet: VS_Main_Languages + LclSource: lclFilesInRepo + LclPackageId: '' + RepoType: gitHub + condition: '' + +jobs: +- job: OneLocBuild + + dependsOn: ${{ parameters.dependsOn }} + + displayName: OneLocBuild + + pool: ${{ parameters.pool }} + + variables: + - group: OneLocBuildVariables # Contains the CeapexPat and GithubPat + - name: _GenerateLocProjectArguments + value: -SourcesDirectory ${{ parameters.SourcesDirectory }} + -LanguageSet "${{ parameters.LanguageSet }}" + -CreateNeutralXlfs + - ${{ if eq(parameters.UseCheckedInLocProjectJson, 'true') }}: + - name: _GenerateLocProjectArguments + value: ${{ variables._GenerateLocProjectArguments }} -UseCheckedInLocProjectJson + + + steps: + - task: Powershell@2 + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/generate-locproject.ps1 + arguments: $(_GenerateLocProjectArguments) + displayName: Generate LocProject.json + condition: ${{ parameters.condition }} + + - task: OneLocBuild@2 + displayName: OneLocBuild + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + inputs: + locProj: Localize/LocProject.json + outDir: $(Build.ArtifactStagingDirectory) + lclSource: ${{ parameters.LclSource }} + lclPackageId: ${{ parameters.LclPackageId }} + isCreatePrSelected: ${{ parameters.CreatePr }} + ${{ if eq(parameters.CreatePr, true) }}: + isAutoCompletePrSelected: ${{ parameters.AutoCompletePr }} + packageSourceAuth: patAuth + patVariable: ${{ parameters.CeapexPat }} + ${{ if eq(parameters.RepoType, 'gitHub') }}: + repoType: ${{ parameters.RepoType }} + gitHubPatVariable: "${{ parameters.GithubPat }}" + condition: ${{ parameters.condition }} + + - task: PublishBuildArtifacts@1 + displayName: Publish Localization Files + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)/loc' + PublishLocation: Container + ArtifactName: Loc + condition: ${{ parameters.condition }} + + - task: PublishBuildArtifacts@1 + displayName: Publish LocProject.json + inputs: + PathtoPublish: '$(Build.SourcesDirectory)/Localize/' + PublishLocation: Container + ArtifactName: Loc + condition: ${{ parameters.condition }} \ No newline at end of file diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml index 9104ab1870be..7784d2dc9f3b 100644 --- a/eng/common/templates/post-build/post-build.yml +++ b/eng/common/templates/post-build/post-build.yml @@ -391,6 +391,30 @@ stages: shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json' +- template: \eng\common\templates\post-build\channels\generic-public-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'VS16_11_Publishing' + channelName: 'VS 16.11' + channelId: 1926 + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-transport/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json' + +- template: \eng\common\templates\post-build\channels\generic-public-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'VS17_0_Publishing' + channelName: 'VS 17.0' + channelId: 1853 + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-transport/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json' + - template: \eng\common\templates\post-build\channels\generic-public-channel.yml parameters: artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index 2205c9e81173..e5a3374921a6 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -211,12 +211,12 @@ function InstallDotNetSdk([string] $dotnetRoot, [string] $version, [string] $arc InstallDotNet -dotnetRoot $dotnetRoot -version $version -architecture $architecture -skipNonVersionedFiles $false -runtimeSourceFeed $runtimeSourceFeed -runtimeSourceFeedKey $runtimeSourceFeedKey } -function InstallDotNet([string] $dotnetRoot, - [string] $version, - [string] $architecture = "", - [string] $runtime = "", - [bool] $skipNonVersionedFiles = $false, - [string] $runtimeSourceFeed = "", +function InstallDotNet([string] $dotnetRoot, + [string] $version, + [string] $architecture = "", + [string] $runtime = "", + [bool] $skipNonVersionedFiles = $false, + [string] $runtimeSourceFeed = "", [string] $runtimeSourceFeedKey = "") { $installScript = GetDotNetInstallScript $dotnetRoot @@ -323,7 +323,16 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = } $msbuildVersionDir = if ([int]$vsMajorVersion -lt 16) { "$vsMajorVersion.0" } else { "Current" } - return $global:_MSBuildExe = Join-Path $vsInstallDir "MSBuild\$msbuildVersionDir\Bin\msbuild.exe" + + $local:BinFolder = Join-Path $vsInstallDir "MSBuild\$msbuildVersionDir\Bin" + $local:Prefer64bit = if (Get-Member -InputObject $vsRequirements -Name 'Prefer64bit') { $vsRequirements.Prefer64bit } else { $false } + if ($local:Prefer64bit -and (Test-Path(Join-Path $local:BinFolder "amd64"))) { + $global:_MSBuildExe = Join-Path $local:BinFolder "amd64\msbuild.exe" + } else { + $global:_MSBuildExe = Join-Path $local:BinFolder "msbuild.exe" + } + + return $global:_MSBuildExe } function InitializeVisualStudioEnvironmentVariables([string] $vsInstallDir, [string] $vsMajorVersion) { @@ -523,7 +532,7 @@ function InitializeNativeTools() { } } -function InitializeToolset([string] $runtimeSourceFeed, [string] $runtimeSourceFeedKey) +function InitializeToolset([string] $runtimeSourceFeed, [string] $runtimeSourceFeedKey) { if (Test-Path variable:global:_ToolsetBuildProj) { return $global:_ToolsetBuildProj diff --git a/global.json b/global.json index 9b0d0b847e6e..65d989f87a28 100644 --- a/global.json +++ b/global.json @@ -7,8 +7,8 @@ "python": "2.7.15" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.21161.1", - "Microsoft.DotNet.Helix.Sdk": "2.0.0-beta.21161.1", + "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.21376.2", + "Microsoft.DotNet.Helix.Sdk": "2.0.0-beta.21376.2", "Microsoft.Build.NoTargets": "1.0.53", "Microsoft.Build.Traversal": "2.0.2" }