[release/3.1] Update dependencies from dotnet/arcade

eng/Version.Details.xml

@@ -2,21 +2,21 @@
   <ProductDependencies>
   </ProductDependencies>
   <ToolsetDependencies>
-    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="1.0.0-beta.21161.1">
+    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="1.0.0-beta.21376.2">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>9467b1074927ee3fe98d539d3b72f5686c861958</Sha>
+      <Sha>c444d7967c5b79d16f2150370504368591883268</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="2.0.0-beta.21161.1">
+    <Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="2.0.0-beta.21376.2">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>9467b1074927ee3fe98d539d3b72f5686c861958</Sha>
+      <Sha>c444d7967c5b79d16f2150370504368591883268</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.Build.Tasks.Feed" Version="2.2.0-beta.21161.1">
+    <Dependency Name="Microsoft.DotNet.Build.Tasks.Feed" Version="2.2.0-beta.21376.2">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>9467b1074927ee3fe98d539d3b72f5686c861958</Sha>
+      <Sha>c444d7967c5b79d16f2150370504368591883268</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.Build.Tasks.Packaging" Version="1.0.0-beta.21161.1">
+    <Dependency Name="Microsoft.DotNet.Build.Tasks.Packaging" Version="1.0.0-beta.21376.2">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>9467b1074927ee3fe98d539d3b72f5686c861958</Sha>
+      <Sha>c444d7967c5b79d16f2150370504368591883268</Sha>
     </Dependency>
     <Dependency Name="Microsoft.Private.CoreFx.NETCoreApp" Version="4.7.0-servicing.20460.1">
       <Uri>https://github.com/dotnet/corefx</Uri>

eng/Versions.props

@@ -15,9 +15,9 @@
     <UsingToolXliff>false</UsingToolXliff>
     <!-- Package versions -->
     <!-- arcade -->
-    <MicrosoftDotNetBuildTasksFeedVersion>2.2.0-beta.21161.1</MicrosoftDotNetBuildTasksFeedVersion>
-    <MicrosoftDotNetBuildTasksPackagingVersion>1.0.0-beta.21161.1</MicrosoftDotNetBuildTasksPackagingVersion>
-    <MicrosoftDotNetHelixSdkVersion>2.0.0-beta.21161.1</MicrosoftDotNetHelixSdkVersion>
+    <MicrosoftDotNetBuildTasksFeedVersion>2.2.0-beta.21376.2</MicrosoftDotNetBuildTasksFeedVersion>
+    <MicrosoftDotNetBuildTasksPackagingVersion>1.0.0-beta.21376.2</MicrosoftDotNetBuildTasksPackagingVersion>
+    <MicrosoftDotNetHelixSdkVersion>2.0.0-beta.21376.2</MicrosoftDotNetHelixSdkVersion>
     <MicrosoftDotNetXUnitConsoleRunnerVersion>2.5.1-beta.19278.1</MicrosoftDotNetXUnitConsoleRunnerVersion>
     <!-- roslyn -->
     <MicrosoftNetCompilersToolsetVersion>3.3.0-beta2-19367-02</MicrosoftNetCompilersToolsetVersion>

eng/common/generate-locproject.ps1

@@ -0,0 +1,110 @@
+Param(
+    [Parameter(Mandatory=$true)][string] $SourcesDirectory,     # Directory where source files live; if using a Localize directory it should live in here
+    [string] $LanguageSet = 'VS_Main_Languages',                # Language set to be used in the LocProject.json
+    [switch] $UseCheckedInLocProjectJson,                       # When set, generates a LocProject.json and compares it to one that already exists in the repo; otherwise just generates one
+    [switch] $CreateNeutralXlfs                                 # Creates neutral xlf files. Only set to false when running locally
+)
+
+# Generates LocProject.json files for the OneLocBuild task. OneLocBuildTask is described here:
+# https://ceapex.visualstudio.com/CEINTL/_wiki/wikis/CEINTL.wiki/107/Localization-with-OneLocBuild-Task
+
+Set-StrictMode -Version 2.0
+$ErrorActionPreference = "Stop"
+. $PSScriptRoot\tools.ps1
+
+Import-Module -Name (Join-Path $PSScriptRoot 'native\CommonLibrary.psm1')
+
+$exclusionsFilePath = "$SourcesDirectory\Localize\LocExclusions.json"
+$exclusions = @{ Exclusions = @() }
+if (Test-Path -Path $exclusionsFilePath)
+{
+    $exclusions = Get-Content "$exclusionsFilePath" | ConvertFrom-Json
+}
+
+Push-Location "$SourcesDirectory" # push location for Resolve-Path -Relative to work
+
+# Template files
+$jsonFiles = @()
+$jsonFiles += Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "\.template\.config\\localize\\en\..+\.json" } # .NET templating pattern
+$jsonFiles += Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "en\\strings\.json" } # current winforms pattern
+
+$xlfFiles = @()
+
+$allXlfFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory\*\*.xlf"
+$langXlfFiles = @()
+if ($allXlfFiles) {
+    $null = $allXlfFiles[0].FullName -Match "\.([\w-]+)\.xlf" # matches '[langcode].xlf'
+    $firstLangCode = $Matches.1
+    $langXlfFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory\*\*.$firstLangCode.xlf"
+}
+$langXlfFiles | ForEach-Object {
+    $null = $_.Name -Match "(.+)\.[\w-]+\.xlf" # matches '[filename].[langcode].xlf'
+
+    $destinationFile = "$($_.Directory.FullName)\$($Matches.1).xlf"
+    $xlfFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru
+}
+
+$locFiles = $jsonFiles + $xlfFiles
+
+$locJson = @{
+    Projects = @(
+        @{
+            LanguageSet = $LanguageSet
+            LocItems = @(
+                $locFiles | ForEach-Object {
+                    $outputPath = "$(($_.DirectoryName | Resolve-Path -Relative) + "\")" 
+                    $continue = $true
+                    foreach ($exclusion in $exclusions.Exclusions) {
+                        if ($outputPath.Contains($exclusion))
+                        {
+                            $continue = $false
+                        }
+                    }
+                    $sourceFile = ($_.FullName | Resolve-Path -Relative)
+                    if (!$CreateNeutralXlfs -and $_.Extension -eq '.xlf') {
+                        Remove-Item -Path $sourceFile
+                    }
+                    if ($continue)
+                    {
+                        if ($_.Directory.Name -eq 'en' -and $_.Extension -eq '.json') {
+                            return @{
+                                SourceFile = $sourceFile
+                                CopyOption = "LangIDOnPath"
+                                OutputPath = "$($_.Directory.Parent.FullName | Resolve-Path -Relative)\"
+                            }
+                        }
+                        else {
+                            return @{
+                                SourceFile = $sourceFile
+                                CopyOption = "LangIDOnName"
+                                OutputPath = $outputPath
+                            }
+                        }
+                    }
+                }
+            )
+        }
+    )
+}
+
+$json = ConvertTo-Json $locJson -Depth 5
+Write-Host "LocProject.json generated:`n`n$json`n`n"
+Pop-Location
+
+if (!$UseCheckedInLocProjectJson) {
+    New-Item "$SourcesDirectory\Localize\LocProject.json" -Force # Need this to make sure the Localize directory is created
+    Set-Content "$SourcesDirectory\Localize\LocProject.json" $json
+}
+else {
+    New-Item "$SourcesDirectory\Localize\LocProject-generated.json" -Force # Need this to make sure the Localize directory is created
+    Set-Content "$SourcesDirectory\Localize\LocProject-generated.json" $json
+
+    if ((Get-FileHash "$SourcesDirectory\Localize\LocProject-generated.json").Hash -ne (Get-FileHash "$SourcesDirectory\Localize\LocProject.json").Hash) {
+        Write-PipelineTelemetryError -Category "OneLocBuild" -Message "Existing LocProject.json differs from generated LocProject.json. Download LocProject-generated.json and compare them."
+
+        exit 1
+    }
+    else {
+        Write-Host "Generated LocProject.json and current LocProject.json are identical."
+    }
+}

eng/common/sdl/execute-all-sdl-tools.ps1

@@ -29,7 +29,7 @@ Param(
 
 $ErrorActionPreference = "Stop"
 Set-StrictMode -Version 2.0
-$LASTEXITCODE = 0
+$global:LASTEXITCODE = 0
 
 #Replace repo names to the format of org/repo
 if (!($Repository.contains('/'))) {

eng/common/sdl/init-sdl.ps1

@@ -9,7 +9,7 @@ Param(
 
 $ErrorActionPreference = "Stop"
 Set-StrictMode -Version 2.0
-$LASTEXITCODE = 0
+$global:LASTEXITCODE = 0
 
 # `tools.ps1` checks $ci to perform some actions. Since the SDL
 # scripts don't necessarily execute in the same agent that run the

eng/common/sdl/push-gdn.ps1

@@ -8,7 +8,7 @@ Param(
 
 $ErrorActionPreference = "Stop"
 Set-StrictMode -Version 2.0
-$LASTEXITCODE = 0
+$global:LASTEXITCODE = 0
 
 # We create the temp directory where we'll store the sdl-config repository
 $sdlDir = Join-Path $env:TEMP "sdl"
@@ -36,16 +36,26 @@ git add .
 if ($LASTEXITCODE -ne 0) {
   Write-Error "Git add failed with exit code $LASTEXITCODE."
 }
-Write-Host "git -c user.email=`"dn-bot@microsoft.com`" -c user.name=`"Dotnet Bot`" commit -m `"$PushReason for $Repository/$BranchName`""
-git -c user.email="dn-bot@microsoft.com" -c user.name="Dotnet Bot" commit -m "$PushReason for $Repository/$BranchName"
+# check if there are any staged changes (0 = no changes, 1 = changes)
+# if we don't do this and there's nothing to commit `git commit` will return
+# exit code 1 and we will fail
+Write-Host "git diff --cached --exit-code"
+git diff --cached --exit-code
+Write-Host "git diff exit code: $LASTEXITCODE"
 if ($LASTEXITCODE -ne 0) {
-  Write-Error "Git commit failed with exit code $LASTEXITCODE."
+  Write-Host "git -c user.email=`"dn-bot@microsoft.com`" -c user.name=`"Dotnet Bot`" commit -m `"$PushReason for $Repository/$BranchName`""
+  git -c user.email="dn-bot@microsoft.com" -c user.name="Dotnet Bot" commit -m "$PushReason for $Repository/$BranchName"
+  if ($LASTEXITCODE -ne 0) {
+    Write-Error "Git commit failed with exit code $LASTEXITCODE."
+  }
+  Write-Host "git push"
+  git push
+  if ($LASTEXITCODE -ne 0) {
+    Write-Error "Git push failed with exit code $LASTEXITCODE."
+  }
 }
-Write-Host "git push"
-git push
-if ($LASTEXITCODE -ne 0) {
-  Write-Error "Git push failed with exit code $LASTEXITCODE."
+else {
+  Write-Host "Skipping commit and push because there is nothing to commit"
 }
-
 # Return to the original directory
 Pop-Location

eng/common/sdl/run-sdl.ps1

@@ -12,7 +12,7 @@ Param(
 
 $ErrorActionPreference = "Stop"
 Set-StrictMode -Version 2.0
-$LASTEXITCODE = 0
+$global:LASTEXITCODE = 0
 
 # We store config files in the r directory of .gdn
 Write-Host $ToolsList
@@ -21,7 +21,7 @@ $ValidPath = Test-Path $GuardianCliLocation
 
 if ($ValidPath -eq $False)
 {
-  Write-Host "Invalid Guardian CLI Location."
+  Write-Error "Invalid Guardian CLI Location."
   exit 1
 }
 
@@ -35,15 +35,15 @@ foreach ($tool in $ToolsList) {
     Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" TargetDirectory < $TargetDirectory `" `" OutputType < pre `" $(If ($CrScanAdditionalRunConfigParams) {$CrScanAdditionalRunConfigParams})"
     & $GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args " TargetDirectory < $TargetDirectory " "OutputType < pre" $(If ($CrScanAdditionalRunConfigParams) {$CrScanAdditionalRunConfigParams})
     if ($LASTEXITCODE -ne 0) {
-      Write-Host "Guardian configure for $tool failed with exit code $LASTEXITCODE."
+      Write-Error "Guardian configure for $tool failed with exit code $LASTEXITCODE."
       exit $LASTEXITCODE
     }
   }
   if ($tool -eq "policheck") {
     Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" Target < $TargetDirectory `" $(If ($PoliCheckAdditionalRunConfigParams) {$PoliCheckAdditionalRunConfigParams})"
     & $GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args " Target < $TargetDirectory " $(If ($PoliCheckAdditionalRunConfigParams) {$PoliCheckAdditionalRunConfigParams})
     if ($LASTEXITCODE -ne 0) {
-      Write-Host "Guardian configure for $tool failed with exit code $LASTEXITCODE."
+      Write-Error "Guardian configure for $tool failed with exit code $LASTEXITCODE."
       exit $LASTEXITCODE
     }
   }
@@ -54,6 +54,6 @@ foreach ($tool in $ToolsList) {
 Write-Host "$GuardianCliLocation run --working-directory $WorkingDirectory --baseline mainbaseline --update-baseline $UpdateBaseline --logger-level $GuardianLoggerLevel $configParam"
 & $GuardianCliLocation run --working-directory $WorkingDirectory --tool $tool --baseline mainbaseline --update-baseline $UpdateBaseline --logger-level $GuardianLoggerLevel $configParam
 if ($LASTEXITCODE -ne 0) {
-  Write-Host "Guardian run for $ToolsList using $configParam failed with exit code $LASTEXITCODE."
+  Write-Error "Guardian run for $ToolsList using $configParam failed with exit code $LASTEXITCODE."
   exit $LASTEXITCODE
 }

eng/common/templates/job/onelocbuild.yml

@@ -0,0 +1,83 @@
+parameters:
+  # Optional: dependencies of the job
+  dependsOn: ''
+
+  # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool
+  pool:
+    vmImage: vs2017-win2016
+
+  CeapexPat: $(dn-bot-ceapex-package-r) # PAT for the loc AzDO instance https://dev.azure.com/ceapex
+  GithubPat: $(BotAccount-dotnet-bot-repo-PAT)
+
+  SourcesDirectory: $(Build.SourcesDirectory)
+  CreatePr: true
+  AutoCompletePr: false
+  UseCheckedInLocProjectJson: false
+  LanguageSet: VS_Main_Languages
+  LclSource: lclFilesInRepo
+  LclPackageId: ''
+  RepoType: gitHub
+  condition: ''
+
+jobs:
+- job: OneLocBuild
+
+  dependsOn: ${{ parameters.dependsOn }}
+
+  displayName: OneLocBuild
+
+  pool: ${{ parameters.pool }}
+
+  variables:
+    - group: OneLocBuildVariables # Contains the CeapexPat and GithubPat
+    - name: _GenerateLocProjectArguments
+      value: -SourcesDirectory ${{ parameters.SourcesDirectory }}
+        -LanguageSet "${{ parameters.LanguageSet }}"
+        -CreateNeutralXlfs
+    - ${{ if eq(parameters.UseCheckedInLocProjectJson, 'true') }}:
+      - name: _GenerateLocProjectArguments
+        value: ${{ variables._GenerateLocProjectArguments }} -UseCheckedInLocProjectJson
+
+
+  steps:
+    - task: Powershell@2
+      inputs:
+        filePath: $(Build.SourcesDirectory)/eng/common/generate-locproject.ps1
+        arguments: $(_GenerateLocProjectArguments)
+      displayName: Generate LocProject.json
+      condition: ${{ parameters.condition }}
+
+    - task: OneLocBuild@2
+      displayName: OneLocBuild
+      env:
+        SYSTEM_ACCESSTOKEN: $(System.AccessToken)
+      inputs:
+        locProj: Localize/LocProject.json
+        outDir: $(Build.ArtifactStagingDirectory)
+        lclSource: ${{ parameters.LclSource }}
+        lclPackageId: ${{ parameters.LclPackageId }}
+        isCreatePrSelected: ${{ parameters.CreatePr }}
+        ${{ if eq(parameters.CreatePr, true) }}:
+          isAutoCompletePrSelected: ${{ parameters.AutoCompletePr }}
+        packageSourceAuth: patAuth
+        patVariable: ${{ parameters.CeapexPat }}
+        ${{ if eq(parameters.RepoType, 'gitHub') }}:
+          repoType: ${{ parameters.RepoType }}
+          gitHubPatVariable: "${{ parameters.GithubPat }}"
+      condition: ${{ parameters.condition }}
+
+    - task: PublishBuildArtifacts@1
+      displayName: Publish Localization Files
+      inputs:
+        PathtoPublish: '$(Build.ArtifactStagingDirectory)/loc'
+        PublishLocation: Container
+        ArtifactName: Loc
+      condition: ${{ parameters.condition }}
+
+    - task: PublishBuildArtifacts@1
+      displayName: Publish LocProject.json
+      inputs:
+        PathtoPublish: '$(Build.SourcesDirectory)/Localize/'
+        PublishLocation: Container
+        ArtifactName: Loc
+      condition: ${{ parameters.condition }}

eng/common/templates/post-build/post-build.yml

@@ -391,6 +391,30 @@ stages:
     shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json'
     symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json'
 
+- template: \eng\common\templates\post-build\channels\generic-public-channel.yml
+  parameters:
+    artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+    publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
+    symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
+    stageName: 'VS16_11_Publishing'
+    channelName: 'VS 16.11'
+    channelId: 1926
+    transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-transport/nuget/v3/index.json'
+    shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json'
+    symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json'
+
+- template: \eng\common\templates\post-build\channels\generic-public-channel.yml
+  parameters:
+    artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+    publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
+    symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
+    stageName: 'VS17_0_Publishing'
+    channelName: 'VS 17.0'
+    channelId: 1853
+    transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-transport/nuget/v3/index.json'
+    shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json'
+    symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json'
+
 - template: \eng\common\templates\post-build\channels\generic-public-channel.yml
   parameters:
     artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}