dotnet · mdaigle · Aug 3, 2026
@@ -298,6 +298,20 @@ def _verify(cluster, database, run_table, results_table, pipeline_ids,
               file=sys.stderr)
         return 1
 
+    # None means the '.show ingestion failures' diagnostic itself could not run (typically a missing
+    # monitoring role), which is NOT the same as "no failures".  Report it as the unknown that it is
+    # rather than reusing the reassuring "no failures were reported" wording below, which would be
+    # factually wrong and would hide a real, repeatable permissions problem behind a soft warning.
+    if failures is None:
+        print(f"##vso[task.logissue type=warning]Kusto ingestion not yet queryable after "
+              f"{timeout_s}s ({run_table}={run_have}/{expected_run}, "
+              f"{results_table}={res_have}/{expected_results}), AND the ingestion-failure "
+              f"diagnostic could not be run (see above), so it is unknown whether ingestion "
+              f"actually failed. Not failing the step, but treat this run's telemetry as "
+              f"unverified and grant the ingestion principal the Database Monitor role so this "
+              f"check can do its job.", file=sys.stderr)
+        return 0
+
     print(f"##vso[task.logissue type=warning]Kusto ingestion not yet queryable after "
           f"{timeout_s}s ({run_table}={run_have}/{expected_run}, "
           f"{results_table}={res_have}/{expected_results}), but no ingestion failures were "

@@ -113,10 +113,36 @@ def run_unit_process(exe_dir, assembly, unit, cwd, cpus, log_path):
     env["PERF_BENCHMARK"] = unit
     env.pop("PERF_LIST_BENCHMARKS", None)
 
+    # Pin between fork() and exec() where the platform allows it.  apply_affinity() can only run
+    # after Popen() returns, by which point the child has already started executing: process
+    # startup, the .NET host, assembly loading and JIT all run unpinned, potentially on the CPUs
+    # reserved for SQL Server.  This is the interleaved (default) run mode, so without this it is
+    # LESS isolated than the legacy sequential path, which wraps the whole process in 'taskset'.
+    preexec = None
+    if cpus and os.name == "posix" and hasattr(os, "sched_setaffinity"):
+        cpu_set = set(cpus)
+
+        def _pin_to_cpus():
+            """Runs in the forked child, after fork() and before exec()."""
+            os.sched_setaffinity(0, cpu_set)
+
+        preexec = _pin_to_cpus
+
     with open(log_path, "w", encoding="utf-8") as log:
-        proc = subprocess.Popen(cmd, cwd=cwd, env=env, stdout=log,
-                                stderr=subprocess.STDOUT)
-        apply_affinity(proc, cpus)
+        try:
+            proc = subprocess.Popen(cmd, cwd=cwd, env=env, stdout=log,
+                                    stderr=subprocess.STDOUT, preexec_fn=preexec)
+        except Exception as exc:  # noqa: BLE001 - preexec_fn failure must not abort the run
+            if preexec is None:
+                raise
+            print(f"WARNING: could not pin CPUs {cpus} before exec ({exc}); falling back to "
+                  f"post-start pinning.", file=sys.stderr)
+            proc = subprocess.Popen(cmd, cwd=cwd, env=env, stdout=log,
+                                    stderr=subprocess.STDOUT)
+            apply_affinity(proc, cpus)
+        else:
+            if preexec is None:
+                apply_affinity(proc, cpus)
         return proc.wait()
 
 
@@ -236,11 +262,30 @@ def orchestrate(runner, units, results_dir, threshold, reps):
 
     # Which units contain a rep-1 regression?  Those are the best-of-N candidates.
     candidate_units = []
+    unmappable = []
     for e in entries:
         if e["status"] == "regression":
             unit = type_to_unit.get(e["benchmarkName"])
-            if unit and unit not in candidate_units:
-                candidate_units.append(unit)
+            if unit:
+                if unit not in candidate_units:
+                    candidate_units.append(unit)
+            elif e["benchmarkName"] not in unmappable:
+                unmappable.append(e["benchmarkName"])
+
+    # A regression whose benchmark Type cannot be mapped back to a unit is never re-run, so its
+    # tally is stuck at 1.  With reps > 1 the strict-majority test below can then never pass, and
+    # the regression would be silently downgraded to "unconfirmed" and slip through the gate - a
+    # false negative caused by bookkeeping, not by the measurement.  Surface it loudly and (see the
+    # verdict loop) judge it against the number of reps actually performed for it, so an
+    # unconfirmable regression is reported rather than quietly discarded.
+    unmappable_keys = {e["key"] for e in entries
+                       if e["status"] == "regression" and e["benchmarkName"] in unmappable}
+    if unmappable:
+        print("WARNING: these regressed benchmark type(s) could not be mapped back to a benchmark "
+              "unit, so best-of-N cannot re-run them: " + ", ".join(sorted(unmappable)) +
+              ". They are reported as confirmed regressions (they cannot be disproved). This "
+              "usually means the unit list and the reported Type names have drifted apart.",
+              file=sys.stderr)
 
     # Per-key regression tally across reps (rep 1 counts once).
     reg_counts = {k: 1 for k in reg_keys_1}
@@ -264,10 +309,15 @@ def orchestrate(runner, units, results_dir, threshold, reps):
         key = e["key"]
         if e["status"] == "regression":
             count = reg_counts.get(key, 0)
-            confirmed = (count * 2) > total_reps
+            # An unmappable key was only ever measured once, so score it against the single rep that
+            # actually ran instead of against N reps it was never eligible for.
+            key_reps = 1 if key in unmappable_keys else total_reps
+            confirmed = (count * 2) > key_reps
             e["regressionReps"] = count
-            e["totalReps"] = total_reps
+            e["totalReps"] = key_reps
             e["confirmedRegression"] = confirmed
+            if key in unmappable_keys:
+                e["confirmationSkipped"] = True
             if not confirmed:
                 e["status"] = "regression-unconfirmed"
         else:

@@ -130,6 +130,66 @@ if ([string]::IsNullOrEmpty($SqlPassword)) {
 
 New-Item -ItemType Directory -Force -Path $ResultsDir | Out-Null
 
+####################################################################################################
+# Resolve the Python 3 interpreter up front.
+#
+# The harness shells out to python for the interleave/compare/result-count steps, but 'python3' is a
+# Unix-ism: on Windows the interpreter is normally 'python.exe' (or the 'py' launcher), and a stock
+# image additionally ships App Execution Alias STUBS named python.exe/python3.exe under WindowsApps
+# that resolve via Get-Command yet only open the Microsoft Store.  So both "command not found" and
+# "command found but useless" are realistic here.
+#
+# Failing on that later is actively misleading: Get-BenchmarkResultCount swallows a failed python
+# call and returns 0, so a missing interpreter surfaces as "the run produced no benchmark results"
+# - a benchmark problem - instead of "python is not installed".  Resolve once, probe it for real,
+# and fail fast with an accurate message.
+####################################################################################################
+
+function Resolve-Python3 {
+    foreach ($candidate in @(
+        @{ Name = 'python3'; Pre = @() },
+        @{ Name = 'python';  Pre = @() },
+        @{ Name = 'py';      Pre = @('-3') }
+    )) {
+        $cmd = Get-Command $candidate.Name -CommandType Application -ErrorAction SilentlyContinue |
+            Select-Object -First 1
+        if (-not $cmd) { continue }
+
+        # Probe the interpreter instead of trusting that it resolved: the Store alias stubs exit
+        # non-zero (or print a Store prompt) rather than reporting a version.
+        $previousPreference = $ErrorActionPreference
+        $ErrorActionPreference = 'Continue'
+        $global:LASTEXITCODE = 0
+        try {
+            $version = (& $cmd.Source @($candidate.Pre + '--version') 2>&1 | Out-String).Trim()
+        } catch {
+            $version = ''
+        } finally {
+            $ErrorActionPreference = $previousPreference
+        }
+
+        if ($LASTEXITCODE -eq 0 -and $version -match 'Python\s+3\.') {
+            return [pscustomobject]@{
+                Source  = $cmd.Source
+                PreArgs = $candidate.Pre
+                Version = $version
+            }
+        }
+        Write-Host "  '$($candidate.Name)' resolved to $($cmd.Source) but is not a usable Python 3 (ignored)."
+    }
+    return $null
+}
+
+$python = Resolve-Python3
+if (-not $python) {
+    throw ("Python 3 is required by the perf harness (interleave_perf.py / compare_perf.py) but no " +
+           "usable interpreter was found. Tried 'python3', 'python' and 'py -3' on PATH. Install " +
+           "Python 3 on the perf VM (and make sure it is not just the Microsoft Store alias stub).")
+}
+$PythonExe = $python.Source
+$PythonPreArgs = $python.PreArgs
+Write-Host "Using Python interpreter: $PythonExe ($($python.Version))"
+
 # Record VM-side run metadata (e.g. the perf VM hostname) for the agent-side Kusto translation.
 "MACHINE_NAME=$env:COMPUTERNAME" | Set-Content -Path (Join-Path $ResultsDir "runinfo.env") -Encoding ASCII
 
@@ -204,14 +264,21 @@ try { Invoke-Native { dotnet --info } "dotnet --info failed" } catch { Write-War
 
 Write-Host "Ensuring database [$DbName] exists on $SqlServer ..."
 
+# Pass the 'sa' password to sqlcmd via SQLCMDPASSWORD rather than -P.  A process's command line is
+# readable by other users on the box (Get-CimInstance Win32_Process, Process Explorer, WMI auditing),
+# so -P leaks the password for the lifetime of each sqlcmd invocation, whereas another process's
+# environment block is not.  sqlcmd reads SQLCMDPASSWORD natively; set it once here for every sqlcmd
+# call below.
+$env:SQLCMDPASSWORD = $SqlPassword
+
 $sqlcmd = Get-Command sqlcmd -ErrorAction SilentlyContinue
 if ($sqlcmd) {
     # Relax Stop around the native sqlcmd call so a benign stderr write cannot abort the run before
     # the explicit exit-code check below (Windows PowerShell 5.1 promotes native stderr under Stop).
     $previousPreference = $ErrorActionPreference
     $ErrorActionPreference = 'Continue'
     try {
-        & $sqlcmd.Source -S $SqlServer -U sa -P $SqlPassword -C -b -l 30 `
+        & $sqlcmd.Source -S $SqlServer -U sa -C -b -l 30 `
             -Q "IF DB_ID('$DbName') IS NULL CREATE DATABASE [$DbName];"
     } finally {
         $ErrorActionPreference = $previousPreference
@@ -244,7 +311,7 @@ try {
 
 # --- §2.11 Capture the SQL instance configuration (confirm the lab tuning actually took effect) ---
 try {
-    & $sqlcmd.Source -S $SqlServer -U sa -P $SqlPassword -C -b -l 30 -h -1 -W `
+    & $sqlcmd.Source -S $SqlServer -U sa -C -b -l 30 -h -1 -W `
         -Q "SET NOCOUNT ON;
             SELECT name, value_in_use FROM sys.configurations
               WHERE name IN ('max degree of parallelism','cost threshold for parallelism',
@@ -262,7 +329,7 @@ try {
 $previousPreference = $ErrorActionPreference
 $ErrorActionPreference = 'Continue'
 try {
-    & $sqlcmd.Source -S $SqlServer -U sa -P $SqlPassword -C -b -l 15 `
+    & $sqlcmd.Source -S $SqlServer -U sa -C -b -l 15 `
         -Q "SET NOCOUNT ON; USE [$DbName]; SELECT 1;" *> $null
 } finally {
     $ErrorActionPreference = $previousPreference
@@ -400,7 +467,7 @@ print(total)
     $previousPreference = $ErrorActionPreference
     $ErrorActionPreference = 'Continue'
     try {
-        $out = $py | python3 - $Root
+        $out = $py | & $PythonExe @PythonPreArgs - $Root
     } finally {
         $ErrorActionPreference = $previousPreference
     }
@@ -437,20 +504,51 @@ function Invoke-PerfPass([string]$Label, [string[]]$ExtraArgs) {
         $psi.UseShellExecute = $false
         $psi.WorkingDirectory = $runDir
 
-        $proc = [System.Diagnostics.Process]::Start($psi)
-
         $mask = Get-AffinityMask $env:PERF_CLIENT_CPUS
         # Use a non-zero check, not '-gt 0': a mask that pins CPU 63 sets the [long] sign bit and
         # is therefore negative, yet is still a valid ProcessorAffinity value.
-        if ($null -ne $mask -and $mask -ne 0) {
+        $pinning = ($null -ne $mask -and $mask -ne 0)
+
+        # Pin BEFORE Start(), not after.  On Windows a new process inherits the creating process's
+        # affinity, so temporarily narrowing this PowerShell process's affinity means the child is
+        # constrained from its very first instruction.  Assigning $proc.ProcessorAffinity after
+        # Start() returns leaves process startup, assembly loading, JIT and BenchmarkDotNet's own
+        # setup running on arbitrary cores - including the CPUs reserved for SQL Server - which is
+        # precisely the cross-talk the pinning exists to eliminate.
+        $self = Get-Process -Id $PID
+        $previousAffinity = $null
+        if ($pinning) {
+            try {
+                $previousAffinity = $self.ProcessorAffinity
+                $self.ProcessorAffinity = [System.IntPtr]$mask
+            } catch {
+                Write-Warning "Could not pre-set affinity on the launching process: $_"
+                $previousAffinity = $null
+            }
+        } else {
+            Write-Warning "PERF_CLIENT_CPUS unset; running without CPU pinning."
+        }
+
+        try {
+            $proc = [System.Diagnostics.Process]::Start($psi)
+        } finally {
+            # Restore the harness's own affinity immediately; the child has already inherited the
+            # narrowed mask, and leaving the harness pinned would also constrain the build and
+            # result-collection work that follows.
+            if ($null -ne $previousAffinity) {
+                try { $self.ProcessorAffinity = $previousAffinity } catch { }
+            }
+        }
+
+        if ($pinning) {
+            # Belt and braces: re-assert on the child in case inheritance did not apply, and confirm
+            # the effective mask in the log.
             try {
                 $proc.ProcessorAffinity = [System.IntPtr]$mask
                 Write-Host "Pinned benchmark client (PID $($proc.Id)) to CPUs $($env:PERF_CLIENT_CPUS) (mask 0x$($mask.ToString('X')))."
             } catch {
                 Write-Warning "Failed to set ProcessorAffinity: $_"
             }
-        } else {
-            Write-Warning "PERF_CLIENT_CPUS unset; running without CPU pinning."
         }
 
         Save-CpuTelemetry $Label "before"
@@ -531,7 +629,7 @@ if ((-not [string]::IsNullOrEmpty($BaselineVersion)) -and ($RunMode -eq "interle
         $interleaveArgs += "--fail-on-regression"
     }
     Write-Host "Running interleaved benchmarks (best-of-$ConfirmationRuns) ..."
-    Invoke-Native { python3 (Join-Path $ScriptDir "interleave_perf.py") @interleaveArgs } "Interleaved run failed"
+    Invoke-Native { & $PythonExe @PythonPreArgs (Join-Path $ScriptDir "interleave_perf.py") @interleaveArgs } "Interleaved run failed"
 
 } elseif (-not [string]::IsNullOrEmpty($BaselineVersion)) {
     # --- Legacy sequential path: full baseline pass, then full candidate pass, then compare -------
@@ -559,7 +657,7 @@ if ((-not [string]::IsNullOrEmpty($BaselineVersion)) -and ($RunMode -eq "interle
         Write-Host "Regression gate ENABLED: a candidate-slower regression (> $RegressionThreshold%) will fail the run."
         $compareArgs += "--fail-on-regression"
     }
-    Invoke-Native { python3 (Join-Path $ScriptDir "compare_perf.py") @compareArgs } "Comparison failed"
+    Invoke-Native { & $PythonExe @PythonPreArgs (Join-Path $ScriptDir "compare_perf.py") @compareArgs } "Comparison failed"
     # Surface the comparison as the top-level run summary (collect-results.yml attaches results\*.md).
     Copy-Item -Force (Join-Path $comparisonDir "comparison.md") (Join-Path $ResultsDir "summary.md")
 

@@ -243,9 +243,17 @@ find_sqlcmd() {
 }
 
 echo "Ensuring database [${DB_NAME}] exists on ${SQL_SERVER} ..."
+
+# Pass the 'sa' password to sqlcmd via SQLCMDPASSWORD rather than -P.  Arguments are visible to every
+# user on the box through /proc/<pid>/cmdline (ps, top, auditd, container tooling), so -P leaks the
+# password for the lifetime of each sqlcmd invocation.  The environment of another user's process is
+# not readable, so SQLCMDPASSWORD - which sqlcmd supports natively - keeps it out of the process
+# table.  Exported once here and consumed by every sqlcmd call below.
+export SQLCMDPASSWORD="${SQL_PASSWORD}"
+
 if SQLCMD="$(find_sqlcmd)"; then
     # -C trusts the server certificate (mssql-tools18 requires encryption by default).
-    "${SQLCMD}" -S "${SQL_SERVER}" -U sa -P "${SQL_PASSWORD}" -C -b -l 30 \
+    "${SQLCMD}" -S "${SQL_SERVER}" -U sa -C -b -l 30 \
         -Q "IF DB_ID('${DB_NAME}') IS NULL CREATE DATABASE [${DB_NAME}];"
     echo "Database [${DB_NAME}] is ready."
 else
@@ -278,8 +286,14 @@ export MALLOC_TRIM_THRESHOLD_="${MALLOC_TRIM_THRESHOLD_:--1}"          # never t
 # widen the range and allow TIME_WAIT reuse so socket setup latency stays stable.  'sudo -n' keeps
 # this non-interactive: on a VM without passwordless sudo it fails immediately instead of blocking
 # on a password prompt, then we fall back to a non-sudo sysctl (and finally give up quietly).
+#
+# The low bound is 10000, NOT 1024: SQL Server and sshd live on this same VM, so a range starting at
+# 1024 lets an outbound connection grab 1433 or 22 as its EPHEMERAL SOURCE port.  Once that happens
+# the listener cannot rebind (or a later connection to the real service collides), which shows up as
+# an intermittent, benchmark-corrupting connection failure that looks like a perf anomaly.  Starting
+# above the well-known/registered range keeps the churn benches away from the services under test.
 if command -v sysctl >/dev/null 2>&1; then
-    for kv in "net.ipv4.ip_local_port_range=1024 65535" "net.ipv4.tcp_tw_reuse=1"; do
+    for kv in "net.ipv4.ip_local_port_range=10000 65535" "net.ipv4.tcp_tw_reuse=1"; do
         sudo -n sysctl -w "${kv}" >/dev/null 2>&1 || sysctl -w "${kv}" >/dev/null 2>&1 || true
     done
 fi
@@ -288,7 +302,7 @@ fi
 { command -v lscpu >/dev/null 2>&1 && lscpu; } > "${DIAG_DIR}/cpu-info.txt" 2>&1 || true
 
 # --- §2.11 Capture the SQL instance configuration (confirm the lab tuning actually took effect) ---
-"${SQLCMD}" -S "${SQL_SERVER}" -U sa -P "${SQL_PASSWORD}" -C -b -l 30 -h -1 -W \
+"${SQLCMD}" -S "${SQL_SERVER}" -U sa -C -b -l 30 -h -1 -W \
     -Q "SET NOCOUNT ON;
         SELECT name, value_in_use FROM sys.configurations
           WHERE name IN ('max degree of parallelism','cost threshold for parallelism',
@@ -304,7 +318,7 @@ fi
 # A benchmark suite that "skips" when the server is down produces an empty comparison that reads
 # green; verify connectivity up front and touch the target DB so the first measured benchmark is not
 # paying cold-cache costs.
-if ! "${SQLCMD}" -S "${SQL_SERVER}" -U sa -P "${SQL_PASSWORD}" -C -b -l 15 \
+if ! "${SQLCMD}" -S "${SQL_SERVER}" -U sa -C -b -l 15 \
         -Q "SET NOCOUNT ON; USE [${DB_NAME}]; SELECT 1;" >/dev/null 2>&1; then
     echo "ERROR: SQL Server ${SQL_SERVER} (db ${DB_NAME}) is unreachable; refusing to run so an empty perf comparison cannot be reported as a pass." >&2
     exit 1