ci: record benchmark host provenance and storage telemetry

.github/actions/run-benchmark/action.yml

@@ -52,7 +52,9 @@ runs:
         fi
         run_dir="data/runs/${{ inputs.platform }}/microvm-v2/1vcpu"
         acceptance_output="${run_dir}/acceptance.json"
-        rm -f "${acceptance_output}"
+        attempt_log="${run_dir}/acceptance-attempts.json"
+        rm -f "${acceptance_output}" "${attempt_log}" \
+          "${run_dir}"/acceptance-attempt-*.json
         for attempt in 1 2; do
           attempt_output="${run_dir}/acceptance-attempt-${attempt}.json"
           python3 scripts/nvx.py benchmark \
@@ -71,7 +73,8 @@ runs:
           validation_status=0
           python3 scripts/nvx.py performance validate-openvmm \
             --platform "${{ inputs.platform }}" \
-            --input "${attempt_output}" || validation_status=$?
+            --input "${attempt_output}" \
+            --attempt-log "${attempt_log}" || validation_status=$?
           if [[ "${validation_status}" -eq 0 ]]; then
             cp "${attempt_output}" "${acceptance_output}"
             break
@@ -220,8 +223,15 @@ runs:
         }
         $RunDir = "data/runs/${{ inputs.platform }}/microvm-v2/1vcpu"
         $AcceptanceOutput = "$RunDir/acceptance.json"
-        if (Test-Path -LiteralPath $AcceptanceOutput) {
-          Remove-Item -LiteralPath $AcceptanceOutput
+        $AttemptLog = "$RunDir/acceptance-attempts.json"
+        foreach ($Stale in $AcceptanceOutput, $AttemptLog) {
+          if (Test-Path -LiteralPath $Stale) {
+            Remove-Item -LiteralPath $Stale
+          }
+        }
+        if (Test-Path -LiteralPath $RunDir) {
+          Get-ChildItem -LiteralPath $RunDir -Filter "acceptance-attempt-*.json" |
+            Remove-Item
         }
         foreach ($Attempt in 1, 2) {
           $AttemptOutput = "$RunDir/acceptance-attempt-$Attempt.json"
@@ -242,7 +252,8 @@ runs:
           if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
           python scripts\nvx.py performance validate-openvmm `
             --platform "${{ inputs.platform }}" `
-            --input $AttemptOutput
+            --input $AttemptOutput `
+            --attempt-log $AttemptLog
           $ValidationStatus = $LASTEXITCODE
           if ($ValidationStatus -eq 0) {
             Copy-Item -LiteralPath $AttemptOutput -Destination $AcceptanceOutput
@@ -373,6 +384,7 @@ runs:
           --require-shell-snapshot \
           --require-shared-suite \
           --lifecycle-input "${run_dir}/acceptance.json" \
+          --lifecycle-attempt-log "${run_dir}/acceptance-attempts.json" \
           --summary "${GITHUB_STEP_SUMMARY}"
         device_run_dir="${run_dir}/device-io"
         device_results_dir="${device_run_dir}/results"
@@ -411,6 +423,7 @@ runs:
           --require-shell-snapshot `
           --require-shared-suite `
           --lifecycle-input "$RunDir/acceptance.json" `
+          --lifecycle-attempt-log "$RunDir/acceptance-attempts.json" `
           --summary $env:GITHUB_STEP_SUMMARY
         if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
         $DeviceRunDir = "$RunDir/device-io"

.github/actions/validate-nvx/action.yml

@@ -10,7 +10,7 @@ runs:
       run: |
         set -euo pipefail
         python3 -m compileall -q scripts
-        python3 -m unittest scripts/test_performance.py scripts/test_nvx_tools.py scripts/test_microvm_tests.py scripts/test_development_release.py scripts/test_egress_policy.py scripts/test_control_session.py scripts/test_managed_agent.py scripts/test_guest_time.py scripts/test_time_abi.py scripts/test_tool_versions.py -v
+        python3 -m unittest scripts/test_performance.py scripts/test_nvx_tools.py scripts/test_microvm_tests.py scripts/test_development_release.py scripts/test_egress_policy.py scripts/test_control_session.py scripts/test_managed_agent.py scripts/test_guest_time.py scripts/test_host_telemetry.py scripts/test_time_abi.py scripts/test_tool_versions.py -v
         python3 scripts/test_adversarial.py -v
         python3 .github/skills/nvx-host-connect/scripts/test_hosts.py -v
         python3 scripts/nvx.py --help
@@ -29,7 +29,7 @@ runs:
         $ErrorActionPreference = "Stop"
         python -m compileall -q scripts
         if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
-        python -m unittest scripts/test_performance.py scripts/test_nvx_tools.py scripts/test_microvm_tests.py scripts/test_development_release.py scripts/test_egress_policy.py scripts/test_control_session.py scripts/test_managed_agent.py scripts/test_guest_time.py scripts/test_time_abi.py scripts/test_tool_versions.py -v
+        python -m unittest scripts/test_performance.py scripts/test_nvx_tools.py scripts/test_microvm_tests.py scripts/test_development_release.py scripts/test_egress_policy.py scripts/test_control_session.py scripts/test_managed_agent.py scripts/test_guest_time.py scripts/test_host_telemetry.py scripts/test_time_abi.py scripts/test_tool_versions.py -v
         if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
         python scripts\test_adversarial.py -v
         if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }

.github/skills/nvx-benchmark/SKILL.md

@@ -213,6 +213,8 @@ Verify that the retained result set contains the acceptance JSON, raw logs under
 `performance/`, `device-io/device-io.log`, `summary.md`, collected CSV files, and
 `benchmark-metadata.json` records with the expected commit, platform, and sampling
 counts. Require the expected artifact hashes only in the device-I/O metadata record.
+Check that the `host` provenance in the acceptance JSON and metadata names the
+measured machine and the scratch volume that the run used.
 
 ## Report
 

doc/benchmarks.md

@@ -502,6 +502,32 @@ minor and major faults, total page faults, and, when `smaps_rollup` is available
 and private RSS bytes. Windows reports working set, peak working set, private commit, and page
 faults.
 
+On Windows, each retained capture sample also carries `storage_telemetry`. From just before the
+host requests the snapshot until the source process exits, a coordinator thread samples
+cumulative host counters about every 50 ms: read and write bytes, request counts, response
+times, idle time, and queue depth of the scratch, workspace, and system volumes; CPU idle,
+kernel, and user time; and the system cache's dirty pages and threshold, lazy-writer and flush
+pages, and modified- and mapped-page-writer pages. About every 500 ms, and in the first and last
+samples, it adds the CPU time and I/O transfers of the Microsoft Defender processes
+`MsMpEng.exe` and `MpDefenderCoreService.exe`. CI jobs run as Network Service, which cannot read
+performance counters, so the sampler uses only interfaces that any account can query: volume
+performance IOCTLs, `GetSystemTimes`, and `NtQuerySystemInformation`. Windows maintains disk
+counters only on demand, and the volume IOCTL returns none until a consumer asks for them, so the
+sampler keeps the disk performance WMI data block, which every account may query, open while it
+samples; it changes no system setting. A failed read is recorded
+in `errors` and never fails the benchmark. The raw `samples` are retained, and `phases`
+summarizes `capture.snapshot_generation` and `capture.mapped_memory_flush`. Each counter group,
+a volume, CPU times, cache counters, or Defender's processes, spans the samples nearest to each
+phase's start and end that carry it, so a failed read does not hide what neighboring samples
+recorded. The observer clock is aligned with OpenVMM's by the smallest observed-minus-reported
+time of the capture's records. Each summary reports the sampled interval it spans; per-volume
+throughput, IOPS, average latency, average queue length, busy time, and peak queue depth; CPU busy
+time; the lazy-writer, flush, and page-writer rates; the peak dirty cache and its threshold; and
+Defender CPU time and I/O. A counter group whose samples differ from the summary's, such as
+Defender's sparser process counters, reports its own interval.
+Linux captures do not flush a file-backed guest RAM image, take about 1 to 4 ms, and record no
+storage telemetry.
+
 `profile.phases` groups samples by `operation.phase` and stores raw `samples_ms` plus p50,
 nearest-rank p95, minimum, and maximum. The top-level JSON path is
 `snapshot_profile_matrix.<backend>.<memory_mib>`. Each memory entry contains the capture result and
@@ -582,6 +608,16 @@ bimodal host-stall series into topology-wide history. On the Linux CI runners,
 snapshot generation takes about 1 ms on KVM and about 4 ms on MSHV, so a
 sub-millisecond CPU stall in two samples is enough to cross the gap limit.
 
+Each validation also appends the attempt's file, SHA-256 digest, outcome (`valid`,
+`unstable`, or `invalid`), exit status, and error to `acceptance-attempts.json`,
+which the acceptance step removes, with any earlier attempt files, before measuring.
+The publish step passes that log to `performance collect --lifecycle-attempt-log`.
+Collection fails before it writes a CSV unless `acceptance.json` is a byte-identical
+copy of exactly one valid attempt; it then prints the attempt that supplied the
+published lifecycle metrics. The step summary lists every attempt with its outcome,
+snapshot-generation p50, and whether it was published, and reports the storage
+telemetry of each attempt, rejected ones included.
+
 Before updating the run-scoped `benchmark-<platform>-<run-id>` artifact, each platform
 uploads its raw results and lifecycle profiles to the immutable
 `benchmark-diagnostics-<platform>-<run-id>-attempt-<run-attempt>` artifact, including
@@ -592,6 +628,25 @@ platform results from an earlier workflow attempt. The `acceptance-attempt-N.jso
 files identify the bounded lifecycle remeasurements within one workflow attempt,
 not the workflow's `github.run_attempt`.
 
+Lifecycle JSON results and workload `benchmark-metadata.json` files record host
+provenance under `host`: the runner name from `RUNNER_NAME`, the machine name,
+operating system, CPU model, logical processor count, and physical memory, and the
+volume, file system, size, system-volume flag, disk number, and SCSI address behind
+the workspace, temporary, and scratch directories. The disk number and SCSI address
+tell the Azure OS disk from a data-disk LUN or the temporary disk. On a possible Azure
+VM, the coordinator queries the instance metadata service once, directly rather than
+through a proxy, and keeps only the VM size; each managed disk's SKU, size, caching
+policy, write-accelerator and ephemeral settings, and data-disk LUN; the throughput
+and IOPS limits that Ultra Disks report; and the temporary disk's size. It never
+records names, resource or VM identifiers, the subscription, tags, network addresses,
+or credentials, and records an unreachable service as `azure.error`. Acceptance and
+diagnostic suites print the provenance before they measure, and the step summary
+includes it as the benchmark host table. A device-io resume ignores `host` when it
+compares controls and provenance. Runner and machine names are diagnostic only: the
+regression gate pools every runner of a series into one history, and a series should
+be split by runner only if evidence shows a persistent runner-specific offset larger
+than the gate's tolerance.
+
 When investigating instability, compare each attempt's
 `snapshot_capture.<backend>.profile.raw_samples` with its `samples_ms`. For example, a
 slow `capture.mapped_memory_flush` with otherwise stable capture phases localizes
@@ -602,6 +657,25 @@ Reproduce with the exact executable and guest artifact hashes on the same host
 before attributing the delay to a source change. Keep the stability thresholds and
 bounded remeasurement unchanged when collecting diagnostic evidence.
 
+On Windows, the job log prints each measured capture's `capture.mapped_memory_flush`
+time with the host activity during it, and the step summary tabulates the same
+telemetry for every attempt: the scratch volume's write throughput, average write
+latency, average queue length, and busy time; writes to the other monitored
+volumes; CPU busy time; the peak dirty cache; lazy-writer throughput; and Defender
+CPU time. Compare slow flushes with fast ones in the same series:
+
+- Disk throttling: the slow flushes write scratch at a lower, flat rate with longer
+  latency and queue while it stays busy, and the other columns stay quiet. Compare
+  that rate with the published limits of the disk SKU and size in the host table.
+- Scanning: Defender CPU time or I/O rises during the slow flushes.
+- Writeback: lazy writes, the dirty cache, or writes to the other volumes rise
+  during the slow flushes.
+- Unrelated host load: CPU busy time rises while scratch stays idle for part of
+  the flush.
+
+In the JSON, the flush's own page writes appear in `memory.cache_flush_mib_per_second`,
+and the raw samples show how the rates change within a flush.
+
 Snapshot generation is bounded by the write throughput of the benchmark's
 temporary directory, because capture flushes guest RAM through a backing file
 created there. The Windows backing file stays dense so restore keeps the
@@ -621,6 +695,18 @@ translation of the same directory instead of falling back to WSL's `/tmp`.
 Use `--scratch-dir` for manual runs whose temporary directory shares a volume
 with other I/O-heavy work.
 
+Treat a change to the storage behind benchmark scratch or the runner workspace as a
+performance-platform change. This includes a different VM size, disk SKU, size,
+performance tier, caching policy, or bursting setting, and moving scratch between
+the OS, data, and temporary disks. Because snapshot generation is bounded by the
+scratch volume's write throughput, such a change shifts
+`openvmm_snapshot_generation` and can shift the other lifecycle metrics. Apply it to
+every pooled runner of the series at once; a change on some runners only would mix
+two storage regimes in one history. In the same pull request, reset the affected
+metrics by removing them from the series' history files, such as
+`data/windows-whp-virtual-machine-microvm-v2-1vcpu.csv`, so that the gate restarts
+their ten-point warmup instead of comparing the new storage with the old baseline.
+
 The regression gate compares the target p50 with the median of the latest 10
 p50 values on the pull request's base branch and requires all 10
 matching history points. A metric regresses only when it is more than 40%

doc/usage.md

@@ -756,7 +756,7 @@ python3 scripts/nvx.py benchmark [OPTIONS]
 | `--timeout SECONDS` | `10` | Set the time allowed for each boot marker. |
 | `--teardown-mode {guest-exit,host-terminate,host-sigterm}` | `guest-exit` | Select how to stop a measured VM; `host-sigterm` is a deprecated alias. |
 | `--skip-build` | off | Reuse existing release binaries. |
-| `--snapshot-profile` | off | Retain OpenVMM lifecycle phase samples and host counters; implied by the `snapshot-profile` suite. |
+| `--snapshot-profile` | off | Retain OpenVMM lifecycle phase samples and host counters, and on Windows sample storage, CPU, writeback, and Defender telemetry during each capture; implied by the `snapshot-profile` suite. |
 | `--cache-state {warm,cold,both}` | `both` | Select artifact cache states for the `snapshot-profile` suite. |
 | `--output PATH` | none | Write the benchmark result as JSON. |
 | `--output-dir PATH` | none | Write canonical workload logs to a directory. |
@@ -778,6 +778,8 @@ too: `auto` rejects such a catalog without falling back.
 The `e2e` suite uses the general memory size and measures cold start, snapshot
 generation, snapshot restore, guest-exit teardown, and peak RSS against the
 shell-ready markers. CI uses the default 128 MiB baseline.
+JSON results and workload metadata record the benchmark host's provenance under
+`host`, without credentials or Azure resource identifiers.
 See [Benchmark](benchmarks.md) for suite semantics, platform support, metric
 definitions, and complete examples.
 
@@ -799,6 +801,7 @@ python3 scripts/nvx.py performance collect
     [--require-shared-suite]
     [--require-shell-snapshot-restore-512]
     [--lifecycle-input PATH]
+    [--lifecycle-attempt-log PATH]
     [--summary PATH]
 ```
 
@@ -810,20 +813,31 @@ restore metric from a 2-, 4-, or 8-vCPU run.
 result, producing the 29-metric microVM CI result. A directory whose metadata
 selects `device-io` is collected as five additional ABI-2, one-vCPU `ops/s`
 metrics; CI merges them into a 34-metric one-vCPU result.
-`--summary` writes the p50 table plus lifecycle min/max/sample-count and RSS diagnostics.
+`--lifecycle-attempt-log` reads the attempt log that `validate-openvmm`
+records. The `--lifecycle-input` file must be a byte-identical copy of exactly
+one valid logged attempt, or collection fails before it writes a CSV; the
+command names that attempt.
+`--summary` writes the p50 table plus lifecycle min/max/sample-count and RSS
+diagnostics, the benchmark host provenance, and any snapshot-generation storage
+telemetry. With `--lifecycle-attempt-log`, it also lists every attempt, marks
+the published one, and reports the telemetry of rejected attempts.
 
 #### `performance validate-openvmm`
 
 ```text
 python3 scripts/nvx.py performance validate-openvmm
     --platform PLATFORM
     --input PATH
+    [--attempt-log PATH]
 ```
 
 Validates a complete 128 MiB, guest-exit OpenVMM `e2e` result without
 writing a CSV. Snapshot-generation instability exits with status 75 so
 callers can remeasure the temporary host condition selectively; other
-malformed or incomplete inputs exit with status 2.
+malformed or incomplete inputs exit with status 2. `--attempt-log` appends
+the input's path, SHA-256 digest, outcome (`valid`, `unstable`, or
+`invalid`), exit status, and error to a JSON attempt log, creating it on the
+first attempt.
 
 #### `performance collect-openvmm`