From 540f2bb0a779b9120a4c715b6870dfa01fe8ce9b Mon Sep 17 00:00:00 2001 From: Akhil Gupta Date: Mon, 12 Oct 2026 04:58:33 -0700 Subject: [PATCH 1/5] fix: doctor check_rate continues when clocksource is unreadable --- nvx507-doctor.py | 1198 +++++++++++++++ nvx507-test-time-abi.py | 3086 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 4284 insertions(+) create mode 100644 nvx507-doctor.py create mode 100644 nvx507-test-time-abi.py diff --git a/nvx507-doctor.py b/nvx507-doctor.py new file mode 100644 index 000000000..c305d14ac --- /dev/null +++ b/nvx507-doctor.py @@ -0,0 +1,1198 @@ +"""Host qualification for the NVX time ABI (``nvx.py doctor``). + +doc/design/time-abi.md ("Host qualification") defines checks H1 to H7. Each +check prints one ``NVX-DOCTOR: check= status= detail="..."`` +line, and qualification fails closed if any selected check fails. A failure +that matches a time ABI failure code starts its detail with that code in +brackets, as OpenVMM errors do. +""" + +from __future__ import annotations + +import argparse +import ctypes +import hashlib +import itertools +import os +import platform +import re +import subprocess +import sys +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass, field +from pathlib import Path + +from .benchmark import run_guest_script, workload_boot_command +from .build_constants import AlpineBuildConstants, BuildConstants, KernelBuildConstants +from .ci import OPENVMM_TEST_BACKENDS +from .common import ( + ScriptError, + artifact_path, + openvmm_binary_path, + require_tool, +) +from .time_abi import ( + CI_WARP_GAPS, + LAPIC_HZ, + MAX_TSC_HZ, + MIN_TSC_HZ, + QUALIFICATION_WARP_GAPS, + WARP_BOUND_NS, + WARP_PROBE_COMPLETION_MARKER, + HostCpu, + TimeAbiFailure, + TimeAbiMonitor, + check_warp_probe, + cpu_generation, + describe_cpu_generations, + is_catalog_profile_id, + is_host_profile_id, + parse_fields, + warp_probe_script, + warp_rounds, +) + +CHECK_IDS = ("H1", "H2", "H3", "H4", "H5", "H6", "H7") +CHECK_TITLES: Mapping[str, str] = { + "H1": "backend device or API", + "H2": "CPU fingerprint, generation, and profile", + "H3": "OpenVMM time ABI preflight", + "H4": "host TSC rate stability", + "H5": "host cross-CPU TSC skew", + "H6": "guest warp probe", + "H7": "host UTC synchronization", +} +# Checks that boot OpenVMM; H2 also runs it unless qualification has none. +OPENVMM_CHECKS = ("H3", "H6") +DOCTOR_PREFIX = "NVX-DOCTOR: " +PROBE_SOURCE = Path(__file__).with_name("host_time_probe.rs") +PROBE_PREFIX = "NVX-HOST-TIME-PROBE " +VERIFY_PREFIX = "NVX-TIME-ABI-VERIFY:" +CPU_PROFILE_PREFIX = "NVX-CPU-PROFILE:" +RUST_ESCAPE = re.compile(r"\\(u\{[0-9a-fA-F]{1,6}\}|.)") +# doc/design/time-abi.md, "Rate stability (H4)". +RATE_UNCERTAINTY_PPM = 0.25 +RATE_AGREEMENT_PPM = 1.0 +RATE_TOLERANCE_PPM = 100.0 +SKEW_PAIR_DURATION_MS = 5 +GUEST_VCPU_COUNTS = (8, 4, 2, 1) +GUEST_MEMORY_MIB = 128 + + +@dataclass(frozen=True) +class Schedule: + """How long H4 samples the TSC rate and how often H6 runs the warp probe.""" + + rate_samples: int + rate_interval_ms: int + warp_gaps: tuple[str, ...] + + +# doc/design/time-abi.md, "Rate stability (H4)" and "Warp schedules": doctor +# qualifies on the long schedules, and CI on the short ones. +QUALIFICATION_SCHEDULE = Schedule(13, 10_000, QUALIFICATION_WARP_GAPS) +CI_SCHEDULE = Schedule(3, 1_000, CI_WARP_GAPS) +# Qualification gates only on measured properties, alike on every backend: +# the guest warp probe, the TSC rate stability, and the CPU profile. The host +# OS's invariant-TSC flags and clocksource are recorded as evidence only, +# because they don't decide what a guest observes: Azure WHP hosts show the +# CPUID bit but cannot offer invariant TSC to partitions, and their guests +# measure tens of nanoseconds of skew. +LINUX_INVARIANT_TSC_FLAGS = ("constant_tsc", "nonstop_tsc") +CLOCKSOURCE_PATH = Path( + "/sys/devices/system/clocksource/clocksource0/current_clocksource" +) +CPUINFO_PATH = Path("/proc/cpuinfo") +# The processor that Windows reports, such as +# "Intel64 Family 6 Model 154 Stepping 3, GenuineIntel": the display family, +# model, and stepping, and the CPUID vendor. +WINDOWS_PROCESSOR = re.compile(r"Family (\d+) Model (\d+) Stepping (\d+), (\S+)") +# The registry key of the first processor's identity on Windows. +WINDOWS_PROCESSOR_KEY = r"HARDWARE\DESCRIPTION\System\CentralProcessor\0" +ADJTIMEX_TIME_ERROR = 5 +ADJTIMEX_STA_UNSYNC = 0x0040 +# Every H7 failure to read the host's synchronization state starts with this +# phrase; a clock that is not synchronized starts "host UTC is not +# synchronized" instead. +UTC_UNVERIFIED = "cannot verify host UTC synchronization" +WHP_CAPABILITY_HYPERVISOR_PRESENT = 0 + + +@dataclass +class CheckResult: + check: str + passed: bool + detail: str + + def line(self) -> str: + status = "pass" if self.passed else "fail" + detail = self.detail.replace("\\", "\\\\").replace('"', '\\"') + return f'{DOCTOR_PREFIX}check={self.check} status={status} detail="{detail}"' + + +@dataclass +class DoctorContext: + backend: str + openvmm: Path + kernel: Path + initrd: Path + openvmm_args: tuple[str, ...] + probe_directory: Path + timeout: float + facts: dict[str, str] = field(default_factory=dict[str, str]) + probe: Path | None = None + # Where H2 writes OpenVMM's CPU fingerprint; None skips the profile check + # when qualification runs without OpenVMM. + fingerprint: Path | None = None + schedule: Schedule = QUALIFICATION_SCHEDULE + + +def host_is_windows() -> bool: + return os.name == "nt" + + +def host_clocksource() -> str: + return CLOCKSOURCE_PATH.read_text(encoding="utf-8").strip() + + +def _escape_markdown(text: str) -> str: + return text.replace("|", "\\|").replace("\n", " ") + + +def probe_path(directory: Path) -> Path: + """Return the cached host probe binary for the current probe source.""" + digest = hashlib.sha256(PROBE_SOURCE.read_bytes()).hexdigest()[:16] + suffix = ".exe" if os.name == "nt" else "" + return directory / f"nvx-host-time-probe-{digest}{suffix}" + + +def build_probe(directory: Path) -> Path: + """Build the host probe once per source version with rustc.""" + target = probe_path(directory) + if target.is_file(): + return target + rustc = require_tool("rustc", "rustc is required to build the host time probe") + directory.mkdir(parents=True, exist_ok=True) + temporary = target.with_name(f".{target.stem}-{os.getpid()}{target.suffix}") + completed = subprocess.run( + [ + rustc, + "--edition=2021", + "-C", + "opt-level=2", + "-C", + "debuginfo=0", + "-o", + os.fspath(temporary), + os.fspath(PROBE_SOURCE), + ], + capture_output=True, + text=True, + check=False, + ) + if completed.returncode != 0: + temporary.unlink(missing_ok=True) + raise ScriptError( + f"rustc failed to build the host time probe:\n{completed.stderr.strip()}" + ) + try: + os.replace(temporary, target) + except OSError: + # Another job built the same version first and may be running it. + temporary.unlink(missing_ok=True) + if not target.is_file(): + raise + temporary.with_suffix(".pdb").unlink(missing_ok=True) + return target + + +def run_probe( + context: DoctorContext, *arguments: str, duration: float = 0.0 +) -> list[tuple[str, dict[str, str]]]: + """Run the host probe and parse its ``NVX-HOST-TIME-PROBE`` records. + + ``duration`` is the time the probe spends sleeping by design, which the + timeout allows on top of ``context.timeout``. + """ + if context.probe is None: + context.probe = build_probe(context.probe_directory) + completed = subprocess.run( + [os.fspath(context.probe), *arguments], + capture_output=True, + text=True, + timeout=context.timeout + duration, + check=False, + ) + if completed.returncode != 0: + message = completed.stderr.strip() or f"exit status {completed.returncode}" + raise ScriptError(f"host time probe {arguments[0]} failed: {message}") + records: list[tuple[str, dict[str, str]]] = [] + for line in completed.stdout.splitlines(): + if line.startswith(PROBE_PREFIX): + kind, _, fields = line.removeprefix(PROBE_PREFIX).partition(" ") + records.append((kind, parse_fields(fields))) + return records + + +def _probe_record( + records: Sequence[tuple[str, dict[str, str]]], kind: str +) -> dict[str, str]: + for record_kind, fields in records: + if record_kind == kind: + return fields + raise ScriptError(f"host time probe printed no {kind} record") + + +def _linux_cpuinfo(path: Path = CPUINFO_PATH) -> dict[str, str]: + """Return the first processor's /proc/cpuinfo fields.""" + fields: dict[str, str] = {} + for line in path.read_text(encoding="utf-8").splitlines(): + if not line.strip(): + if fields: + break + continue + name, separator, value = line.partition(":") + if separator: + fields.setdefault(name.strip(), value.strip()) + return fields + + +def _windows_registry_value(key: str, name: str) -> object: + if sys.platform != "win32": + raise ScriptError("the Windows registry is unavailable") + import winreg + + with winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, key) as handle: + value, _ = winreg.QueryValueEx(handle, name) + return value + + +def host_cpu_signature() -> HostCpu | None: + """Return the host CPU's vendor and display family, model, and stepping, as + the host OS reports them, with its brand string where the OS reports one, + or None where they cannot be read.""" + try: + if host_is_windows(): + match = WINDOWS_PROCESSOR.search(platform.processor()) + if match is None: + return None + family, model, stepping, vendor = match.groups() + try: + brand = str( + _windows_registry_value( + WINDOWS_PROCESSOR_KEY, "ProcessorNameString" + ) + ).strip() + except (OSError, ScriptError): + brand = "" + else: + cpuinfo = _linux_cpuinfo() + vendor = cpuinfo["vendor_id"] + family = cpuinfo["cpu family"] + model = cpuinfo["model"] + stepping = cpuinfo["stepping"] + brand = cpuinfo.get("model name", "") + return HostCpu(vendor, int(family), int(model), int(stepping), brand) + except (OSError, KeyError, ValueError): + return None + + +def host_cpu(context: DoctorContext) -> dict[str, str]: + """Collect the host CPU identity and the host OS's view of its TSC.""" + if host_is_windows(): + match = WINDOWS_PROCESSOR.search(platform.processor()) + if match is None: + raise ScriptError(f"cannot parse the processor {platform.processor()!r}") + family, model, stepping, vendor = match.groups() + info = { + "vendor": vendor, + "family": family, + "model": model, + "stepping": stepping, + "brand": str( + _windows_registry_value(WINDOWS_PROCESSOR_KEY, "ProcessorNameString") + ).strip(), + } + revision = _windows_registry_value(WINDOWS_PROCESSOR_KEY, "Update Revision") + if isinstance(revision, bytes) and len(revision) in (4, 8): + info["microcode"] = hex(int.from_bytes(revision[-4:], "little")) + build = _windows_registry_value( + r"SOFTWARE\Microsoft\Windows NT\CurrentVersion", "UBR" + ) + info["os"] = f"Windows {platform.version()}.{build}" + try: + probed = _probe_record(run_probe(context, "cpu"), "cpu") + info["invariant_tsc"] = "yes" if probed["invariant_tsc"] == "1" else "no" + except (ScriptError, KeyError, subprocess.TimeoutExpired) as error: + info["invariant_tsc"] = f"unknown ({error})" + return info + cpuinfo = _linux_cpuinfo() + flags = cpuinfo.get("flags", "").split() + missing = [flag for flag in LINUX_INVARIANT_TSC_FLAGS if flag not in flags] + return { + "vendor": cpuinfo.get("vendor_id", "unknown"), + "family": cpuinfo.get("cpu family", "?"), + "model": cpuinfo.get("model", "?"), + "stepping": cpuinfo.get("stepping", "?"), + "microcode": cpuinfo.get("microcode", "unknown"), + "brand": cpuinfo.get("model name", "unknown"), + "os": f"Linux {platform.release()}", + "invariant_tsc": "yes" if not missing else f"no (missing {' '.join(missing)})", + } + + +def _whp_hypervisor_present() -> bool: + if sys.platform != "win32": + raise ScriptError("WHP qualification requires Windows") + try: + library = ctypes.WinDLL("WinHvPlatform.dll") + except OSError as error: + raise ScriptError(f"WinHvPlatform.dll is unavailable: {error}") from error + present = ctypes.c_int32(0) + written = ctypes.c_uint32(0) + result = int( + library.WHvGetCapability( + WHP_CAPABILITY_HYPERVISOR_PRESENT, + ctypes.byref(present), + ctypes.sizeof(present), + ctypes.byref(written), + ) + ) + if result != 0: + raise ScriptError( + f"WHvGetCapability failed with HRESULT 0x{result & 0xFFFFFFFF:08x}" + ) + return bool(present.value) + + +def check_backend(context: DoctorContext) -> CheckResult: + if context.backend == "whp": + if not host_is_windows(): + return CheckResult("H1", False, "WHP qualification requires Windows") + if not _whp_hypervisor_present(): + return CheckResult( + "H1", False, "the Windows Hypervisor Platform reports no hypervisor" + ) + return CheckResult("H1", True, "the Windows Hypervisor Platform is present") + if host_is_windows(): + return CheckResult( + "H1", False, f"{context.backend} qualification requires Linux" + ) + device = Path("/dev") / context.backend + if not device.exists(): + return CheckResult("H1", False, f"{device} does not exist") + if not os.access(device, os.R_OK | os.W_OK): + return CheckResult("H1", False, f"{device} is not readable and writable") + if context.backend == "kvm" and Path("/dev/mshv").exists(): + return CheckResult( + "H1", False, "/dev/mshv exists, so OpenVMM would select MSHV, not KVM" + ) + return CheckResult("H1", True, f"{device} is readable and writable") + + +def check_cpu(context: DoctorContext) -> CheckResult: + info = host_cpu(context) + signature = f"{info['vendor']} {info['family']}/{info['model']}/{info['stepping']}" + context.facts.update( + { + "cpu": signature, + "brand": info["brand"], + "microcode": info.get("microcode", "unknown"), + "os": info["os"], + "invariant_tsc": info["invariant_tsc"], + } + ) + problems: list[str] = [] + if context.fingerprint is None: + # Without OpenVMM, NVX's copy of OpenVMM's catalog maps the host. + try: + generation = cpu_generation( + info["vendor"], + int(info["family"]), + int(info["model"]), + int(info["stepping"]), + ) + except ValueError: + generation = None + name = generation.name if generation else "unknown" + profile = generation.profile_id if generation else "none" + if generation is None: + problems.append( + f"[E_PROFILE_HOST_UNKNOWN] {signature} is not a time ABI generation " + f"({describe_cpu_generations()})" + ) + profile_detail = "CPU profile not checked: qualification runs without OpenVMM" + else: + name, profile, profile_detail, profile_problems = _check_cpu_profile(context) + problems.extend(profile_problems) + context.facts.update({"generation": name, "profile": profile}) + detail = ( + f"generation={name} profile={profile} cpu={signature} " + f"microcode={info.get('microcode', 'unknown')} os={info['os']} " + f"invariant_tsc={info['invariant_tsc']} brand={info['brand']}; " + f"{profile_detail}" + ) + if problems: + return CheckResult("H2", False, "; ".join(problems) + f"; {detail}") + return CheckResult("H2", True, detail) + + +def parse_cpu_profile_line(line: str) -> dict[str, str]: + """Parse OpenVMM's ``NVX-CPU-PROFILE:`` line; ``detail`` is a Rust string.""" + text = line.strip().removeprefix(CPU_PROFILE_PREFIX) + head, quoted, detail = text.partition(' detail="') + fields = parse_fields(head) + if quoted: + fields["detail"] = RUST_ESCAPE.sub(_rust_unescape, detail.removesuffix('"')) + return fields + + +def _rust_unescape(match: re.Match[str]) -> str: + escape = match.group(1) + if escape.startswith("u{"): + return chr(int(escape[2:-1], 16)) + return {"n": "\n", "r": "\r", "t": "\t", "0": "\0"}.get(escape, escape) + + +def _same_profile_lineage(selected: str, expected: str) -> bool: + """Whether ``selected`` is a revision of the profile ``expected`` names.""" + lineage = re.escape(expected.rpartition(".v")[0]) + return re.fullmatch(rf"{lineage}\.v\d+", selected) is not None + + +def _check_cpu_profile(context: DoctorContext) -> tuple[str, str, str, list[str]]: + """Fingerprint the host with OpenVMM and check the profile that it + selects. Return the generation and the profile as OpenVMM reports them, + the detail, and the problems: OpenVMM's catalog, not NVX's copy, decides + which hosts it serves.""" + assert context.fingerprint is not None + if not context.openvmm.is_file(): + return ( + "unknown", + "none", + "CPU profile not checked", + [ + f"OpenVMM was not found at {context.openvmm}; H2 checks the CPU " + "profile with its --cpu-fingerprint tool (or pass --no-openvmm)" + ], + ) + context.fingerprint.parent.mkdir(parents=True, exist_ok=True) + environment = os.environ.copy() + environment["OPENVMM_LOG"] = "off" + completed = subprocess.run( + [ + os.fspath(context.openvmm), + "--hypervisor", + context.backend, + "--cpu-fingerprint", + os.fspath(context.fingerprint), + ], + capture_output=True, + text=True, + timeout=context.timeout, + env=environment, + check=False, + ) + output = f"{completed.stdout}\n{completed.stderr}" + line = next( + ( + line.strip() + for line in output.splitlines() + if line.strip().startswith(CPU_PROFILE_PREFIX) + ), + None, + ) + if line is None: + last = next( + (line.strip() for line in reversed(output.splitlines()) if line.strip()), + "no output", + ) + if "--cpu-fingerprint" in output and "unexpected argument" in output: + last = "this OpenVMM predates the --cpu-fingerprint tool" + return ( + "unknown", + "none", + "CPU profile not checked", + [ + f"OpenVMM's CPU fingerprint exited {completed.returncode} without " + f"an {CPU_PROFILE_PREFIX} line: {last}" + ], + ) + fields = parse_cpu_profile_line(line) + generation = fields.get("generation") or "none" + profile = fields.get("profile") or "none" + for name in ("profile_digest", "surface_digest"): + if fields.get(name, "none") != "none": + context.facts[name] = fields[name] + context.facts["cpu_fingerprint"] = os.fspath(context.fingerprint) + detail = ( + f"CPU profile check status={fields.get('status')} " + f"profile={profile} " + f"profile_digest={fields.get('profile_digest', 'none')} " + f"surface_digest={fields.get('surface_digest')} " + f"fingerprint={context.fingerprint}" + ) + problems: list[str] = [] + if completed.returncode != 0 or fields.get("status") != "pass": + code = fields.get("code", "") + prefix = f"[{code}] " if code.startswith("E_") else "" + problems.append( + f"{prefix}OpenVMM's CPU profile check failed (exit " + f"{completed.returncode}): {fields.get('detail', line)}" + ) + return generation, profile, detail, problems + if fields.get("backend") != context.backend: + problems.append(f"OpenVMM fingerprinted the {fields.get('backend')} backend") + unreported = [name for name in ("generation", "profile") if not fields.get(name)] + if unreported: + problems.append( + "OpenVMM's CPU profile check passed without reporting its " + + " and ".join(unreported) + ) + elif is_host_profile_id(profile): + problems.append( + f"OpenVMM reported host profile {profile}, which qualification " + "never accepts" + ) + elif not is_catalog_profile_id(profile): + problems.append( + f"OpenVMM reported CPU profile {profile}, which is not a catalog " + "profile ID (vendor.generation.vN)" + ) + elif profile.split(".")[1] != generation: + problems.append( + f"OpenVMM reported CPU profile {profile} of generation " + f"{profile.split('.')[1]} for generation {generation}" + ) + return generation, profile, detail, problems + + +def _guest_vcpus() -> int: + host_cpus = os.cpu_count() or 1 + return next(count for count in GUEST_VCPU_COUNTS if count <= host_cpus) + + +def check_openvmm_preflight(context: DoctorContext) -> CheckResult: + for path, description in ( + (context.openvmm, "OpenVMM"), + (context.kernel, "guest kernel"), + (context.initrd, "guest initramfs"), + ): + if not path.is_file(): + return CheckResult("H3", False, f"{description} was not found at {path}") + command = [ + os.fspath(context.openvmm), + "--machine", + "microvm", + "--processors", + str(_guest_vcpus()), + "--hypervisor", + context.backend, + "--kernel", + os.fspath(context.kernel), + "--initrd", + os.fspath(context.initrd), + *context.openvmm_args, + "--x-time-abi-verify", + ] + environment = os.environ.copy() + environment["OPENVMM_LOG"] = "off" + completed = subprocess.run( + command, + capture_output=True, + text=True, + timeout=context.timeout, + env=environment, + check=False, + ) + output = f"{completed.stdout}\n{completed.stderr}" + verify = next( + ( + line.strip() + for line in output.splitlines() + if line.strip().startswith(VERIFY_PREFIX) + ), + None, + ) + if verify is None: + last = next( + (line.strip() for line in reversed(output.splitlines()) if line.strip()), + "no output", + ) + if "unexpected argument '--x-time-abi-verify'" in output: + last = "this OpenVMM predates the time ABI's --x-time-abi-verify mode" + return CheckResult( + "H3", + False, + f"OpenVMM verification exited {completed.returncode} without a " + f"verification line: {last}", + ) + fields = parse_fields(verify.removeprefix(VERIFY_PREFIX).strip()) + if fields.get("status") != "ok" or completed.returncode != 0: + code = fields.get("code", "none") + prefix = f"[{code}] " if code.startswith("E_") else "" + return CheckResult( + "H3", + False, + f"{prefix}OpenVMM verification failed (exit {completed.returncode}): " + f"{fields.get('detail', verify)}", + ) + problems: list[str] = [] + if fields.get("backend") != context.backend: + problems.append(f"OpenVMM verified backend {fields.get('backend')}") + for name in ("tsc_hz", "native_tsc_hz"): + try: + rate = int(fields.get(name, "")) + if not MIN_TSC_HZ <= rate <= MAX_TSC_HZ: + problems.append(f"[E_TSC_RATE_IMPLAUSIBLE] {name}={rate}") + except ValueError: + problems.append(f"{name}={fields.get(name)!r} is not an integer") + expected_lapic = LAPIC_HZ[context.backend] + if fields.get("lapic_hz") != str(expected_lapic): + problems.append( + f"[E_LAPIC_RATE_MISMATCH] lapic_hz={fields.get('lapic_hz')} is not " + f"{expected_lapic}" + ) + expected_profile = context.facts.get("profile", "none") + selected_profile = fields.get("cpu_profile") or "none" + # A missing, host, or malformed profile fails even when H2 doesn't run + # with H3. Which generations have profiles is OpenVMM's to decide. + if not is_catalog_profile_id(selected_profile): + problems.append( + "OpenVMM verified no catalog CPU profile: " + f"cpu_profile={selected_profile}" + + ( + " (a host profile, to which --cpu-profile auto falls back on a " + "CPU that no built-in profile serves, and which qualification " + "never accepts)" + if is_host_profile_id(selected_profile) + else "" + ) + ) + elif expected_profile != "none" and not _same_profile_lineage( + selected_profile, expected_profile + ): + problems.append( + f"OpenVMM selected CPU profile {selected_profile}, but H2 maps the " + f"host to {expected_profile}" + ) + detail = " ".join(f"{name}={value}" for name, value in fields.items()) + context.facts.update( + { + "tsc_hz": fields.get("tsc_hz", ""), + "native_tsc_hz": fields.get("native_tsc_hz", ""), + "lapic_hz": fields.get("lapic_hz", ""), + "profile": selected_profile, + "route": fields.get("msr_route", ""), + "sync": fields.get("sync", ""), + } + ) + if problems: + return CheckResult("H3", False, "; ".join(problems) + f"; {detail}") + return CheckResult("H3", True, detail) + + +def _rate_points( + samples: Sequence[Mapping[str, str]], clock: Mapping[str, str], role: str +) -> list[tuple[int, int, float]]: + """Return each sample's (TSC, bracket midpoint in half clock ticks, + uncertainty in clock ticks: half the bracket plus half the resolution).""" + half_resolution = float(clock["resolution_ns"]) * int(clock["hz"]) / 2e9 + return [ + ( + int(sample[f"{role}_tsc"]), + 2 * int(sample[f"{role}_clock"]) + int(sample[f"{role}_bracket"]), + int(sample[f"{role}_bracket"]) / 2 + half_resolution, + ) + for sample in samples + ] + + +def _window_rate(points: Sequence[tuple[int, int, float]], clock_hz: int) -> float: + seconds = (points[-1][1] - points[0][1]) / (2 * clock_hz) + if seconds <= 0: + raise ScriptError("the host clock did not advance between samples") + return (points[-1][0] - points[0][0]) / seconds + + +def check_rate(context: DoctorContext) -> CheckResult: + clocksource = "" + if not host_is_windows(): + try: + clocksource = host_clocksource() + except OSError as error: + # The host clocksource is recorded as evidence only; an + # unreadable file must not fail the check. Record it explicitly + # and still run the TSC rate/stability measurements below. + clocksource = f"unknown ({error})" + context.facts["host_clocksource"] = clocksource + schedule = context.schedule + # The probe sleeps between samples for the whole window, 120 s on the + # qualification schedule, so the timeout covers the window as well. + records = run_probe( + context, + "rate", + "--samples", + str(schedule.rate_samples), + "--interval-ms", + str(schedule.rate_interval_ms), + duration=(schedule.rate_samples - 1) * schedule.rate_interval_ms / 1000, + ) + clocks = {fields.get("role"): fields for kind, fields in records if kind == "clock"} + for role in ("rate", "stability"): + if role not in clocks: + raise ScriptError(f"host time probe printed no {role} clock") + samples = [fields for kind, fields in records if kind == "sample"] + if len(samples) != schedule.rate_samples: + raise ScriptError( + f"host time probe printed {len(samples)} samples, not " + f"{schedule.rate_samples}" + ) + # The rate clock, which time synchronization disciplines, gives the true + # rate. The stability clock, which it never steers, shows whether the TSC + # keeps its rate across idle: chrony's frequency updates move + # CLOCK_MONOTONIC's rate by several ppm between seconds. + rate_clock, stability_clock = clocks["rate"], clocks["stability"] + measured = _window_rate( + _rate_points(samples, rate_clock, "rate"), int(rate_clock["hz"]) + ) + clock_hz = int(stability_clock["hz"]) + points = _rate_points(samples, stability_clock, "stability") + rates: list[float] = [] + uncertainties_ppm: list[float] = [] + for (tsc0, mid0, error0), (tsc1, mid1, error1) in itertools.pairwise(points): + if mid1 <= mid0: + raise ScriptError("the host clock did not advance between samples") + seconds = (mid1 - mid0) / (2 * clock_hz) + rates.append((tsc1 - tsc0) / seconds) + uncertainties_ppm.append((error0 + error1) / clock_hz / seconds * 1e6) + stable_rate = _window_rate(points, clock_hz) + agreement_ppm = (max(rates) - min(rates)) / stable_rate * 1e6 + uncertainty_ppm = max(uncertainties_ppm) + context.facts["measured_tsc_hz"] = f"{measured:.0f}" + context.facts["rate_agreement_ppm"] = f"{agreement_ppm:.3f}" + context.facts["rate_uncertainty_ppm"] = f"{uncertainty_ppm:.3f}" + detail = ( + f"tsc_hz={measured:.0f} against {rate_clock['name']} from {len(samples)} " + f"samples {schedule.rate_interval_ms / 1000:g} s apart; against " + f"{stability_clock['name']} (resolution " + f"{float(stability_clock['resolution_ns']):g} ns) the interval rates agree " + f"within {agreement_ppm:.3f} ppm, largest interval uncertainty " + f"{uncertainty_ppm:.3f} ppm" + ) + problems: list[str] = [] + if not MIN_TSC_HZ <= measured <= MAX_TSC_HZ: + problems.append(f"[E_TSC_RATE_IMPLAUSIBLE] measured {measured:.0f} Hz") + if uncertainty_ppm > RATE_UNCERTAINTY_PPM: + problems.append( + f"the measurement is inconclusive: an interval's uncertainty is " + f"{uncertainty_ppm:.3f} ppm > {RATE_UNCERTAINTY_PPM} ppm" + ) + if agreement_ppm > RATE_AGREEMENT_PPM: + problems.append( + f"the TSC rate is unstable: interval rates differ by " + f"{agreement_ppm:.3f} ppm > {RATE_AGREEMENT_PPM} ppm" + ) + declared = context.facts.get("native_tsc_hz") or context.facts.get("tsc_hz") + if declared is None: + # validate-runner runs before the OpenVMM binary is available; the + # VM jobs run H3 with H4 to compare against the backend's rate. + detail += "; not compared with the backend's rate (H3 did not run)" + else: + deviation_ppm = abs(measured - int(declared)) / int(declared) * 1e6 + context.facts["rate_deviation_ppm"] = f"{deviation_ppm:.1f}" + detail += f"; {deviation_ppm:.1f} ppm from the backend's {declared} Hz" + if deviation_ppm > RATE_TOLERANCE_PPM: + problems.append( + f"the backend rate is {deviation_ppm:.1f} ppm from the measured " + f"rate (limit {RATE_TOLERANCE_PPM:g} ppm)" + ) + if clocksource: + detail += f"; host clocksource {clocksource} (evidence)" + if problems: + return CheckResult("H4", False, "; ".join(problems) + f"; {detail}") + return CheckResult("H4", True, detail) + + +def check_host_skew(context: DoctorContext) -> CheckResult: + arguments = ["skew", "--duration-ms", str(SKEW_PAIR_DURATION_MS)] + rate = context.facts.get("measured_tsc_hz") or context.facts.get("tsc_hz") + if rate is not None: + arguments.extend(("--tsc-hz", rate)) + summary = _probe_record(run_probe(context, *arguments), "skew") + offset = int(summary["max_abs_offset_ns"]) + uncertainty = int(summary["max_uncertainty_ns"]) + stalled = int(summary["stalled_pairs"]) + context.facts["host_skew_ns"] = str(offset) + detail = ( + f"pairs={summary['pairs']} cpus={summary['cpus']} max_abs_offset_ns={offset} " + f"max_uncertainty_ns={uncertainty}" + ) + problems: list[str] = [] + if offset > WARP_BOUND_NS: + problems.append(f"max_abs_offset_ns={offset} exceeds {WARP_BOUND_NS}") + if stalled: + problems.append(f"{stalled} CPU pair(s) stalled") + if summary.get("conclusive") != "1": + problems.append( + f"the measurement is inconclusive (uncertainty {uncertainty} ns)" + ) + if problems: + return CheckResult("H5", False, "; ".join(problems) + f"; {detail}") + return CheckResult("H5", True, detail) + + +def _warp_guest( + context: DoctorContext, vcpus: int, gaps: Sequence[str] +) -> tuple[dict[str, str], list[dict[str, str]]]: + """Boot a guest, check its boot marker, and run a warp schedule in it.""" + command = workload_boot_command( + context.openvmm, + context.backend, + context.kernel, + context.initrd, + GUEST_MEMORY_MIB, + "quiet loglevel=0", + processors=vcpus, + ) + command.extend(context.openvmm_args) + result = run_guest_script( + command, + warp_probe_script(gaps) + "nvx-exit 0\n", + WARP_PROBE_COMPLETION_MARKER, + timeout=context.timeout, + time_abi_status=True, + ) + monitor = TimeAbiMonitor(command) + monitor.feed(result["text"].encode()) + monitor.finish() + monitor.require_status("the warp probe") + boot = monitor.require_boot("the warp probe", online_cpus=vcpus) + rounds = check_warp_probe( + result["text"], cpus=vcpus, context="H6", rounds=warp_rounds(vcpus, gaps) + ) + return boot, rounds + + +def check_guest_warp(context: DoctorContext) -> CheckResult: + for path, description in ( + (context.openvmm, "OpenVMM"), + (context.kernel, "guest kernel"), + (context.initrd, "guest initramfs"), + ): + if not path.is_file(): + return CheckResult("H6", False, f"{description} was not found at {path}") + vcpus = _guest_vcpus() + # The largest guest runs the schedule, then a 1-vCPU guest runs the + # probe once. + guests: list[tuple[int, tuple[str, ...]]] = [] + if vcpus > 1: + guests.append((vcpus, context.schedule.warp_gaps)) + guests.append((1, ())) + boots: list[dict[str, str]] = [] + rounds: list[dict[str, str]] = [] + runs: list[str] = [] + for count, count_gaps in guests: + try: + boot, count_rounds = _warp_guest(context, count, count_gaps) + except (RuntimeError, TimeAbiFailure) as error: + first = str(error).splitlines()[0] + return CheckResult("H6", False, f"vcpus={count}: {first}") + boots.append(boot) + rounds.extend(count_rounds) + # The boot check's wall time and, where the guest reports it, its CPU + # time, which the spec budgets; both are evidence only. + cpu_us = f" boot_cpu_us={boot['cpu_us']}" if "cpu_us" in boot else "" + runs.append( + f"vcpus={count} rounds={len(count_rounds)} " + f"idle_gaps_s={','.join(count_gaps) or 'none'} " + f"boot_elapsed_us={boot.get('elapsed_us', '?')}{cpu_us}" + ) + + def worst(name: str) -> int: + return max(int(warp[name]) for warp in rounds) + + context.facts["guest_warp_ns"] = str(worst("max_abs_offset_ns")) + context.facts.setdefault("tsc_hz", boots[0]["tsc_hz"]) + context.facts.setdefault("lapic_hz", boots[0]["lapic_hz"]) + return CheckResult( + "H6", + True, + "; ".join(runs) + f"; max_backward_ns={worst('max_backward_ns')} " + f"max_abs_offset_ns={worst('max_abs_offset_ns')} " + f"max_uncertainty_ns={worst('max_uncertainty_ns')}", + ) + + +def _linux_clock_state() -> tuple[int, int]: + """Return adjtimex's clock state and status without changing the clock.""" + try: + libc = ctypes.CDLL(None, use_errno=True) + buffer = ctypes.create_string_buffer(256) + state = int(libc.adjtimex(buffer)) + except (AttributeError, OSError) as error: + raise ScriptError( + f"{UTC_UNVERIFIED}: adjtimex is unavailable: {error}" + ) from error + if state < 0: + raise ScriptError( + f"{UTC_UNVERIFIED}: adjtimex failed: {os.strerror(ctypes.get_errno())}" + ) + # struct timex: unsigned modes, then the long offset, freq, maxerror, and + # esterror fields, then the int status. + status = int.from_bytes(buffer.raw[40:44], sys.byteorder) + return state, status + + +def _windows_time_source() -> tuple[bool, str]: + try: + completed = subprocess.run( + ["w32tm", "/query", "/status"], + capture_output=True, + text=True, + timeout=30, + check=False, + ) + except (OSError, subprocess.SubprocessError, ValueError) as error: + raise ScriptError(f"{UTC_UNVERIFIED}: w32tm /query /status: {error}") from error + if completed.returncode != 0: + raise ScriptError( + f"{UTC_UNVERIFIED}: w32tm /query /status failed: " + + (completed.stdout.strip() or completed.stderr.strip()) + ) + fields: dict[str, str] = {} + for line in completed.stdout.splitlines(): + name, separator, value = line.partition(":") + if separator: + fields[name.strip()] = value.strip() + source = fields.get("Source", "") + leap = fields.get("Leap Indicator", "") + # w32tm prints its labels in the display language. Output without both + # English labels and a 0-3 leap indicator names no source, so the check + # fails rather than certify an unverified one. + if not source or not leap.startswith(("0", "1", "2", "3")): + raise ScriptError( + f"{UTC_UNVERIFIED}: w32tm /query /status reported " + f"source={source or 'none'} leap_indicator={leap or 'none'}" + ) + unsynchronized = ( + leap.startswith("3") + or "Local CMOS Clock" in source + or "Free-running System Clock" in source + ) + return not unsynchronized, f"source={source} leap_indicator={leap}" + + +def check_utc(context: DoctorContext) -> CheckResult: + del context + if host_is_windows(): + synchronized, detail = _windows_time_source() + if not synchronized: + return CheckResult("H7", False, f"host UTC is not synchronized: {detail}") + return CheckResult("H7", True, detail) + state, status = _linux_clock_state() + detail = f"adjtimex state={state} status=0x{status:04x}" + if state > ADJTIMEX_TIME_ERROR: + return CheckResult( + "H7", False, f"{UTC_UNVERIFIED}: unknown clock state; {detail}" + ) + if state == ADJTIMEX_TIME_ERROR or status & ADJTIMEX_STA_UNSYNC: + return CheckResult( + "H7", False, f"host UTC is not synchronized (STA_UNSYNC); {detail}" + ) + return CheckResult("H7", True, detail) + + +CHECKS: Mapping[str, Callable[[DoctorContext], CheckResult]] = { + "H1": check_backend, + "H2": check_cpu, + "H3": check_openvmm_preflight, + "H4": check_rate, + "H5": check_host_skew, + "H6": check_guest_warp, + "H7": check_utc, +} + + +def run_checks(context: DoctorContext, checks: Sequence[str]) -> list[CheckResult]: + """Run checks in spec order; an exception fails only its own check.""" + results: list[CheckResult] = [] + for check in CHECK_IDS: + if check not in checks: + continue + try: + result = CHECKS[check](context) + except ( + ArithmeticError, + IndexError, + KeyError, + OSError, + RuntimeError, + subprocess.SubprocessError, + TypeError, + ValueError, + ) as error: + first = str(error).splitlines()[0] if str(error) else type(error).__name__ + result = CheckResult(check, False, first) + results.append(result) + print(result.line(), flush=True) + return results + + +def summary_markdown( + backend: str, results: Sequence[CheckResult], facts: Mapping[str, str] +) -> str: + status = "passed" if all(result.passed for result in results) else "failed" + lines = [ + f"### Time ABI host qualification ({backend}): {status}", + "", + "| Check | Name | Status | Detail |", + "| --- | --- | --- | --- |", + ] + for result in results: + lines.append( + f"| {result.check} | {CHECK_TITLES[result.check]} | " + f"{'pass' if result.passed else '**fail**'} | " + f"{_escape_markdown(result.detail)} |" + ) + reported = ( + ("generation", "Generation"), + ("cpu", "CPU"), + ("microcode", "Microcode"), + ("os", "Host OS"), + ("profile", "Profile"), + ("profile_digest", "Profile digest"), + ("surface_digest", "CPU surface digest"), + ("tsc_hz", "Declared TSC rate (Hz)"), + ("lapic_hz", "LAPIC rate (Hz)"), + ("measured_tsc_hz", "Measured TSC rate (Hz)"), + ("rate_deviation_ppm", "Rate deviation from the backend (ppm)"), + ("rate_agreement_ppm", "Interval rate agreement (ppm)"), + ("rate_uncertainty_ppm", "Largest interval uncertainty (ppm)"), + ("host_skew_ns", "Host skew (ns)"), + ("guest_warp_ns", "Guest warp offset (ns)"), + ("invariant_tsc", "Host invariant TSC (evidence)"), + ("host_clocksource", "Host clocksource (evidence)"), + ) + facts_line = " ยท ".join( + f"{label}: `{facts[name]}`" for name, label in reported if name in facts + ) + if facts_line: + lines.extend(("", facts_line)) + return "\n".join(lines) + "\n\n" + + +def run(args: argparse.Namespace) -> int: + if args.backend not in OPENVMM_TEST_BACKENDS: + raise ScriptError(f"unsupported backend {args.backend!r}") + checks = tuple(dict.fromkeys(args.checks or CHECK_IDS)) + if args.no_openvmm: + booting = [check for check in checks if check in OPENVMM_CHECKS] + if booting: + raise ScriptError( + f"--no-openvmm cannot run {' and '.join(booting)}, which boot OpenVMM" + ) + probe_directory = args.probe_dir or default_probe_directory() + fingerprint: Path | None = None + if not args.no_openvmm: + fingerprint = args.cpu_fingerprint or ( + probe_directory / f"nvx-cpu-fingerprint-{args.backend}.json" + ) + context = DoctorContext( + backend=args.backend, + openvmm=args.openvmm or openvmm_binary_path(), + kernel=args.kernel or artifact_path(KernelBuildConstants.BINARY_NAME), + initrd=args.initrd or artifact_path(AlpineBuildConstants.INITRAMFS_NAME), + openvmm_args=tuple(args.openvmm_arg), + probe_directory=probe_directory, + timeout=args.timeout, + fingerprint=fingerprint, + schedule=CI_SCHEDULE if args.ci_schedule else QUALIFICATION_SCHEDULE, + ) + results = run_checks(context, checks) + failed = [result.check for result in results if not result.passed] + generation = context.facts.get("generation") + profile = context.facts.get("profile") + print( + f"Time ABI host qualification on {args.backend}: " + + ("passed" if not failed else f"failed ({', '.join(failed)})") + + (f"; generation {generation}" if generation else "") + + (f", profile {profile}" if profile else ""), + flush=True, + ) + if args.summary is not None: + with args.summary.open("a", encoding="utf-8") as summary: + summary.write(summary_markdown(args.backend, results, context.facts)) + return 1 if failed else 0 + + +def default_probe_directory() -> Path: + tool_cache = os.environ.get("RUNNER_TOOL_CACHE") + if tool_cache: + return Path(tool_cache) / "nvx-host-time-probe" + return BuildConstants.BUILD_DIR / "host-time-probe" + + +def configure_parser(parser: argparse.ArgumentParser) -> None: + parser.description = ( + "Qualify this host for the NVX time ABI (doc/design/time-abi.md, " + "'Host qualification')." + ) + parser.add_argument("--backend", choices=OPENVMM_TEST_BACKENDS, required=True) + parser.add_argument( + "--checks", + nargs="+", + choices=CHECK_IDS, + metavar="ID", + help="checks to run, in spec order (default: H1 to H7)", + ) + parser.add_argument( + "--openvmm", type=Path, help="OpenVMM binary for H2, H3, and H6" + ) + parser.add_argument("--kernel", type=Path, help="guest kernel for H3 and H6") + parser.add_argument("--initrd", type=Path, help="guest initramfs for H3 and H6") + openvmm = parser.add_mutually_exclusive_group() + openvmm.add_argument( + "--cpu-fingerprint", + type=Path, + help="where H2 writes OpenVMM's CPU fingerprint " + "(default: nvx-cpu-fingerprint-.json in the probe directory)", + ) + openvmm.add_argument( + "--no-openvmm", + action="store_true", + help="qualify without an OpenVMM binary: H2 checks the CPU identity and " + "generation but not the CPU profile, and H3 and H6 cannot run", + ) + parser.add_argument( + "--ci-schedule", + action="store_true", + help="run H4 and H6 on CI's short schedules: H4 takes 3 samples 1 s " + "apart instead of 13 samples 10 s apart, and H6 runs the warp probe " + "twice instead of five times", + ) + parser.add_argument( + "--probe-dir", + type=Path, + help="cache directory for the host probe binary " + "(default: $RUNNER_TOOL_CACHE or build/)", + ) + parser.add_argument( + "--summary", + type=Path, + help="append a Markdown summary, for example $GITHUB_STEP_SUMMARY", + ) + parser.add_argument( + "--timeout", + type=float, + default=120.0, + help="seconds allowed for each probe or guest, on top of H4's sampling " + "window (default: 120)", + ) + parser.add_argument( + "--openvmm-arg", + action="append", + default=[], + help=argparse.SUPPRESS, + ) + parser.set_defaults(handler=run) diff --git a/nvx507-test-time-abi.py b/nvx507-test-time-abi.py new file mode 100644 index 000000000..15441fb2f --- /dev/null +++ b/nvx507-test-time-abi.py @@ -0,0 +1,3086 @@ +"""Unit tests for the harness-side NVX time ABI checks and host qualification.""" + +from __future__ import annotations + +import contextlib +import io +import json +import queue +import re +import shutil +import socket +import subprocess +import sys +import tempfile +import threading +import unittest +from pathlib import Path +from unittest.mock import patch + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from nvx_tools import benchmark, doctor, openvmm_process, time_abi # noqa: E402 +from nvx_tools.time_abi import TimeAbiFailure, TimeAbiMonitor # noqa: E402 + +BOOT_LINE = ( + "NVX-TIME-ABI: v=1 phase=boot status=ok cpus=4 tsc_hz=2194804000 " + "lapic_hz=1000000000 generation=0 elapsed_us=1873\n" +) +RESTORE_LINE = ( + "NVX-TIME-ABI: v=1 phase=restore status=ok cpus=4 tsc_hz=2194804000 " + "lapic_hz=1000000000 generation=2 elapsed_us=41\n" +) +# nvx-time status prints the last check's marker, then this line. +RUNTIME_LINE = ( + "NVX-TIME-ABI: v=1 phase=runtime status=unsynchronized generation=0 " + "discontinuities=0 offset_ns=0 uncertainty_ns=0 rejected_samples=0 " + "last_sample_error=none\n" +) +STATUS_OK = "NVX-TIME-STATUS-EXIT status=0\n" +KVM_BOOT = ["openvmm", "--hypervisor", "kvm", "--kernel", "vmlinux"] +MSHV_RESTORE = ["openvmm", "--hypervisor", "mshv", "--restore-snapshot", "snap"] +# OpenVMM's openvmm_entry fatal_error_message, after a guest prompt. +FATAL_DETAIL = "[E_TSC_SYNC_UNSUPPORTED] failed to launch vm worker" +FATAL = f"~ # fatal error: {FATAL_DETAIL}\r\n\r\nCaused by:\r\n".encode() + + +def warp_output( + *, + pairs: int = 6, + backward: int = 0, + offset: int = 15, + verdict: str = "PASS", + stalled: int = 0, + conclusive: int = 1, +) -> str: + return ( + "pair 0-1: warp_iterations=881154 warps=0 max_backward_ns=0\n" + f"NVX-TIME-PROBE warp max_backward_cycles=0 max_backward_ns={backward} " + f"max_abs_offset_ns={offset} pairs={pairs} verdict={verdict} bound_ns=1000\r\n" + "NVX-TIME-PROBE warp-detail max_abs_offset_cycles=33.0 max_uncertainty_ns=127 " + "max_skew_bound_ns=130 total_warps=0 inconsistent_pairs=0 " + f"stalled_pairs={stalled} cpus=0-3 duration_ms=100 tsc_hz=2194804000 " + f"tsc_hz_source=kmsg-detected-processor conclusive={conclusive}\n" + ) + + +def write_openvmm_profiles( + checkout: Path, *generations: time_abi.CpuGeneration, revision: int = 1 +) -> Path: + """Write ``generations`` as revision ``revision`` of the pinned CPU + profiles of an OpenVMM checkout at ``checkout``, with the fields that NVX + reads, and return the checkout.""" + directory = checkout / time_abi.OPENVMM_CPU_PROFILES + directory.mkdir(parents=True, exist_ok=True) + for generation in generations: + profile_id = ( + f"{time_abi.PROFILE_VENDORS[generation.vendor]}.{generation.name}" + f".v{revision}" + ) + document = { + "id": profile_id, + "vendor": generation.vendor, + "generation": { + "name": generation.name, + "cpus": [ + { + "family": cpu.family, + "model": cpu.model, + "steppings": [cpu.steppings.start, cpu.steppings.stop - 1], + } + for cpu in generation.cpus + ], + }, + } + (directory / f"{profile_id}.json").write_text( + json.dumps(document), encoding="utf-8" + ) + return checkout + + +class FieldParsingTests(unittest.TestCase): + def test_parses_plain_and_quoted_fields_with_guest_escapes(self): + fields = time_abi.parse_fields( + 'v=1 code=G_TSC_WARP detail="say \\"hi\\" \\\\ \\x07 end" phase=boot' + ) + self.assertEqual( + fields, + { + "v": "1", + "code": "G_TSC_WARP", + "detail": 'say "hi" \\ \x07 end', + "phase": "boot", + }, + ) + + def test_rejects_malformed_fields(self): + for text in ( + "novalue", + "=x", + 'detail="unterminated', + 'detail="bad \\q escape"', + 'detail="\\x4"', + "v=1 v=2", + ): + with self.subTest(text=text), self.assertRaises(ValueError): + time_abi.parse_fields(text) + + def test_parses_markers_and_violations(self): + marker = time_abi.parse_marker(BOOT_LINE) + self.assertIsNotNone(marker) + assert marker is not None + self.assertEqual(marker["phase"], "boot") + self.assertEqual(marker["tsc_hz"], "2194804000") + self.assertIsNone(time_abi.parse_marker("NVX-TIME-REPORT: v=1 phase=boot")) + with self.assertRaises(ValueError): + time_abi.parse_marker("NVX-TIME-ABI: v=1 status=ok") + violation = time_abi.parse_violation( + "noise NVX-TIME-ABI-VIOLATION: v=1 code=G_RCU_STALL source=watcher " + 'phase=runtime generation=1 boottime_ns=5 detail="rcu: INFO: x"\r' + ) + self.assertIsNotNone(violation) + assert violation is not None + self.assertEqual(violation["code"], "G_RCU_STALL") + self.assertEqual(violation["detail"], "rcu: INFO: x") + + def test_describes_only_time_abi_exit_statuses(self): + self.assertIn("conformance", time_abi.describe_exit_status(193) or "") + self.assertIn("runtime violation", time_abi.describe_exit_status(194) or "") + self.assertIn("restore repair", time_abi.describe_exit_status(195) or "") + for status in (None, 0, 1, 37, 192, 196, 255): + self.assertIsNone(time_abi.describe_exit_status(status)) + + def test_maps_cpu_generations(self): + def name(model: int, stepping: int, vendor: str = "GenuineIntel"): + generation = time_abi.cpu_generation(vendor, 6, model, stepping) + return generation.name if generation else None + + self.assertEqual(name(85, 4), "skylake-sp") + self.assertEqual(name(106, 6), "icelake-sp") + self.assertEqual(name(207, 2), "emeraldrapids") + # Alder Lake-S and Alder Lake-P and -H share one profile. + self.assertEqual(name(151, 2), "alderlake") + self.assertEqual(name(154, 3), "alderlake") + # Cascade Lake and Cooper Lake share model 85 but have no profile. + self.assertIsNone(name(85, 7)) + self.assertIsNone(name(85, 11)) + self.assertIsNone(name(143, 8)) + # Tiger Lake and Raptor Lake have no profile either. + self.assertIsNone(name(140, 1)) + self.assertIsNone(name(183, 1)) + self.assertIsNone(name(1, 1, "AuthenticAMD")) + # Milan's profile serves every stepping of AMD's family 25 model 1, + # Milan-X's 2 included, Genoa's every stepping of model 17, and + # Turin's every stepping of family 26 model 2, and none of them + # another vendor's CPU with its signature. + for family, model, expected in ( + (25, 1, "milan"), + (25, 17, "genoa"), + (26, 2, "turin"), + ): + for stepping in (0, 1, 2, 15): + generation = time_abi.cpu_generation( + "AuthenticAMD", family, model, stepping + ) + assert generation is not None + self.assertEqual(generation.name, expected) + # Vermeer and Raphael (Ryzen), Turin Dense, and Granite Ridge have no + # profile, and neither has another vendor's CPU with the signature of + # an AMD generation. + for vendor, family, model in ( + ("AuthenticAMD", 25, 33), + ("AuthenticAMD", 25, 97), + ("AuthenticAMD", 26, 17), + ("AuthenticAMD", 26, 68), + ("GenuineIntel", 25, 1), + ("HygonGenuine", 25, 1), + ("GenuineIntel", 26, 2), + ): + with self.subTest(vendor=vendor, family=family, model=model): + self.assertIsNone(time_abi.cpu_generation(vendor, family, model, 1)) + + def test_names_the_catalog_profiles(self): + # One profile per generation serves every backend. + self.assertEqual( + [generation.profile_id for generation in time_abi.CPU_GENERATIONS], + [ + "intel.skylake-sp.v1", + "intel.icelake-sp.v1", + "intel.emeraldrapids.v1", + "intel.alderlake.v1", + "amd.milan.v1", + "amd.genoa.v1", + "amd.turin.v1", + ], + ) + self.assertEqual( + time_abi.describe_cpu_generations(), + "skylake-sp 6/85 steppings 0-4, icelake-sp 6/106, emeraldrapids 6/207, " + "alderlake 6/151 and 6/154, milan 25/1, genoa 25/17, turin 26/2", + ) + + def test_the_catalog_copy_matches_openvmm_pinned_profiles(self): + # NVX's copy must not drift from the profiles that OpenVMM pins. They + # are read from the gitlink's commit, not from the submodule's working + # tree, which can hold another revision: a self-hosted CI runner keeps + # an earlier job's OpenVMM checkout until the job checks out its own. + root = Path(__file__).resolve().parents[1] + submodule = root / "openvmm" + if not (submodule / ".git").exists(): + self.skipTest("the OpenVMM submodule is not initialized") + + def git(directory: Path, *arguments: str) -> str | None: + result = subprocess.run( + ["git", "-C", str(directory), *arguments], + capture_output=True, + encoding="utf-8", + check=False, + ) + return result.stdout if result.returncode == 0 else None + + pin = (git(root, "rev-parse", ":openvmm") or "").strip() + if not pin or git(submodule, "cat-file", "-e", f"{pin}^{{commit}}") is None: + self.skipTest( + f"the OpenVMM submodule lacks its pinned revision {pin or '(unknown)'}" + ) + listing = git( + submodule, + "ls-tree", + "-z", + "--name-only", + pin, + "--", + "vmm_core/cpu_profile/profiles/", + ) + assert listing is not None + pinned: dict[str, tuple[str, str, tuple[tuple[int, int, range], ...]]] = {} + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + checkout = Path(temporary.name) + profiles = checkout / time_abi.OPENVMM_CPU_PROFILES + profiles.mkdir(parents=True) + for path in sorted(listing.split("\0")): + if not path.endswith(".json"): + continue + content = git(submodule, "show", f"{pin}:{path}") + assert content is not None + (profiles / Path(path).name).write_text(content, encoding="utf-8") + document = json.loads(content) + generation = document["generation"] + # Keyed by profile ID, so that a second pinned revision of a + # generation fails the comparison instead of replacing the first. + pinned[document["id"]] = ( + generation["name"], + document["vendor"], + tuple( + ( + cpu["family"], + cpu["model"], + range(cpu["steppings"][0], cpu["steppings"][1] + 1), + ) + for cpu in generation["cpus"] + ), + ) + self.assertTrue(pinned) + copy = { + generation.profile_id: ( + generation.name, + generation.vendor, + tuple( + (cpu.family, cpu.model, cpu.steppings) for cpu in generation.cpus + ), + ) + for generation in time_abi.CPU_GENERATIONS + } + self.assertEqual(copy, pinned) + # benchmark reads the same generations from these documents, as it + # reads them from the OpenVMM checkout that it builds. + self.assertEqual( + set(time_abi.openvmm_cpu_generations(checkout)), + set(time_abi.CPU_GENERATIONS), + ) + + def test_reads_the_cpu_profiles_of_an_openvmm_checkout(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + directory = Path(temporary.name) + icelake, alderlake = time_abi.CPU_GENERATIONS[1], time_abi.CPU_GENERATIONS[3] + granite_rapids = time_abi.CpuGeneration( + "graniterapids", "GenuineIntel", (time_abi.CpuModel(6, 173),) + ) + checkout = write_openvmm_profiles( + directory / "openvmm", icelake, alderlake, granite_rapids + ) + # A second revision of a generation's profile adds no generation. + write_openvmm_profiles(checkout, icelake, revision=2) + generations = time_abi.openvmm_cpu_generations(checkout) + self.assertEqual(set(generations), {icelake, alderlake, granite_rapids}) + self.assertEqual(len(generations), 3) + # A CPU that only the checkout's profiles cover is benchmarked with + # them, and not with NVX's copy of the catalog. + granite_rapids_host = time_abi.HostCpu("GenuineIntel", 6, 173, 1) + self.assertIsNone( + time_abi.benchmark_cpu_profile_refusal( + granite_rapids_host, "openvmm", "kvm", generations + ) + ) + self.assertIsNotNone( + time_abi.benchmark_cpu_profile_refusal( + granite_rapids_host, "openvmm", "kvm" + ) + ) + # A checkout that pins no profile, and a malformed profile, are errors, + # naming the checkout or the profile. + with self.assertRaisesRegex(ValueError, "pins no CPU profile"): + time_abi.openvmm_cpu_generations(directory / "empty") + valid = { + "vendor": "GenuineIntel", + "generation": { + "name": "icelake-sp", + "cpus": [{"family": 6, "model": 106, "steppings": [0, 15]}], + }, + } + cases: tuple[tuple[str, object, str], ...] = ( + ("not-json", None, "cannot read the CPU profile .*not-json.json"), + ("list", [], "it is not a JSON object"), + ("no-generation", {"vendor": "GenuineIntel"}, "names no vendor"), + ( + "no-cpus", + {**valid, "generation": {"name": "icelake-sp", "cpus": []}}, + "names no CPU models", + ), + ( + "stepping-pair", + { + **valid, + "generation": { + "name": "icelake-sp", + "cpus": [{"family": 6, "model": 106, "steppings": [3]}], + }, + }, + "not a first and a last", + ), + ( + "stepping-order", + { + **valid, + "generation": { + "name": "icelake-sp", + "cpus": [{"family": 6, "model": 106, "steppings": [9, 3]}], + }, + }, + "steppings 9 to 3 are not a range", + ), + ( + "boolean-family", + { + **valid, + "generation": { + "name": "icelake-sp", + "cpus": [{"family": True, "model": 106, "steppings": [0, 1]}], + }, + }, + "its family is not an integer", + ), + ) + for name, document, message in cases: + with self.subTest(name=name): + malformed = directory / name + profiles = malformed / time_abi.OPENVMM_CPU_PROFILES + profiles.mkdir(parents=True) + (profiles / f"{name}.json").write_text( + "{" if document is None else json.dumps(document), + encoding="utf-8", + ) + with self.assertRaisesRegex(ValueError, message): + time_abi.openvmm_cpu_generations(malformed) + + def test_tells_catalog_profiles_from_host_profiles(self): + for profile_id, catalog, host in ( + ("intel.icelake-sp.v1", True, False), + ("intel.raptorlake.v2", True, False), + ("amd.milan.v1", True, False), + ("intel.host.v1", False, True), + ("amd.host.v1", False, True), + # Host profile IDs take the revision syntax of every profile ID. + ("intel.host.v0", False, False), + ("interim.host.kvm.v1", False, False), + ("intel.icelake-sp.v2-rc", False, False), + ("intel.icelake-sp.v0", False, False), + ("none", False, False), + ("", False, False), + ): + with self.subTest(profile_id=profile_id): + self.assertEqual(time_abi.is_catalog_profile_id(profile_id), catalog) + self.assertEqual(time_abi.is_host_profile_id(profile_id), host) + + def test_guides_hosts_that_no_built_in_profile_serves(self): + def guide(cpu_profile: str | None, host: time_abi.HostCpu | None) -> str | None: + return time_abi.host_cpu_profile_guidance( + cpu_profile, host, "/nvx/openvmm", "kvm" + ) + + tiger_lake = time_abi.HostCpu( + "GenuineIntel", 6, 140, 1, "11th Gen Intel(R) Core(TM) i7-1185G7" + ) + guidance = guide(None, tiger_lake) + assert guidance is not None + lines = guidance.splitlines() + self.assertTrue(all(line.startswith("nvx: ") for line in lines), guidance) + # auto falls back on the CPU, so the guidance says what auto does there, + # not which code ended this run, which run cannot read. + self.assertEqual( + lines[0], + "nvx: no built-in CPU profile serves this host's CPU, GenuineIntel " + "6/140/1 (11th Gen Intel(R) Core(TM) i7-1185G7), so --cpu-profile " + "auto, the default, falls back on it to a CPU profile derived from " + "this host, which OpenVMM warns about with NVX-CPU-PROFILE-FALLBACK:; " + f"the built-in profiles cover {time_abi.describe_cpu_generations()}.", + ) + self.assertIn("OpenVMM's error above names the cause", lines[1]) + self.assertNotIn("E_PROFILE_HOST_UNKNOWN", guidance) + self.assertNotIn("rerun with", guidance) + # It asks for the CPU's fingerprint through the CPU profile request + # form, which the link prefills with the CPU. + self.assertEqual( + lines[2:], + [ + "nvx: Help NVX add a built-in profile for this CPU: run", + "nvx: /nvx/openvmm --hypervisor kvm --cpu-fingerprint " + "fingerprint.json", + "nvx: and attach fingerprint.json to a CPU profile request:", + f"nvx: {time_abi.cpu_profile_request_url(tiger_lake)}", + "nvx: https://github.com/microsoft/nvx/issues/408 tracks CPU " + "profiles for more Intel CPUs.", + ], + ) + self.assertEqual(guidance, guide("auto", tiger_lake)) + # A host profile that fails on this CPU fails for a reason that + # OpenVMM's own error explains. + self.assertIsNone(guide("host", tiger_lake)) + + # Host profiles serve AMD CPUs too: an AMD CPU without a built-in + # profile, such as Raphael (Ryzen 7000), falls back as an Intel one + # does, with the issue that tracks profiles for more AMD CPUs. + raphael = time_abi.HostCpu("AuthenticAMD", 25, 97, 2) + guidance = guide(None, raphael) + assert guidance is not None + self.assertIn( + "this host's CPU, AuthenticAMD 25/97/2, so --cpu-profile auto, the " + "default, falls back on it", + guidance, + ) + self.assertIn(time_abi.cpu_profile_request_url(raphael), guidance) + self.assertIn( + "https://github.com/microsoft/nvx/issues/409 tracks CPU profiles for " + "more AMD CPUs.", + guidance, + ) + self.assertNotIn("issues/408", guidance) + self.assertIsNone(guide("host", raphael)) + # Another vendor's CPU fails with either request, and gets no request + # form and no issue: the time ABI does not serve its CPUs. + hygon = time_abi.HostCpu("HygonGenuine", 24, 0, 1) + guidance = guide("auto", hygon) + assert guidance is not None + self.assertIn( + "this host's CPU, HygonGenuine 24/0/1, so a cold boot with " + "--cpu-profile auto, the default, fails on it with " + "E_PROFILE_HOST_UNKNOWN;", + guidance, + ) + self.assertIn("host CPU profiles serve only Intel and AMD CPUs", guidance) + self.assertNotIn("falls back", guidance) + self.assertNotIn("github.com", guidance) + self.assertEqual(guidance, guide("host", hygon)) + + # A CPU that a built-in profile serves, an explicit profile, and an + # unknown CPU get none. + alder_lake = time_abi.HostCpu("GenuineIntel", 6, 154, 3) + milan = time_abi.HostCpu("AuthenticAMD", 25, 1, 1) + genoa = time_abi.HostCpu("AuthenticAMD", 25, 17, 1) + turin = time_abi.HostCpu("AuthenticAMD", 26, 2, 1) + for cpu_profile, host in ( + (None, alder_lake), + ("host", alder_lake), + (None, milan), + ("host", milan), + (None, genoa), + (None, turin), + ("host", turin), + ("intel.alderlake.v1", tiger_lake), + ("intel.alderlake.v1", raphael), + ("intel.skylake-sp.v1", milan), + ("intel.skylake-sp.v1", hygon), + (None, None), + ): + with self.subTest(cpu_profile=cpu_profile, host=host): + self.assertIsNone(guide(cpu_profile, host)) + + def test_names_cpus_and_prefills_cpu_profile_requests(self): + granite_rapids = time_abi.HostCpu( + "GenuineIntel", 6, 173, 1, "Intel(R) Xeon(R) 6973P-C" + ) + self.assertEqual(granite_rapids.signature(), "GenuineIntel 6/173/1") + self.assertEqual( + granite_rapids.describe(), "GenuineIntel 6/173/1 (Intel(R) Xeon(R) 6973P-C)" + ) + # The link that OpenVMM's fallback warning gives the same CPU + # (openvmm_core's time_abi tests). + self.assertEqual( + time_abi.cpu_profile_request_url(granite_rapids), + "https://github.com/microsoft/nvx/issues/new?template=cpu-profile.yml" + "&title=CPU%20profile%3A%20GenuineIntel%206%2F173%2F1%20%28Intel%28R%29" + "%20Xeon%28R%29%206973P-C%29&signature=GenuineIntel%206%2F173%2F1" + "&cpu=Intel%28R%29%20Xeon%28R%29%206973P-C", + ) + # A CPU without a brand string prefills only its signature. + brandless = time_abi.HostCpu("GenuineIntel", 6, 173, 1) + self.assertEqual(brandless.describe(), "GenuineIntel 6/173/1") + self.assertEqual( + time_abi.cpu_profile_request_url(brandless), + f"{time_abi.CPU_PROFILE_REQUEST_URL}&title=CPU%20profile%3A%20" + "GenuineIntel%206%2F173%2F1&signature=GenuineIntel%206%2F173%2F1", + ) + self.assertIn( + "%26%3D%3F%23%25%2B", + time_abi.cpu_profile_request_url( + time_abi.HostCpu("GenuineIntel", 6, 173, 1, "&=?#%+") + ), + ) + + def test_parses_the_cpu_profile_fallback_warning(self): + digest = "ab" * 32 + warning = ( + " 0.012345s WARN openvmm_core::worker::dispatch::time_abi: " + "NVX-CPU-PROFILE-FALLBACK: no built-in CPU profile serves this host's " + "CPU, GenuineIntel 6/173/1 (Intel(R) Xeon(R) 6973P-C), so --cpu-profile " + f"auto fell back to intel.host.v1 (sha256:{digest}), a profile derived " + "from this host's kvm backend for development.\n" + "It is not pinned: a microcode, firmware, hypervisor, or OS update can " + "change it.\n" + ) + self.assertEqual(time_abi.parse_cpu_profile_fallback(warning), "intel.host.v1") + self.assertEqual( + time_abi.parse_cpu_profile_fallback( + warning.replace("intel.host.v1", "amd.host.v1") + ), + "amd.host.v1", + ) + self.assertIsNone(time_abi.parse_cpu_profile_fallback("no warning\n")) + for output, message in ( + (warning + warning, "logged 2 NVX-CPU-PROFILE-FALLBACK: warnings"), + (warning.replace("intel.host.v1", "intel.icelake-sp.v1"), "malformed"), + (warning.replace(f"sha256:{digest}", "sha256:ab"), "malformed"), + (warning.replace("fell back to", "selected"), "malformed"), + ): + with self.subTest(message=message): + with self.assertRaisesRegex(ValueError, message): + time_abi.parse_cpu_profile_fallback(output) + + def test_benchmark_refuses_cpus_that_no_built_in_profile_serves(self): + granite_rapids = time_abi.HostCpu( + "GenuineIntel", 6, 173, 1, "Intel(R) Xeon(R) 6973P-C" + ) + refusal = time_abi.benchmark_cpu_profile_refusal( + granite_rapids, "openvmm", "kvm" + ) + assert refusal is not None + self.assertTrue( + refusal.startswith( + "benchmark refuses this host: no built-in CPU profile serves its CPU, " + "GenuineIntel 6/173/1 (Intel(R) Xeon(R) 6973P-C), so --cpu-profile " + "auto would fall back to a CPU profile derived from this host, whose " + "every cold boot fingerprints the hypervisor first" + ), + refusal, + ) + self.assertIn(time_abi.describe_cpu_generations(), refusal) + self.assertIn( + "\n openvmm --hypervisor kvm --cpu-fingerprint fingerprint.json\n", refusal + ) + self.assertTrue( + refusal.endswith(time_abi.cpu_profile_request_url(granite_rapids)), refusal + ) + # Another vendor's CPU cannot boot at all, and gets no request form. + hygon = time_abi.benchmark_cpu_profile_refusal( + time_abi.HostCpu("HygonGenuine", 24, 0, 1), "openvmm", "kvm" + ) + assert hygon is not None + self.assertIn("host CPU profiles serve only Intel and AMD CPUs", hygon) + self.assertIn("E_PROFILE_HOST_UNKNOWN", hygon) + self.assertNotIn("github.com", hygon) + # A CPU that cannot be identified is refused too: OpenVMM, which reads + # the CPU itself, could fall back on it. + unknown = time_abi.benchmark_cpu_profile_refusal(None, "openvmm", "kvm") + assert unknown is not None + self.assertTrue( + unknown.startswith( + "benchmark refuses this host: it cannot identify the host's CPU" + ), + unknown, + ) + # Its vendor is unknown too, so the refusal qualifies the fallback by + # vendor. + self.assertIn( + "On an Intel or AMD CPU that none serves, --cpu-profile auto falls " + "back to a CPU profile", + unknown, + ) + self.assertIn( + "on another vendor's CPU, every cold boot fails with " + f"{time_abi.PROFILE_HOST_UNKNOWN}", + unknown, + ) + # A CPU that a built-in profile serves is benchmarked. + for host in ( + time_abi.HostCpu("GenuineIntel", 6, 106, 6), + time_abi.HostCpu("AuthenticAMD", 25, 1, 1), + ): + with self.subTest(host=host): + self.assertIsNone( + time_abi.benchmark_cpu_profile_refusal(host, "openvmm", "whp") + ) + # The profiles that decide are those of the OpenVMM that benchmark + # builds: one that adds a profile for a CPU serves it, and one that + # lacks a profile refuses its CPU, listing its own profiles. + graniterapids = time_abi.CpuGeneration( + "graniterapids", "GenuineIntel", (time_abi.CpuModel(6, 173),) + ) + self.assertIsNone( + time_abi.benchmark_cpu_profile_refusal( + granite_rapids, + "openvmm", + "kvm", + (*time_abi.CPU_GENERATIONS, graniterapids), + ) + ) + icelake = time_abi.HostCpu("GenuineIntel", 6, 106, 6) + refusal = time_abi.benchmark_cpu_profile_refusal( + icelake, "openvmm", "kvm", (graniterapids,) + ) + assert refusal is not None + self.assertIn( + "no built-in CPU profile serves its CPU, GenuineIntel 6/106/6", refusal + ) + self.assertIn("The built-in profiles cover graniterapids 6/173.\n", refusal) + # A CPU in more than one generation fails every cold boot, because + # auto rejects it as a catalog defect without falling back, as OpenVMM + # does; revisions of one generation's profile do not make it ambiguous. + icelake_x = time_abi.CpuGeneration( + "icelake-x", "GenuineIntel", (time_abi.CpuModel(6, 106),) + ) + ambiguous = time_abi.benchmark_cpu_profile_refusal( + icelake, "openvmm", "kvm", (*time_abi.CPU_GENERATIONS, icelake_x) + ) + assert ambiguous is not None + self.assertTrue( + ambiguous.startswith( + "benchmark refuses this host: its CPU, GenuineIntel 6/106/6, is in " + "more than one generation of the built-in CPU profiles (icelake-sp, " + "icelake-x), so every cold boot with --cpu-profile auto fails with " + f"{time_abi.PROFILE_HOST_UNKNOWN}." + ), + ambiguous, + ) + self.assertNotIn("github.com", ambiguous) + revised = time_abi.CpuGeneration( + "icelake-sp", + "GenuineIntel", + (time_abi.CpuModel(6, 106), time_abi.CpuModel(6, 108)), + ) + self.assertIsNone( + time_abi.benchmark_cpu_profile_refusal( + icelake, "openvmm", "kvm", (*time_abi.CPU_GENERATIONS, revised) + ) + ) + + def test_cpu_profile_requests_match_openvmm_and_the_issue_form(self): + # OpenVMM's fallback warning and NVX link to the same form, which must + # define the text fields that the links prefill, and NVX matches the + # warning's marker. OpenVMM's constants are read from the gitlink's + # commit, as the catalog copy's comparison reads its profiles. + root = Path(__file__).resolve().parents[1] + form = (root / ".github" / "ISSUE_TEMPLATE" / "cpu-profile.yml").read_text( + encoding="utf-8" + ) + fields = re.findall(r"^ - type: (\w+)\n id: ([\w-]+)$", form, re.MULTILINE) + for prefilled in ("cpu", "signature"): + self.assertIn(("input", prefilled), fields) + self.assertRegex(form, r"(?m)^name: CPU profile request$") + self.assertEqual( + time_abi.CPU_PROFILE_REQUEST_URL.rpartition("template=")[2], + "cpu-profile.yml", + ) + + submodule = root / "openvmm" + if not (submodule / ".git").exists(): + self.skipTest("the OpenVMM submodule is not initialized") + pin = subprocess.run( + ["git", "-C", str(root), "rev-parse", ":openvmm"], + capture_output=True, + encoding="utf-8", + check=False, + ).stdout.strip() + source = subprocess.run( + [ + "git", + "-C", + str(submodule), + "show", + f"{pin}:openvmm/openvmm_core/src/worker/dispatch/time_abi.rs", + ], + capture_output=True, + encoding="utf-8", + check=False, + ) + if not pin or source.returncode != 0: + self.skipTest( + f"the OpenVMM submodule lacks its pinned revision {pin or '(unknown)'}" + ) + constants = dict( + re.findall( + r'const (FALLBACK_MARKER|CPU_PROFILE_REQUEST_URL): &str =\s*"([^"]*)";', + source.stdout, + ) + ) + self.assertEqual( + constants, + { + "FALLBACK_MARKER": time_abi.CPU_PROFILE_FALLBACK_MARKER, + "CPU_PROFILE_REQUEST_URL": time_abi.CPU_PROFILE_REQUEST_URL, + }, + ) + for key in ("&title=", "&signature=", "&cpu="): + self.assertIn(key, source.stdout) + + def test_computes_the_checks_cpu_time_budget(self): + # The spec's final budgets: a base plus an increment per additional + # CPU, one budget per backend and phase. + for backend in ("kvm", "mshv", "whp"): + self.assertEqual( + set(time_abi.CHECK_CPU_BUDGET_US[backend]), + {"boot", "capture", "restore"}, + ) + self.assertEqual(time_abi.check_cpu_budget_us("kvm", "boot", 1), 6_000) + self.assertEqual(time_abi.check_cpu_budget_us("kvm", "capture", 1), 1_500) + self.assertEqual(time_abi.check_cpu_budget_us("kvm", "capture", 8), 4_300) + self.assertEqual(time_abi.check_cpu_budget_us("kvm", "restore", 1), 7_000) + self.assertEqual(time_abi.check_cpu_budget_us("kvm", "restore", 8), 21_000) + # The spec's binding MSHV boot cell: 4 vCPUs (4.94 ms measured). + self.assertEqual(time_abi.check_cpu_budget_us("mshv", "boot", 4), 6_000) + self.assertEqual(time_abi.check_cpu_budget_us("mshv", "restore", 8), 6_000) + self.assertEqual(time_abi.check_cpu_budget_us("whp", "restore", 1), 35_000) + # The spec's binding WHP cells: boot at 2 vCPUs (21.84 ms measured on an + # 8573C runner), capture at 1 (0.986 ms), and restore at 4 (42.20 ms). + self.assertEqual(time_abi.check_cpu_budget_us("whp", "boot", 2), 26_500) + self.assertEqual(time_abi.check_cpu_budget_us("whp", "capture", 1), 1_200) + self.assertEqual(time_abi.check_cpu_budget_us("whp", "capture", 2), 1_600) + self.assertEqual(time_abi.check_cpu_budget_us("whp", "restore", 4), 53_000) + self.assertIsNone(time_abi.check_cpu_budget_us("hvf", "boot", 1)) + self.assertIsNone(time_abi.check_cpu_budget_us("kvm", "runtime", 1)) + self.assertIsNone(time_abi.check_cpu_budget_us("kvm", "boot", 0)) + + +class MonitorTests(unittest.TestCase): + def test_classifies_commands(self): + monitor = TimeAbiMonitor(KVM_BOOT) + self.assertEqual(monitor.backend, "kvm") + self.assertTrue(monitor.cold_boot) + restore = TimeAbiMonitor(MSHV_RESTORE) + self.assertEqual(restore.backend, "mshv") + self.assertFalse(restore.cold_boot) + + def test_records_markers_across_chunk_boundaries(self): + monitor = TimeAbiMonitor(KVM_BOOT) + data = ( + "[ 0.1] early\n" + + " ALPINE-MICROVM-BOOT-OK: 3.22.1\n" + + BOOT_LINE + + RUNTIME_LINE + + "NVX-TIME-ABI: v=1 phase=restore status=ok cpus=4 tsc_hz=2194804000 " + + "lapic_hz=1000000000 generation=1 elapsed_us=12\n" + ).encode() + for index in range(0, len(data), 7): + monitor.feed(data[index : index + 7]) + self.assertTrue(monitor.guest_booted) + self.assertEqual(monitor.require_boot("test", online_cpus=4)["cpus"], "4") + assert monitor.runtime is not None + self.assertEqual(monitor.runtime["status"], "unsynchronized") + self.assertEqual(monitor.restores[0]["generation"], "1") + # The runtime line is the discipline's state and never fails a run. + TimeAbiMonitor(KVM_BOOT).feed( + RUNTIME_LINE.replace("unsynchronized", "uncertain").encode() + ) + + def test_reports_a_check_that_stayed_pending(self): + pending = ( + "NVX-TIME-ABI: v=1 phase=restore status=pending cpus=4 " + "tsc_hz=2194804000 lapic_hz=200000000 generation=1\n" + ) + with self.assertRaisesRegex( + TimeAbiFailure, + "restore check was still pending when nvx-time status stopped " + "waiting after 30 s", + ): + TimeAbiMonitor(MSHV_RESTORE).feed(pending.encode() + STATUS_OK.encode()) + + def test_fails_fast_on_violation_events_and_failed_checks(self): + monitor = TimeAbiMonitor(KVM_BOOT) + with self.assertRaisesRegex(TimeAbiFailure, "violation G_TSC_UNSTABLE"): + monitor.feed( + b"NVX-TIME-ABI-VIOLATION: v=1 code=G_TSC_UNSTABLE source=watcher " + b'phase=runtime generation=0 boottime_ns=1 detail="Marking"\n' + ) + self.assertIn("G_TSC_UNSTABLE", monitor.violation or "") + with self.assertRaisesRegex(TimeAbiFailure, "boot check C9 failed: token"): + TimeAbiMonitor(KVM_BOOT).feed( + b'NVX-TIME-ABI: v=1 phase=boot status=fail check=C9 detail="token"\n' + ) + with self.assertRaisesRegex(TimeAbiFailure, "malformed"): + TimeAbiMonitor(KVM_BOOT).feed(b"NVX-TIME-ABI: v=1 phase=boot\n") + with self.assertRaisesRegex(TimeAbiFailure, "unknown time ABI marker status"): + TimeAbiMonitor(KVM_BOOT).feed( + b"NVX-TIME-ABI: v=1 phase=boot status=maybe\n" + ) + + def test_rejects_a_report_only_boot_when_a_status_query_exits(self): + monitor = TimeAbiMonitor(KVM_BOOT) + monitor.feed( + b"NVX-TIME-REPORT-VIOLATION: v=1 code=G_TSC_WARP source=watcher " + b'phase=boot generation=0 boottime_ns=1 detail="x"\n' + b"NVX-GUEST-BOOT-OK: alpine\n" + ) + self.assertTrue(monitor.report_only) + # A report-only guest whose check failed exits 1; the failure names + # report-only mode, not the exit status. + report = ( + b"NVX-TIME-REPORT: v=1 phase=boot status=fail cpus=1 tsc_hz=2194804000 " + b"lapic_hz=1000000000 generation=0 elapsed_us=9 failures=3\n" + ) + with self.assertRaisesRegex(TimeAbiFailure, "report-only"): + monitor.feed(report + b"NVX-TIME-STATUS-EXIT status=1\n") + restored = TimeAbiMonitor(MSHV_RESTORE) + with self.assertRaisesRegex( + TimeAbiFailure, + "restore marker before nvx-time status exited; .*report-only", + ): + restored.feed( + report.replace(b"phase=boot", b"phase=restore") + + b"NVX-TIME-STATUS-EXIT status=0\n" + ) + + def test_status_queries_require_the_line_of_the_launch_kind(self): + # A quiet guest reaches its shell without printing any time ABI line. + quiet = TimeAbiMonitor(KVM_BOOT) + quiet.feed(b" ALPINE-MICROVM-BOOT-OK: 3.22.1\nNVX-GUEST-BOOT-OK: alpine\n") + self.assertTrue(quiet.guest_booted) + with self.assertRaisesRegex( + TimeAbiFailure, + "nvx-time status did not finish before the scenario; the guest never " + "reported its time ABI state", + ): + quiet.require_status("the scenario") + # The shell's echo of the query is not its exit line. + quiet.feed( + b'/ # /sbin/nvx-time status; echo "NVX-TIME-STATUS-EXIT status=$?"\n' + ) + self.assertEqual(quiet.status_queries, 0) + with self.assertRaisesRegex( + TimeAbiFailure, + "did not report the NVX-TIME-ABI boot marker before nvx-time status " + "exited; the guest image or OpenVMM does not implement time ABI v1", + ): + quiet.feed(b"NVX-TIME-STATUS-EXIT status=0\r\n") + + booted = TimeAbiMonitor(KVM_BOOT) + booted.feed((BOOT_LINE + RUNTIME_LINE + STATUS_OK).encode()) + self.assertEqual(booted.status_queries, 1) + booted.require_status("the scenario") + self.assertEqual(booted.require_boot("x", online_cpus=4)["cpus"], "4") + mismatched = BOOT_LINE.replace("lapic_hz=1000000000", "lapic_hz=200000000") + monitor = TimeAbiMonitor(KVM_BOOT) + monitor.feed(mismatched.encode()) + with self.assertRaisesRegex(TimeAbiFailure, "is not the kvm rate"): + monitor.feed(b"NVX-TIME-STATUS-EXIT status=0\n") + + with self.assertRaisesRegex( + TimeAbiFailure, + "did not report an NVX-TIME-ABI restore marker before nvx-time status " + "exited", + ): + TimeAbiMonitor(MSHV_RESTORE).feed(b"NVX-TIME-STATUS-EXIT status=0\n") + restored = TimeAbiMonitor(MSHV_RESTORE) + restored.feed( + RESTORE_LINE.replace("lapic_hz=1000000000", "lapic_hz=200000000").encode() + + RUNTIME_LINE.replace("generation=0", "generation=2").encode() + + b"NVX-TIME-STATUS-EXIT status=0\n" + ) + self.assertEqual( + restored.require_restore("x", online_cpus=4)["generation"], "2" + ) + # After a restore, status prints every recorded phase, oldest first, + # each with the values from when its check ran: the boot and capture + # lines keep their boot-time CPU count, and only the newest line counts. + activated = TimeAbiMonitor(MSHV_RESTORE) + mshv = "lapic_hz=200000000" + activated.feed( + ( + BOOT_LINE.replace("cpus=4", "cpus=1").replace( + "lapic_hz=1000000000", mshv + ) + + BOOT_LINE.replace("phase=boot", "phase=capture") + .replace("cpus=4", "cpus=1") + .replace("lapic_hz=1000000000", mshv) + + RESTORE_LINE.replace("generation=2", "generation=1").replace( + "lapic_hz=1000000000", mshv + ) + + RUNTIME_LINE.replace("generation=0", "generation=1") + + STATUS_OK + ).encode() + ) + self.assertEqual( + activated.require_restore("x", online_cpus=4, generation=1)["cpus"], "4" + ) + assert activated.boot is not None + self.assertEqual(activated.boot["cpus"], "1") + # A launch that neither boots a kernel nor restores has no line to require. + TimeAbiMonitor(["openvmm"]).feed(b"NVX-TIME-STATUS-EXIT status=0\n") + + with self.assertRaisesRegex( + TimeAbiFailure, "exited 2: the guest image has no status subcommand" + ): + # An old nvx-time's usage text can end without a newline. + TimeAbiMonitor(KVM_BOOT).feed( + b"usage: nvx-time boot|exhaustiveNVX-TIME-STATUS-EXIT status=2\n" + ) + with self.assertRaisesRegex(TimeAbiFailure, "nvx-time status exited 1$"): + TimeAbiMonitor(KVM_BOOT).feed( + BOOT_LINE.encode() + b"NVX-TIME-STATUS-EXIT status=1\n" + ) + + def test_builds_the_status_query(self): + self.assertEqual( + time_abi.status_script(), + '/sbin/nvx-time status; echo "NVX-TIME-STATUS-EXIT status=$?"\n', + ) + # The harness waits longer than the guest, which reports pending checks. + self.assertGreater( + time_abi.STATUS_TIMEOUT_SECONDS, time_abi.STATUS_WAIT_SECONDS + ) + + def test_validates_boot_marker_fields_against_the_backend(self): + cases = ( + ("v=1", "v=2", "version"), + ("lapic_hz=1000000000", "lapic_hz=200000000", "kvm rate"), + ("tsc_hz=2194804000", "tsc_hz=400000000", "outside 500 MHz"), + ("tsc_hz=2194804000", "tsc_hz=fast", "not an integer"), + ("generation=0", "generation=3", "not 0 at cold boot"), + ("cpus=4", "cpus=2", "cpus=2 is not 4"), + ) + for valid, invalid, message in cases: + fields = time_abi.parse_marker(BOOT_LINE.replace(valid, invalid)) + assert fields is not None + with self.subTest(invalid=invalid): + with self.assertRaisesRegex(TimeAbiFailure, message): + time_abi.validate_boot_marker(fields, backend="kvm", online_cpus=4) + mshv = time_abi.parse_marker( + BOOT_LINE.replace("lapic_hz=1000000000", "lapic_hz=200000000") + ) + assert mshv is not None + time_abi.validate_boot_marker(mshv, backend="mshv") + + def test_validates_restore_marker_fields_against_the_backend(self): + restore = time_abi.parse_marker(RESTORE_LINE) + assert restore is not None + time_abi.validate_restore_marker(restore, backend="kvm", online_cpus=4) + cases = ( + ("generation=2", "generation=0", "generation=0 is not 1 or later"), + ("generation=2", "generation=x", "generation=x is not 1 or later"), + ("phase=restore", "phase=boot", "not a passing restore check"), + ("lapic_hz=1000000000", "lapic_hz=200000000", "kvm rate"), + ("cpus=4", "cpus=8", "cpus=8 is not 4"), + ) + for valid, invalid, message in cases: + fields = time_abi.parse_marker(RESTORE_LINE.replace(valid, invalid)) + assert fields is not None + with self.subTest(invalid=invalid): + with self.assertRaisesRegex( + TimeAbiFailure, f"restore marker is invalid: .*{message}" + ): + time_abi.validate_restore_marker( + fields, backend="kvm", online_cpus=4 + ) + # A restore of a snapshot captured at generation 0 carries generation 1. + with self.assertRaisesRegex(TimeAbiFailure, "generation=2 is not 1$"): + time_abi.validate_restore_marker(restore, backend="kvm", generation=1) + time_abi.validate_restore_marker(restore, backend="kvm", generation=2) + + def test_classifies_time_abi_exit_statuses_with_the_violation_event(self): + monitor = TimeAbiMonitor(MSHV_RESTORE) + monitor.check_exit(0) + monitor.check_exit(None) + with self.assertRaisesRegex( + TimeAbiFailure, "status 195 \\(restore repair\\): no NVX-TIME-ABI" + ): + monitor.check_exit(195) + try: + monitor.feed( + b"NVX-TIME-ABI-VIOLATION: v=1 code=G_REPAIR_SAMPLE source=repair " + b'phase=restore generation=1 boottime_ns=9 detail="epsilon"' + ) + monitor.finish() + except TimeAbiFailure: + pass + with self.assertRaisesRegex(TimeAbiFailure, "status 195.*G_REPAIR_SAMPLE"): + monitor.check_exit(195) + + def test_bounds_an_unterminated_line(self): + monitor = TimeAbiMonitor(KVM_BOOT) + monitor.feed(b"x" * (time_abi._MAX_PENDING_LINE + 10)) # pyright: ignore[reportPrivateUsage] + monitor.feed(b"\n" + BOOT_LINE.encode()) + self.assertIsNotNone(monitor.boot) + + def test_records_the_first_fatal_time_abi_error_for_exit_errors(self): + monitor = TimeAbiMonitor(KVM_BOOT) + monitor.feed(b"fatal error: failed to open the kernel\r\n") + self.assertIsNone(monitor.fatal) + self.assertEqual(str(monitor.exit_error(1)), "OpenVMM exited with status 1") + # A guest prompt without a newline can precede OpenVMM's line. + monitor.feed(FATAL + b" 0: [E_TSC_SYNC_UNSUPPORTED] no synchronized set\r\n") + monitor.feed(b"fatal error: [E_TEST_HOOK] later\r\n") + self.assertEqual(monitor.fatal, FATAL_DETAIL) + # The line only explains the exit; it never changes its classification. + monitor.check_exit(1) + self.assertEqual( + str(monitor.exit_error(1, "snapshot source", "during teardown")), + f"snapshot source exited with status 1 during teardown: {FATAL_DETAIL}", + ) + + +class WarpProbeTests(unittest.TestCase): + def test_accepts_conclusive_passing_measurements(self): + results = time_abi.check_warp_probe(warp_output(), cpus=4, context="test") + self.assertEqual(results[0]["max_abs_offset_ns"], "15") + single = time_abi.check_warp_probe( + warp_output(pairs=0, offset=0), cpus=1, context="one vCPU" + ) + self.assertEqual(single[0]["pairs"], "0") + self.assertEqual( + time_abi.warp_probe_command(), + "/sbin/nvx-time-probe warp --bound-ns 1000", + ) + self.assertTrue(time_abi.warp_probe_command(cpus="0-3").endswith("--cpus 0-3")) + + def test_rejects_skew_stalls_and_inconclusive_measurements(self): + cases = ( + (warp_output(backward=1001), "max_backward_ns=1001"), + (warp_output(offset=1500, verdict="FAIL"), "max_abs_offset_ns=1500"), + (warp_output(stalled=1, verdict="FAIL"), "1 CPU pair"), + (warp_output(conclusive=0), "inconclusive"), + (warp_output(pairs=3), "measured 3 CPU pairs instead of 6"), + (warp_output(verdict="FAIL"), "verdict=FAIL"), + ("nothing here\n", "printed no summary"), + ( + warp_output().split("NVX-TIME-PROBE warp-detail")[0], + "1 summaries and 0 detail", + ), + ( + "NVX-TIME-PROBE warp pairs=6\nNVX-TIME-PROBE warp-detail x=1\n", + "incomplete", + ), + ('NVX-TIME-PROBE warp detail="\n', "malformed"), + ) + for text, message in cases: + with self.subTest(message=message): + with self.assertRaisesRegex(TimeAbiFailure, message): + time_abi.check_warp_probe(text, cpus=4, context="restore 4") + + def test_counts_rounds_and_names_the_failing_round(self): + self.assertEqual( + [time_abi.warp_rounds(cpus) for cpus in (1, 2, 8)], + [1, time_abi.WARP_PROBE_ROUNDS, time_abi.WARP_PROBE_ROUNDS], + ) + self.assertEqual(time_abi.CI_WARP_GAPS, ("1",)) + self.assertEqual( + [ + time_abi.warp_rounds(cpus, time_abi.QUALIFICATION_WARP_GAPS) + for cpus in (1, 8) + ], + [1, 5], + ) + with self.assertRaisesRegex(ValueError, "invalid warp probe idle gaps"): + time_abi.warp_probe_script(("1; reboot",)) + two = warp_output() + warp_output(offset=30) + results = time_abi.check_warp_probe(two, cpus=4, context="boot", rounds=2) + self.assertEqual( + [result["max_abs_offset_ns"] for result in results], ["15", "30"] + ) + with self.assertRaisesRegex(TimeAbiFailure, "ran 1 rounds instead of 2"): + time_abi.check_warp_probe(warp_output(), cpus=4, context="boot", rounds=2) + with self.assertRaisesRegex( + TimeAbiFailure, "failed in round 2: max_backward_ns=2000" + ): + time_abi.check_warp_probe( + warp_output() + warp_output(backward=2000, verdict="FAIL"), + cpus=4, + context="boot", + ) + + def test_warp_fragment_idles_between_rounds_and_powers_off_on_failure(self): + shell = shutil.which("sh") + if shell is None and (git := shutil.which("git")) is not None: + candidate = Path(git).parent.parent / "bin" / "sh.exe" + shell = str(candidate) if candidate.is_file() else None + if shell is None: + self.skipTest("POSIX shell is unavailable") + fragment = ( + time_abi.warp_probe_script() + .replace(time_abi.WARP_PROBE_PATH, "probe") + .replace("nvx-exit", "nvx_exit") + ) + + def run_fragment( + fragment: str, online: str, failing_round: int = 0 + ) -> subprocess.CompletedProcess[str]: + assert shell is not None + return subprocess.run( + [shell], + input=( + f"cat() {{ printf '%s\\n' '{online}'; }}\n" + "calls=0\n" + "probe() {\n" + " calls=$((calls + 1))\n" + ' printf "probe %s\\n" "$*"\n' + f' [ "$calls" -ne {failing_round} ]\n' + "}\n" + 'sleep() { printf "SLEEP %s\\n" "$1"; }\n' + 'nvx_exit() { printf "NVX-EXIT %s\\n" "$1"; }\n' + + fragment + + "echo after\n" + ), + text=True, + capture_output=True, + timeout=10, + check=False, + ) + + for online, failing_round, calls, sleeps in ( + ("0-3", 0, 2, 1), + ("0-3", 2, 2, 1), + ("0-3", 1, 1, 0), + ("0", 0, 1, 0), + ): + with self.subTest(online=online, failing_round=failing_round): + result = run_fragment(fragment, online, failing_round) + lines = result.stdout.splitlines() + self.assertEqual( + lines.count(f"probe warp --bound-ns 1000 --cpus {online}"), + calls, + result.stdout + result.stderr, + ) + self.assertEqual( + lines.count(f"SLEEP {time_abi.WARP_PROBE_IDLE_SECONDS}"), sleeps + ) + if failing_round: + self.assertEqual(result.returncode, 97) + self.assertIn( + f"NVX-WARP-PROBE-FAIL status=1 round={failing_round}", lines + ) + self.assertIn("NVX-EXIT 97", lines) + self.assertNotIn("NVX-WARP-PROBE-OK", lines) + self.assertNotIn("after", lines) + else: + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("NVX-WARP-PROBE-OK", lines) + self.assertIn("after", lines) + # H6's qualification schedule idles for each gap in order. + qualification = fragment.replace( + 'warp_gaps="1"', + f'warp_gaps="{" ".join(time_abi.QUALIFICATION_WARP_GAPS)}"', + ) + self.assertEqual( + qualification, + time_abi.warp_probe_script(time_abi.QUALIFICATION_WARP_GAPS) + .replace(time_abi.WARP_PROBE_PATH, "probe") + .replace("nvx-exit", "nvx_exit"), + ) + result = run_fragment(qualification, "0-7") + probe = "probe warp --bound-ns 1000 --cpus 0-7" + self.assertEqual( + result.stdout.splitlines(), + [ + probe, + "SLEEP 0.1", + probe, + "SLEEP 1", + probe, + "SLEEP 5", + probe, + "SLEEP 1", + probe, + "NVX-WARP-PROBE-OK", + "after", + ], + result.stderr, + ) + + +VIOLATION = ( + b"NVX-TIME-ABI-VIOLATION: v=1 code=G_RCU_STALL source=watcher phase=runtime " + b'generation=1 boottime_ns=99 detail="rcu: INFO: rcu_preempt self-detected stall"\n' +) + + +class FakeProcess: + pid = 123 + + def __init__(self, status: int) -> None: + self.status = status + self.exited = False + self.returncode: int | None = None + + def poll(self) -> int | None: + if self.exited: + self.returncode = self.status + return self.returncode + + def wait(self, timeout: float | None = None) -> int: + del timeout + self.exited = True + self.returncode = self.status + return self.status + + def terminate(self) -> None: + self.exited = True + + def kill(self) -> None: + self.exited = True + + +class FakeInteraction: + def __init__(self, chunks: list[bytes], status: int) -> None: + self.process = FakeProcess(status) + self.chunks = chunks + self.writes: list[bytes] = [] + + def read_output(self, chunks: queue.Queue[bytes | None]) -> None: + for chunk in self.chunks: + chunks.put(chunk) + self.process.exited = True + chunks.put(None) + + def read_stderr(self, chunks: queue.Queue[bytes | None]) -> None: + chunks.put(None) + + def write_input(self, data: bytes) -> None: + self.writes.append(data) + + def close(self) -> None: + pass + + +class ScriptedInteraction(FakeInteraction): + """A guest that answers each console write with the next scripted reply + and exits after the last one.""" + + def __init__( + self, chunks: list[bytes], status: int, replies: list[list[bytes]] + ) -> None: + super().__init__(chunks, status) + self.replies = list(replies) + self._sink: queue.Queue[bytes | None] | None = None + + def read_output(self, chunks: queue.Queue[bytes | None]) -> None: + self._sink = chunks + for chunk in self.chunks: + chunks.put(chunk) + if not self.replies: + self._exit() + + def write_input(self, data: bytes) -> None: + self.writes.append(data) + assert self._sink is not None + for chunk in self.replies.pop(0): + self._sink.put(chunk) + if not self.replies: + self._exit() + + def _exit(self) -> None: + assert self._sink is not None + self.process.exited = True + self._sink.put(None) + + +class RunnerWiringTests(unittest.TestCase): + @staticmethod + def _fake_exit(process: FakeProcess, _timeout: float) -> int: + """Wait for a fake OpenVMM, whose PID is not a real process.""" + return process.wait() + + @staticmethod + def _teardown_timeout(process: FakeProcess, timeout: float) -> int: + raise subprocess.TimeoutExpired(str(process.pid), timeout) + + def _measure( + self, + chunks: list[bytes], + status: int, + command: list[str], + *, + teardown_mode: str = "guest-exit", + teardown_timeout: bool = False, + log_path: Path | None = None, + ): + interaction = FakeInteraction(chunks, status) + with ( + patch.object(benchmark, "InteractiveProcess", return_value=interaction), + patch.object(benchmark, "live_peak_rss_bytes", return_value=1024), + # Never pidfd_open the fake's PID. + patch.object( + benchmark, + "wait_for_process_exit", + side_effect=( + self._teardown_timeout if teardown_timeout else self._fake_exit + ), + ), + ): + return benchmark.measure_once( + command, + environment={}, + timeout=5, + marker=benchmark.RESTORE_MARKER, + marker_must_be_line=True, + teardown_mode=teardown_mode, + guest_exit_prequeued=True, + log_path=log_path, + ) + + def test_measure_once_fails_fast_on_a_violation_event(self): + with self.assertRaisesRegex(RuntimeError, "violation G_RCU_STALL") as raised: + self._measure( + [b"restored\n", VIOLATION, benchmark.RESTORE_MARKER + b"\n"], + 194, + MSHV_RESTORE, + ) + self.assertIn("--- OpenVMM output ---", str(raised.exception)) + + def test_measure_once_classifies_a_time_abi_power_off(self): + with self.assertRaisesRegex( + RuntimeError, "status 195 \\(restore repair\\): no NVX-TIME-ABI" + ): + self._measure([b"restoring\n"], 195, MSHV_RESTORE) + with self.assertRaisesRegex(RuntimeError, "OpenVMM exited with status 1"): + self._measure([b"restoring\n"], 1, MSHV_RESTORE) + + def test_exit_errors_lead_with_openvmm_fatal_time_abi_code(self): + expected = f"exited with status 1: {re.escape(FATAL_DETAIL)}\n" + with self.assertRaisesRegex(RuntimeError, f"^OpenVMM {expected}"): + self._measure([b"restoring\n", FATAL], 1, MSHV_RESTORE) + interaction = FakeInteraction([b"booting\n", FATAL], 1) + with tempfile.TemporaryDirectory() as temporary: + with patch.object( + benchmark, "InteractiveProcess", return_value=interaction + ): + with self.assertRaisesRegex( + RuntimeError, f"^snapshot source {expected}" + ): + benchmark.capture_snapshot( + KVM_BOOT, Path(temporary) / "snapshot", timeout=5 + ) + log_path = Path(temporary) / "process.log" + with patch.object( + openvmm_process, + "InteractiveProcess", + return_value=FakeInteraction([FATAL], 1), + ): + with openvmm_process.OpenvmmProcess(KVM_BOOT, log_path) as process: + with self.assertRaisesRegex( + RuntimeError, + f"^OpenVMM exited with status 1 before .*: " + f"{re.escape(FATAL_DETAIL)}\n", + ): + process.wait_for(b"NEVER", 1) + # An expected failure still returns its status and output. + with patch.object( + openvmm_process, + "InteractiveProcess", + return_value=FakeInteraction([FATAL], 1), + ): + with openvmm_process.OpenvmmProcess(KVM_BOOT, log_path) as process: + self.assertEqual(process.wait(1).returncode, 1) + + def test_measure_once_classifies_a_power_off_during_teardown(self): + interaction = FakeInteraction([benchmark.RESTORE_MARKER + b"\n"], 194) + with ( + patch.object(benchmark, "InteractiveProcess", return_value=interaction), + patch.object(benchmark, "live_peak_rss_bytes", return_value=1024), + patch.object(benchmark, "wait_for_process_exit", return_value=194), + ): + with self.assertRaisesRegex(RuntimeError, "status 194 \\(runtime"): + benchmark.measure_once( + MSHV_RESTORE, + environment={}, + timeout=5, + marker=benchmark.RESTORE_MARKER, + marker_must_be_line=True, + guest_exit_prequeued=True, + ) + + def test_measure_once_scans_the_console_after_the_marker(self): + # The guest can print a violation during teardown and still exit 0, + # when its power-off loses the race against the queued guest exit. + marker = benchmark.RESTORE_MARKER + b"\n" + with tempfile.TemporaryDirectory() as temporary: + log_path = Path(temporary) / "restore.log" + for logged in (False, True): + with self.subTest(logged=logged): + with self.assertRaisesRegex( + RuntimeError, "violation G_RCU_STALL" + ) as raised: + self._measure( + [marker, VIOLATION], + 0, + MSHV_RESTORE, + log_path=log_path if logged else None, + ) + self.assertIn("--- OpenVMM output ---", str(raised.exception)) + self.assertIn(VIOLATION, log_path.read_bytes()) + _elapsed, peak, teardown, _wall = self._measure( + [marker, b"~ # "], 0, MSHV_RESTORE + ) + self.assertEqual(peak, 1024) + self.assertIsNotNone(teardown) + + def test_measure_once_scans_an_unterminated_last_line(self): + marker = benchmark.RESTORE_MARKER + b"\n" + failed_check = ( + b'NVX-TIME-ABI: v=1 phase=boot status=fail check=C4 detail="kvm-clock"' + ) + for line, error in ( + (VIOLATION.rstrip(b"\n"), "violation G_RCU_STALL"), + (failed_check, "boot check C4 failed: kvm-clock"), + ): + with self.subTest(error=error): + with self.assertRaisesRegex(RuntimeError, error): + self._measure([marker, line], 0, MSHV_RESTORE) + + def test_measure_once_scans_the_console_after_a_teardown_timeout(self): + # The harness kills OpenVMM and returns the sample without a teardown + # time, unless the console or a power-off that beat the kill fails. + marker = benchmark.RESTORE_MARKER + b"\n" + _elapsed, peak, teardown, _wall = self._measure( + [marker], -9, MSHV_RESTORE, teardown_timeout=True + ) + self.assertEqual(peak, 1024) + self.assertIsNone(teardown) + with self.assertRaisesRegex(RuntimeError, "violation G_RCU_STALL"): + self._measure([marker, VIOLATION], -9, MSHV_RESTORE, teardown_timeout=True) + with self.assertRaisesRegex(RuntimeError, "status 194 \\(runtime violation"): + self._measure([marker], 194, MSHV_RESTORE, teardown_timeout=True) + + def test_measure_once_scans_the_console_of_a_terminated_openvmm(self): + # The status of the termination, -15 on Linux and 1 on Windows, never + # fails the launch, but the console and a power-off that beat the + # termination do. + marker = benchmark.RESTORE_MARKER + b"\n" + for status in (-15, 1): + with self.subTest(status=status): + result = self._measure( + [marker], status, MSHV_RESTORE, teardown_mode="host-terminate" + ) + self.assertEqual(result[1], 1024) + with self.assertRaisesRegex(RuntimeError, "violation G_RCU_STALL"): + self._measure( + [marker, VIOLATION], + status, + MSHV_RESTORE, + teardown_mode="host-terminate", + ) + with self.assertRaisesRegex(RuntimeError, "status 194 \\(runtime violation"): + self._measure([marker], 194, MSHV_RESTORE, teardown_mode="host-terminate") + + def test_measure_once_failure_paths_scan_without_masking_their_error(self): + # The rest of the console still reaches the monitor and the log, but + # no time ABI failure is raised over the error being handled. + with tempfile.TemporaryDirectory() as temporary: + log_path = Path(temporary) / "restore.log" + interaction = FakeInteraction([b"NVX-TEST-FAIL broken\n", VIOLATION], 1) + with patch.object( + benchmark, "InteractiveProcess", return_value=interaction + ): + with self.assertRaises(benchmark.GuestFailureReported) as reported: + benchmark.measure_once( + MSHV_RESTORE, + environment={}, + timeout=5, + marker=benchmark.RESTORE_MARKER, + marker_must_be_line=True, + guest_exit_prequeued=True, + log_path=log_path, + failure_marker=b"NVX-TEST-FAIL", + ) + self.assertEqual(reported.exception.line, "NVX-TEST-FAIL broken") + self.assertIn(VIOLATION, log_path.read_bytes()) + + killed = threading.Event() + + class KilledProcess(FakeProcess): + def kill(self) -> None: + super().kill() + killed.set() + + class LateEventInteraction(FakeInteraction): + """A guest whose event reaches the console as OpenVMM stops.""" + + def read_output(self, chunks: queue.Queue[bytes | None]) -> None: + chunks.put(b"booting\n") + killed.wait(5) + chunks.put(VIOLATION) + chunks.put(None) + + interaction = LateEventInteraction([], -9) + interaction.process = KilledProcess(-9) + with patch.object(benchmark, "InteractiveProcess", return_value=interaction): + with self.assertRaisesRegex( + RuntimeError, "^guest marker was not observed within 0.2s" + ) as timed_out: + benchmark.measure_once( + MSHV_RESTORE, + environment={}, + timeout=0.2, + marker=benchmark.RESTORE_MARKER, + marker_must_be_line=True, + guest_exit_prequeued=True, + ) + self.assertIn("G_RCU_STALL", str(timed_out.exception)) + + def test_run_guest_script_classifies_a_conformance_power_off(self): + interaction = FakeInteraction( + [ + b'NVX-TIME-ABI: v=1 phase=boot status=fail check=C4 detail="kvm-clock"', + ], + 193, + ) + with patch.object(benchmark, "InteractiveProcess", return_value=interaction): + with self.assertRaisesRegex( + RuntimeError, "boot check C4 failed: kvm-clock" + ): + benchmark.run_guest_script( + KVM_BOOT, "true\n", b"DONE", timeout=5, boot_marker=b"BOOT" + ) + + def test_cold_boot_runners_query_the_status_before_other_input(self): + status = time_abi.status_script().encode() + boot = benchmark.BOOT_MARKER + b": 3.22.1\n" + # A quiet guest that answers the query without a boot line fails, and + # never receives the script. + quiet = ScriptedInteraction([boot], 0, [[STATUS_OK.encode()], [b"DONE\n"]]) + with patch.object(benchmark, "InteractiveProcess", return_value=quiet): + with self.assertRaisesRegex(RuntimeError, "NVX-TIME-ABI boot marker"): + benchmark.run_guest_script( + KVM_BOOT, "true\n", b"DONE", timeout=5, time_abi_status=True + ) + self.assertEqual(quiet.writes, [status]) + answered = [BOOT_LINE.encode(), STATUS_OK.encode()] + conformant = ScriptedInteraction([boot], 0, [answered, [b"DONE\n"]]) + with patch.object(benchmark, "InteractiveProcess", return_value=conformant): + result = benchmark.run_guest_script( + KVM_BOOT, "true\n", b"DONE", timeout=5, time_abi_status=True + ) + self.assertEqual(conformant.writes, [status, b"true\n"]) + self.assertIn("phase=boot status=ok", result["text"]) + # A guest that exits during the query names the query. + exiting = ScriptedInteraction([boot], 0, [[]]) + with patch.object(benchmark, "InteractiveProcess", return_value=exiting): + with self.assertRaisesRegex( + RuntimeError, "nvx-time status did not finish before the guest exited" + ): + benchmark.run_guest_script( + KVM_BOOT, "true\n", b"DONE", timeout=5, time_abi_status=True + ) + # Benchmarks and restores never query: the console stays quiet. + for command in (KVM_BOOT, MSHV_RESTORE): + silent = ScriptedInteraction([boot], 0, [[b"DONE\n"]]) + with patch.object(benchmark, "InteractiveProcess", return_value=silent): + benchmark.run_guest_script( + command, + "true\n", + b"DONE", + timeout=5, + time_abi_status=command is MSHV_RESTORE, + ) + self.assertEqual(silent.writes, [b"true\n"]) + self._measure([boot, benchmark.RESTORE_MARKER + b"\n"], 0, KVM_BOOT) + + def test_capture_snapshot_queries_the_status_before_the_capture_script(self): + boot = benchmark.BOOT_MARKER + b": 3.22.1\n" + answered = [BOOT_LINE.encode(), STATUS_OK.encode()] + for queried in (False, True): + replies = [[b"capture script exited\n"]] + if queried: + replies.insert(0, answered) + interaction = ScriptedInteraction([boot], 0, replies) + with tempfile.TemporaryDirectory() as temporary: + with patch.object( + benchmark, "InteractiveProcess", return_value=interaction + ): + with self.assertRaisesRegex( + RuntimeError, "exited before its snapshot request" + ): + benchmark.capture_snapshot( + KVM_BOOT, + Path(temporary) / "snapshot", + timeout=5, + processors=1, + time_abi_status=queried, + ) + with self.subTest(queried=queried): + capture = benchmark.prepare_snapshot_capture_script( + 1, teardown_mode="guest-exit" + ).encode() + status = time_abi.status_script().encode() + expected = [status, capture] if queried else [capture] + self.assertEqual(interaction.writes, expected) + quiet = ScriptedInteraction([boot], 0, [[STATUS_OK.encode()]]) + with tempfile.TemporaryDirectory() as temporary: + with patch.object(benchmark, "InteractiveProcess", return_value=quiet): + with self.assertRaisesRegex(RuntimeError, "NVX-TIME-ABI boot marker"): + benchmark.capture_snapshot( + KVM_BOOT, + Path(temporary) / "snapshot", + timeout=5, + processors=1, + time_abi_status=True, + ) + self.assertEqual(len(quiet.writes), 1) + + def test_openvmm_process_queries_the_status_before_the_first_input(self): + status = time_abi.status_script().encode() + answered = [BOOT_LINE.encode(), STATUS_OK.encode()] + with tempfile.TemporaryDirectory() as temporary: + log_path = Path(temporary) / "process.log" + # The scenario's substring wait can return before the boot line + # ends; its input still follows the finished query. + guest = ScriptedInteraction( + [b"NVX-GUEST-BOOT-OK:"], 0, [[b" alpine\n", *answered], [b"OK\n"]] + ) + with patch.object( + openvmm_process, "InteractiveProcess", return_value=guest + ): + with openvmm_process.OpenvmmProcess(KVM_BOOT, log_path) as process: + process.wait_for(b"NVX-GUEST-BOOT-OK:", 1) + process.send_line("echo OK") + self.assertEqual(process.time_abi.status_queries, 1) + process.wait_for_line(b"OK", 1) + self.assertEqual(process.wait(1).returncode, 0) + self.assertEqual(guest.writes, [status, b"echo OK\n"]) + + quiet = ScriptedInteraction( + [b"NVX-GUEST-BOOT-OK: alpine\n"], 0, [[STATUS_OK.encode()], []] + ) + with patch.object( + openvmm_process, "InteractiveProcess", return_value=quiet + ): + with openvmm_process.OpenvmmProcess(KVM_BOOT, log_path) as process: + process.wait_for(b"NVX-GUEST-BOOT-OK:", 1) + with self.assertRaisesRegex( + RuntimeError, "did not report the NVX-TIME-ABI boot marker" + ): + process.send_line("echo OK") + self.assertEqual(quiet.writes, [status]) + + # Restores, guests whose shell is on another console, and opted-out + # processes receive only the scenario's input. + for command, output, enabled in ( + (MSHV_RESTORE, b"NVX-GUEST-BOOT-OK: alpine\n", True), + (KVM_BOOT, b"booting\n", True), + (KVM_BOOT, b"NVX-GUEST-BOOT-OK: alpine\n", False), + ): + other = ScriptedInteraction([output], 0, [[]]) + with patch.object( + openvmm_process, "InteractiveProcess", return_value=other + ): + with openvmm_process.OpenvmmProcess( + command, log_path, time_abi_status=enabled + ) as process: + process.wait_for(output.rstrip(b"\n"), 1) + process.send_bytes(b"\x01") + self.assertEqual(process.wait(1).returncode, 0) + with self.subTest(command=command, output=output, enabled=enabled): + self.assertEqual(other.writes, [b"\x01"]) + + def test_capture_snapshot_classifies_a_failed_capture_check(self): + interaction = FakeInteraction([b"booting\n"], 193) + with tempfile.TemporaryDirectory() as temporary: + with patch.object( + benchmark, "InteractiveProcess", return_value=interaction + ): + with self.assertRaisesRegex(RuntimeError, "status 193 \\(conformance"): + benchmark.capture_snapshot( + KVM_BOOT, + Path(temporary) / "snapshot", + timeout=5, + ) + + def test_openvmm_process_reports_violations_and_time_abi_statuses(self): + with tempfile.TemporaryDirectory() as temporary: + log_path = Path(temporary) / "process.log" + with patch.object( + openvmm_process, + "InteractiveProcess", + return_value=FakeInteraction([b"FIRST\n", VIOLATION], 194), + ): + with openvmm_process.OpenvmmProcess(KVM_BOOT, log_path) as process: + process.wait_for(b"FIRST", 1) + with self.assertRaisesRegex(RuntimeError, "G_RCU_STALL"): + process.wait_for(b"NEVER", 1) + self.assertIn(b"G_RCU_STALL", log_path.read_bytes()) + with patch.object( + openvmm_process, + "InteractiveProcess", + return_value=FakeInteraction([b"working\n"], 194), + ): + with openvmm_process.OpenvmmProcess(KVM_BOOT, log_path) as process: + with self.assertRaisesRegex(RuntimeError, "status 194"): + process.wait(1) + with patch.object( + openvmm_process, + "InteractiveProcess", + return_value=FakeInteraction([b"working\n"], 195), + ): + with openvmm_process.OpenvmmProcess(KVM_BOOT, log_path) as process: + with self.assertRaisesRegex(RuntimeError, "status 195"): + process.wait_for_line(b"NEVER", 1) + + def test_openvmm_process_waits_for_time_abi_phase_markers(self): + restore = ( + b"NVX-TIME-ABI: v=1 phase=restore status=ok cpus=2 tsc_hz=2300000000 " + b"lapic_hz=200000000 generation=1 elapsed_us=310\n" + ) + with tempfile.TemporaryDirectory() as temporary: + log_path = Path(temporary) / "process.log" + with patch.object( + openvmm_process, + "InteractiveProcess", + return_value=FakeInteraction([b"x\n", restore], 0), + ): + with openvmm_process.OpenvmmProcess(MSHV_RESTORE, log_path) as process: + marker = process.wait_for_time_abi("restore", 1) + self.assertEqual(marker["generation"], "1") + self.assertEqual(process.wait(1).returncode, 0) + with patch.object( + openvmm_process, + "InteractiveProcess", + return_value=FakeInteraction([b"x\n"], 0), + ): + with openvmm_process.OpenvmmProcess(MSHV_RESTORE, log_path) as process: + with self.assertRaisesRegex(RuntimeError, "restore marker"): + process.wait_for_time_abi("restore", 1) + + def test_tcp_console_monitor_reports_violations(self): + connection, peer = socket.socketpair() + console = openvmm_process.TcpConsole(connection, TimeAbiMonitor(KVM_BOOT)) + peer.sendall(VIOLATION) + with self.assertRaisesRegex(TimeAbiFailure, "G_RCU_STALL"): + console.wait_for(b"NEVER", 1.0) + console.close() + peer.close() + + def test_tcp_console_finish_scans_the_tail(self): + # A violation after the last awaited marker still fails the scenario, + # including an unterminated final line. + connection, peer = socket.socketpair() + console = openvmm_process.TcpConsole(connection, TimeAbiMonitor(KVM_BOOT)) + peer.sendall(b"READY\n") + console.wait_for(b"READY", 1.0) + peer.sendall(VIOLATION.rstrip(b"\n")) + peer.close() + with self.assertRaisesRegex(TimeAbiFailure, "G_RCU_STALL"): + console.finish() + self.assertIn(b"G_RCU_STALL", console.finish(check=False)) + with self.assertRaisesRegex(TimeAbiFailure, "G_RCU_STALL"): + console.finish() + + # Error paths keep the tail in the log without raising over the error. + connection, peer = socket.socketpair() + console = openvmm_process.TcpConsole(connection, TimeAbiMonitor(KVM_BOOT)) + peer.sendall(VIOLATION) + peer.close() + self.assertEqual(console.finish(check=False), VIOLATION) + + def test_tcp_console_queries_a_cold_boot_before_the_first_input(self): + connection, peer = socket.socketpair() + peer.settimeout(1.0) + monitor = TimeAbiMonitor(KVM_BOOT) + console = openvmm_process.TcpConsole(connection, monitor, time_abi_status=True) + peer.sendall(b"NVX-GUEST-BOOT-OK: alpine\n") + console.wait_for(b"NVX-GUEST-BOOT-OK", 1.0) + peer.sendall((BOOT_LINE + RUNTIME_LINE + STATUS_OK).encode()) + console.send_bytes(b"INPUT\n") + console.send_bytes(b"MORE\n") + expected = time_abi.status_script().encode() + b"INPUT\nMORE\n" + sent = b"" + while len(sent) < len(expected): + sent += peer.recv(4096) + self.assertEqual(sent, expected) + self.assertEqual(monitor.status_queries, 1) + self.assertIsNotNone(monitor.boot) + console.close() + peer.close() + + # A query without a passing boot line fails before the input is sent. + connection, peer = socket.socketpair() + console = openvmm_process.TcpConsole( + connection, TimeAbiMonitor(KVM_BOOT), time_abi_status=True + ) + peer.sendall(b"NVX-GUEST-BOOT-OK: alpine\n") + console.wait_for(b"NVX-GUEST-BOOT-OK", 1.0) + peer.sendall(STATUS_OK.encode()) + with self.assertRaisesRegex(TimeAbiFailure, "boot marker"): + console.send_bytes(b"INPUT\n") + console.close() + peer.close() + + def test_tcp_console_waits_for_a_status_query_to_exit(self): + connection, peer = socket.socketpair() + monitor = TimeAbiMonitor(MSHV_RESTORE) + console = openvmm_process.TcpConsole(connection, monitor) + restore = RESTORE_LINE.replace("lapic_hz=1000000000", "lapic_hz=200000000") + peer.sendall((restore + RUNTIME_LINE + STATUS_OK).encode()) + console.wait_for_time_abi_status(1.0) + self.assertEqual(monitor.status_queries, 1) + self.assertEqual(len(monitor.restores), 1) + # A query that reports no restore line fails when it exits. + peer.sendall(STATUS_OK.encode()) + monitor.restores.clear() + with self.assertRaisesRegex(TimeAbiFailure, "restore marker"): + console.wait_for_time_abi_status(1.0) + with self.assertRaisesRegex(TimeoutError, "did not exit"): + console.wait_for_time_abi_status(0.2) + console.close() + peer.close() + unmonitored = socket.socket() + self.addCleanup(unmonitored.close) + with self.assertRaisesRegex(ValueError, "no time ABI monitor"): + openvmm_process.TcpConsole(unmonitored).wait_for_time_abi_status(1.0) + + def test_tcp_console_queries_only_cold_boots_that_ask(self): + for command, enabled in ( + (MSHV_RESTORE, True), + (KVM_BOOT, False), + ): + with self.subTest(command=command, enabled=enabled): + connection, peer = socket.socketpair() + peer.settimeout(1.0) + monitor = TimeAbiMonitor(command) + console = openvmm_process.TcpConsole( + connection, monitor, time_abi_status=enabled + ) + peer.sendall(b"NVX-GUEST-BOOT-OK: alpine\n") + console.wait_for(b"NVX-GUEST-BOOT-OK", 1.0) + console.send_bytes(b"Z") + self.assertEqual(peer.recv(4096), b"Z") + self.assertEqual(monitor.status_queries, 0) + console.close() + peer.close() + + +def doctor_context(root: Path, backend: str = "kvm") -> doctor.DoctorContext: + return doctor.DoctorContext( + backend=backend, + openvmm=root / "openvmm", + kernel=root / "vmlinux", + initrd=root / "initramfs.cpio.gz", + openvmm_args=(), + probe_directory=root / "probe", + timeout=30, + ) + + +def completed(stdout: str, returncode: int = 0, stderr: str = ""): + return subprocess.CompletedProcess(["x"], returncode, stdout, stderr) + + +CPU = { + "vendor": "GenuineIntel", + "family": "6", + "model": "106", + "stepping": "6", + "microcode": "0xffffffff", + "brand": "Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz", + "os": "Linux 6.6.150.1-1.azl3", + "invariant_tsc": "yes", +} +# OpenVMM's cpu_profile FingerprintCheck::summary_line, printed on stderr. +PROFILE_PASS = ( + "NVX-CPU-PROFILE: status=pass backend=kvm generation=icelake-sp " + f"profile=intel.icelake-sp.v1 profile_digest=sha256:{'a3' * 32} " + "surface_digest=sha256:6286956acbef host_invariant_tsc=yes\n" +) + + +class DoctorTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.root = Path(self.temporary.name) + + def tearDown(self): + self.temporary.cleanup() + + def test_check_lines_escape_details_for_field_parsers(self): + line = doctor.CheckResult("H5", False, 'say "x" \\ y').line() + self.assertTrue(line.startswith("NVX-DOCTOR: check=H5 status=fail ")) + fields = time_abi.parse_fields(line.removeprefix(doctor.DOCTOR_PREFIX)) + self.assertEqual(fields["detail"], 'say "x" \\ y') + + def test_backend_check_requires_a_usable_device(self): + context = doctor_context(self.root) + existing: set[str] = set() + + def exists(path: Path) -> bool: + return path.as_posix() in existing + + with ( + patch.object(doctor, "host_is_windows", return_value=False), + patch.object(doctor.Path, "exists", exists), + patch.object(doctor.os, "access", return_value=True) as access, + ): + self.assertIn("does not exist", doctor.check_backend(context).detail) + existing.add("/dev/kvm") + self.assertTrue(doctor.check_backend(context).passed) + existing.add("/dev/mshv") + self.assertIn("select MSHV", doctor.check_backend(context).detail) + existing.discard("/dev/mshv") + access.return_value = False + self.assertIn("not readable", doctor.check_backend(context).detail) + with patch.object(doctor, "host_is_windows", return_value=False): + whp = doctor.check_backend(doctor_context(self.root, "whp")) + self.assertFalse(whp.passed) + self.assertIn("requires Windows", whp.detail) + + def test_cpu_check_names_the_generation_and_records_invariant_tsc_as_evidence( + self, + ): + context = doctor_context(self.root) + with patch.object(doctor, "host_cpu", return_value=dict(CPU)): + result = doctor.check_cpu(context) + self.assertTrue(result.passed) + self.assertIn("generation=icelake-sp", result.detail) + self.assertIn("profile=intel.icelake-sp.v1", result.detail) + self.assertEqual(context.facts["generation"], "icelake-sp") + for unknown in (dict(CPU, model="143"), dict(CPU, model="85", stepping="7")): + with patch.object(doctor, "host_cpu", return_value=unknown): + result = doctor.check_cpu(doctor_context(self.root)) + self.assertFalse(result.passed) + self.assertTrue(result.detail.startswith("[E_PROFILE_HOST_UNKNOWN] ")) + self.assertIn(time_abi.describe_cpu_generations(), result.detail) + alder_lake = dict(CPU, model="154", stepping="3") + with patch.object(doctor, "host_cpu", return_value=alder_lake): + result = doctor.check_cpu(doctor_context(self.root, "whp")) + self.assertTrue(result.passed, result.detail) + self.assertIn("generation=alderlake profile=intel.alderlake.v1", result.detail) + emerald = dict(CPU, model="207", stepping="2") + with patch.object(doctor, "host_cpu", return_value=emerald): + for backend in ("kvm", "mshv", "whp"): + result = doctor.check_cpu(doctor_context(self.root, backend)) + self.assertTrue(result.passed, result.detail) + self.assertIn("profile=intel.emeraldrapids.v1", result.detail) + # The host OS's invariant-TSC flags are evidence, never a gate. + variant = dict(CPU, invariant_tsc="no (missing nonstop_tsc)") + with patch.object(doctor, "host_cpu", return_value=variant): + for backend in ("kvm", "mshv", "whp"): + context = doctor_context(self.root, backend) + result = doctor.check_cpu(context) + self.assertTrue(result.passed, result.detail) + self.assertIn("invariant_tsc=no (missing nonstop_tsc)", result.detail) + self.assertEqual( + context.facts["invariant_tsc"], "no (missing nonstop_tsc)" + ) + + def test_cpu_check_verifies_the_profile_with_openvmm_fingerprint(self): + fingerprint = self.root / "out" / "fingerprint.json" + + def context_with_openvmm(backend: str = "kvm") -> doctor.DoctorContext: + context = doctor_context(self.root, backend) + context.openvmm.write_bytes(b"") + context.fingerprint = fingerprint + return context + + def check( + result: subprocess.CompletedProcess[str], + cpu: dict[str, str] = CPU, + backend: str = "kvm", + ) -> tuple[doctor.CheckResult, doctor.DoctorContext]: + context = context_with_openvmm(backend) + with ( + patch.object(doctor, "host_cpu", return_value=dict(cpu)), + patch.object(doctor.subprocess, "run", return_value=result), + ): + return doctor.check_cpu(context), context + + context = context_with_openvmm() + with ( + patch.object(doctor, "host_cpu", return_value=dict(CPU)), + patch.object( + doctor.subprocess, "run", return_value=completed("", 0, PROFILE_PASS) + ) as run, + ): + result = doctor.check_cpu(context) + self.assertTrue(result.passed, result.detail) + self.assertEqual( + run.call_args.args[0], + [ + str(context.openvmm), + "--hypervisor", + "kvm", + "--cpu-fingerprint", + str(fingerprint), + ], + ) + self.assertEqual(run.call_args.kwargs["env"]["OPENVMM_LOG"], "off") + self.assertTrue(fingerprint.parent.is_dir()) + self.assertIn("surface_digest=sha256:6286956acbef", result.detail) + self.assertEqual(context.facts["profile"], "intel.icelake-sp.v1") + self.assertEqual(context.facts["profile_digest"], f"sha256:{'a3' * 32}") + self.assertEqual(context.facts["surface_digest"], "sha256:6286956acbef") + # A later revision of the generation's profile is fine. + revised = PROFILE_PASS.replace("icelake-sp.v1", "icelake-sp.v2") + self.assertTrue(check(completed("", 0, revised))[0].passed) + # OpenVMM's catalog, not NVX's copy, decides which hosts it serves: a + # generation that only OpenVMM pins passes. + raptor_lake = PROFILE_PASS.replace("icelake-sp", "raptorlake") + result, context = check( + completed("", 0, raptor_lake), dict(CPU, model="183", stepping="1") + ) + self.assertTrue(result.passed, result.detail) + self.assertIn( + "generation=raptorlake profile=intel.raptorlake.v1", result.detail + ) + self.assertEqual(context.facts["generation"], "raptorlake") + self.assertEqual(context.facts["profile"], "intel.raptorlake.v1") + + unsupported = ( + "NVX-CPU-PROFILE: status=fail backend=whp generation=icelake-sp " + "profile=intel.icelake-sp.v1 profile_digest=sha256:a3 " + "surface_digest=sha256:d67f host_invariant_tsc=no " + 'code=E_PROFILE_UNSUPPORTED detail="CPUID 0x7.0 EDX bits 26, 27 are ' + 'not supported; leaf \\"7\\"\\u{a0}subleaf"\n' + ) + result, context = check(completed("", 1, unsupported), backend="whp") + self.assertFalse(result.passed) + self.assertTrue( + result.detail.startswith( + "[E_PROFILE_UNSUPPORTED] OpenVMM's CPU profile check failed (exit 1): " + 'CPUID 0x7.0 EDX bits 26, 27 are not supported; leaf "7"\xa0subleaf;' + ), + result.detail, + ) + unknown = ( + "NVX-CPU-PROFILE: status=fail backend=kvm generation=none profile=none " + "surface_digest=sha256:1 host_invariant_tsc=yes " + 'code=E_PROFILE_HOST_UNKNOWN detail="GenuineIntel family 6 model 143"\n' + ) + result, context = check(completed("", 1, unknown), dict(CPU, model="143")) + self.assertFalse(result.passed) + self.assertTrue(result.detail.startswith("[E_PROFILE_HOST_UNKNOWN] ")) + self.assertIn("[E_PROFILE_HOST_UNKNOWN] OpenVMM's CPU profile", result.detail) + self.assertEqual(context.facts["profile"], "none") + for output, message in ( + ( + completed("", 2, "error: unexpected argument '--cpu-fingerprint'"), + "predates the --cpu-fingerprint tool", + ), + (completed("", 1, "fatal error: no backend"), "fatal error: no backend"), + ( + completed("", 0, PROFILE_PASS.replace("=kvm", "=mshv")), + "fingerprinted the mshv backend", + ), + ( + completed( + "", + 0, + PROFILE_PASS.replace( + "generation=icelake-sp", "generation=emeraldrapids" + ), + ), + "CPU profile intel.icelake-sp.v1 of generation icelake-sp for " + "generation emeraldrapids", + ), + ( + completed( + "", + 0, + PROFILE_PASS.replace( + "profile=intel.icelake-sp.v1", "profile=intel.icelake-sp.v2-rc" + ), + ), + "CPU profile intel.icelake-sp.v2-rc, which is not a catalog profile ID", + ), + ( + completed( + "", + 0, + PROFILE_PASS.replace( + "generation=icelake-sp profile=intel.icelake-sp.v1", + "generation=host profile=intel.host.v1", + ), + ), + "host profile intel.host.v1, which qualification never accepts", + ), + ): + with self.subTest(message=message): + result = check(output)[0] + self.assertFalse(result.passed) + self.assertIn(message, result.detail) + # A passing line that lacks the generation or the profile, or leaves + # one empty, says so instead of classifying a profile that OpenVMM did + # not report. + for output_line, unreported in ( + (PROFILE_PASS.replace("generation=icelake-sp ", ""), "generation"), + ( + PROFILE_PASS.replace("profile=intel.icelake-sp.v1 ", "profile= "), + "profile", + ), + ( + PROFILE_PASS.replace( + "generation=icelake-sp profile=intel.icelake-sp.v1 ", "" + ), + "generation and profile", + ), + ): + with self.subTest(unreported=unreported): + result, context = check(completed("", 0, output_line)) + self.assertFalse(result.passed) + self.assertIn( + "OpenVMM's CPU profile check passed without reporting its " + f"{unreported};", + result.detail, + ) + self.assertNotIn("OpenVMM reported", result.detail) + self.assertEqual( + context.facts["generation"], + "none" if "generation" in unreported else "icelake-sp", + ) + self.assertEqual( + context.facts["profile"], + "none" if "profile" in unreported else "intel.icelake-sp.v1", + ) + missing = doctor_context(self.root / "none") + missing.fingerprint = fingerprint + with patch.object(doctor, "host_cpu", return_value=dict(CPU)): + result = doctor.check_cpu(missing) + self.assertFalse(result.passed) + self.assertIn("was not found", result.detail) + self.assertIn("--no-openvmm", result.detail) + # Without OpenVMM, H2 checks the identity and generation only. + with ( + patch.object(doctor, "host_cpu", return_value=dict(CPU)), + patch.object(doctor.subprocess, "run") as run, + ): + result = doctor.check_cpu(doctor_context(self.root)) + self.assertTrue(result.passed) + self.assertIn("CPU profile not checked", result.detail) + run.assert_not_called() + + def test_reads_the_first_processor_from_cpuinfo(self): + path = self.root / "cpuinfo" + path.write_text( + "processor\t: 0\nvendor_id\t: GenuineIntel\ncpu family\t: 6\n" + "model\t\t: 85\nstepping\t: 4\nmicrocode\t: 0x2007108\n" + "flags\t\t: fpu tsc constant_tsc nonstop_tsc\n\n" + "processor\t: 1\nvendor_id\t: Other\n", + encoding="utf-8", + ) + fields = doctor._linux_cpuinfo(path) # pyright: ignore[reportPrivateUsage] + self.assertEqual(fields["vendor_id"], "GenuineIntel") + self.assertEqual(fields["model"], "85") + self.assertIn("nonstop_tsc", fields["flags"]) + + def test_qualification_rejects_a_host_on_which_auto_falls_back(self): + """auto falls back to a host profile on a CPU that no built-in profile + serves, but qualification does not: H2's fingerprint check never falls + back, and H3 rejects the host profile that OpenVMM's preflight selects, + so CI's microVM jobs still require a built-in profile.""" + granite_rapids = dict( + CPU, model="173", stepping="1", brand="Intel(R) Xeon(R) 6973P-C" + ) + fingerprint = self.root / "fingerprint.json" + unknown = ( + "NVX-CPU-PROFILE: status=fail backend=kvm generation=none profile=none " + "surface_digest=sha256:1 host_invariant_tsc=yes " + 'code=E_PROFILE_HOST_UNKNOWN detail="no pinned CPU profile serves ' + 'the host CPU, GenuineIntel family 6 model 173 stepping 1"\n' + ) + fallback = ( + "NVX-TIME-ABI-VERIFY: v=1 status=ok backend=kvm cpu_profile=intel.host.v1 " + "tsc_hz=2793437000 native_tsc_hz=2793437000 lapic_hz=1000000000 " + "msr_route=ExitToVmm sync=CommonOffset\n" + ) + + def run(command: list[str], **_: object) -> subprocess.CompletedProcess[str]: + if "--cpu-fingerprint" in command: + return completed("", 1, unknown) + return completed(fallback) + + context = doctor_context(self.root) + for path in (context.openvmm, context.kernel, context.initrd): + path.write_bytes(b"") + context.fingerprint = fingerprint + with ( + patch.object(doctor, "host_cpu", return_value=granite_rapids), + patch.object(doctor.subprocess, "run", side_effect=run), + contextlib.redirect_stdout(io.StringIO()) as lines, + ): + cpu, preflight = doctor.run_checks(context, ("H2", "H3")) + self.assertIn("NVX-DOCTOR: check=H2 status=fail", lines.getvalue()) + self.assertIn("NVX-DOCTOR: check=H3 status=fail", lines.getvalue()) + self.assertEqual((cpu.check, cpu.passed), ("H2", False)) + self.assertTrue(cpu.detail.startswith("[E_PROFILE_HOST_UNKNOWN] "), cpu.detail) + self.assertEqual((preflight.check, preflight.passed), ("H3", False)) + self.assertIn( + "cpu_profile=intel.host.v1 (a host profile, to which --cpu-profile auto " + "falls back", + preflight.detail, + ) + # Without OpenVMM, NVX's copy of the catalog fails the CPU too. + context.fingerprint = None + with patch.object(doctor, "host_cpu", return_value=granite_rapids): + result = doctor.check_cpu(context) + self.assertFalse(result.passed) + self.assertTrue(result.detail.startswith("[E_PROFILE_HOST_UNKNOWN] ")) + + def test_identifies_the_host_cpu_signature(self): + HostCpu = time_abi.HostCpu + brand = "12th Gen Intel(R) Core(TM) i9-12900H" + + def registry(key: str, name: str) -> object: + self.assertEqual( + (key, name), (doctor.WINDOWS_PROCESSOR_KEY, "ProcessorNameString") + ) + return f"{brand} " + + with ( + patch.object(doctor, "host_is_windows", return_value=True), + patch.object(doctor, "_windows_registry_value", side_effect=registry), + ): + for processor, expected in ( + ( + "Intel64 Family 6 Model 154 Stepping 3, GenuineIntel", + HostCpu("GenuineIntel", 6, 154, 3, brand), + ), + ( + "AMD64 Family 25 Model 33 Stepping 0, AuthenticAMD", + HostCpu("AuthenticAMD", 25, 33, 0, brand), + ), + ("", None), + ): + with ( + self.subTest(processor=processor), + patch.object(doctor.platform, "processor", return_value=processor), + ): + self.assertEqual(doctor.host_cpu_signature(), expected) + # A brand string that the registry lacks leaves the signature. + with ( + patch.object(doctor, "host_is_windows", return_value=True), + patch.object( + doctor.platform, + "processor", + return_value="Intel64 Family 6 Model 154 Stepping 3, GenuineIntel", + ), + patch.object( + doctor, "_windows_registry_value", side_effect=OSError("no value") + ), + ): + self.assertEqual( + doctor.host_cpu_signature(), HostCpu("GenuineIntel", 6, 154, 3) + ) + cpuinfo = { + "vendor_id": "AuthenticAMD", + "cpu family": "25", + "model": "17", + "stepping": "1", + } + with patch.object(doctor, "host_is_windows", return_value=False): + for fields, expected in ( + (cpuinfo, HostCpu("AuthenticAMD", 25, 17, 1)), + ( + {**cpuinfo, "model name": "AMD EPYC 9V74 80-Core Processor"}, + HostCpu( + "AuthenticAMD", 25, 17, 1, "AMD EPYC 9V74 80-Core Processor" + ), + ), + ({**cpuinfo, "model": "?"}, None), + ({"vendor_id": "AuthenticAMD"}, None), + ): + with ( + self.subTest(fields=fields), + patch.object(doctor, "_linux_cpuinfo", return_value=fields), + ): + self.assertEqual(doctor.host_cpu_signature(), expected) + with patch.object( + doctor, "_linux_cpuinfo", side_effect=FileNotFoundError("cpuinfo") + ): + self.assertIsNone(doctor.host_cpu_signature()) + + def test_preflight_parses_openvmm_verification(self): + # The line format of OpenVMM's openvmm_entry verify.rs. + line = ( + "NVX-TIME-ABI-VERIFY: v=1 status=ok backend=kvm " + "cpu_profile=intel.icelake-sp.v1 tsc_hz=2793437000 " + "native_tsc_hz=2793437000 lapic_hz=1000000000 msr_route=ExitToVmm " + "sync=CommonOffset\n" + ) + + def context_with_files(backend: str = "kvm") -> doctor.DoctorContext: + context = doctor_context(self.root, backend) + for path in (context.openvmm, context.kernel, context.initrd): + path.write_bytes(b"") + return context + + context = context_with_files() + with patch.object( + doctor.subprocess, "run", return_value=completed(line) + ) as run: + result = doctor.check_openvmm_preflight(context) + self.assertTrue(result.passed, result.detail) + self.assertEqual(context.facts["native_tsc_hz"], "2793437000") + self.assertEqual(context.facts["profile"], "intel.icelake-sp.v1") + command = run.call_args.args[0] + self.assertEqual(command[-1], "--x-time-abi-verify") + self.assertEqual(command[command.index("--machine") + 1], "microvm") + self.assertEqual(command[command.index("--kernel") + 1], str(context.kernel)) + # --openvmm-arg passes development options, such as a test hook. + hooked = context_with_files() + hooked.openvmm_args = ("--x-time-abi-test-hook", "force-utc-downtime") + with patch.object( + doctor.subprocess, "run", return_value=completed(line) + ) as run: + self.assertTrue(doctor.check_openvmm_preflight(hooked).passed) + self.assertEqual( + run.call_args.args[0][-3:], + ["--x-time-abi-test-hook", "force-utc-downtime", "--x-time-abi-verify"], + ) + failure = ( + "NVX-TIME-ABI-VERIFY: v=1 status=fail backend=kvm " + 'code=E_TSC_SYNC_UNSUPPORTED detail="failed: \\"sync\\" unsupported"\n' + ) + cases = ( + ( + completed(failure, 1), + '[E_TSC_SYNC_UNSUPPORTED] OpenVMM verification failed (exit 1): failed: "sync" unsupported', + ), + ( + completed( + "", 2, "error: unexpected argument '--x-time-abi-verify' found" + ), + "predates the time ABI", + ), + # The flip removed --x-time-abi-v1; passing it is reported as is. + ( + completed("", 2, "error: unexpected argument '--x-time-abi-v1' found"), + "unexpected argument '--x-time-abi-v1' found", + ), + (completed(line.replace("1000000000", "200000000")), "[E_LAPIC_RATE_"), + ( + completed(line.replace("tsc_hz=2793437000", "tsc_hz=4000", 1)), + "[E_TSC_RATE_", + ), + ( + completed(line.replace("backend=kvm", "backend=mshv")), + "verified backend mshv", + ), + # Without H2 in the same run, H3 still requires a catalog profile. + ( + completed(line.replace(" cpu_profile=intel.icelake-sp.v1", "")), + "verified no catalog CPU profile: cpu_profile=none", + ), + ( + completed(line.replace("intel.icelake-sp.v1", "interim.host.kvm.v1")), + "verified no catalog CPU profile: cpu_profile=interim.host.kvm.v1", + ), + # Qualification never accepts a host profile, such as the one to + # which auto falls back on a CPU that no built-in profile serves. + ( + completed(line.replace("intel.icelake-sp.v1", "intel.host.v1")), + "verified no catalog CPU profile: cpu_profile=intel.host.v1 (a host " + "profile, to which --cpu-profile auto falls back on a CPU that no " + "built-in profile serves, and which qualification never accepts)", + ), + ) + for result_value, message in cases: + with self.subTest(message=message): + with patch.object(doctor.subprocess, "run", return_value=result_value): + result = doctor.check_openvmm_preflight(context_with_files()) + self.assertFalse(result.passed) + self.assertIn(message, result.detail) + # H3 records a missing profile as none, as H2 does. + context = context_with_files() + unprofiled = line.replace(" cpu_profile=intel.icelake-sp.v1", "") + with patch.object(doctor.subprocess, "run", return_value=completed(unprofiled)): + self.assertFalse(doctor.check_openvmm_preflight(context).passed) + self.assertEqual(context.facts["profile"], "none") + for selected, passed in ( + ("intel.icelake-sp.v2", True), + ("intel.skylake-sp.v1", False), + # A revision is the lineage plus .v and a number. + ("intel.icelake-sp.v2-rc", False), + # OpenVMM selects catalog profiles, so an interim one is a regression. + ("interim.host.kvm.v1", False), + ): + with self.subTest(selected=selected): + context = context_with_files() + context.facts["profile"] = "intel.icelake-sp.v1" + output = line.replace("intel.icelake-sp.v1", selected) + with patch.object( + doctor.subprocess, "run", return_value=completed(output) + ): + result = doctor.check_openvmm_preflight(context) + self.assertEqual(result.passed, passed, result.detail) + self.assertEqual(context.facts["profile"], selected) + missing = doctor.check_openvmm_preflight(doctor_context(self.root / "none")) + self.assertFalse(missing.passed) + self.assertIn("was not found", missing.detail) + # Without H2, a catalog profile of a generation that only OpenVMM pins + # passes. + with patch.object( + doctor.subprocess, + "run", + return_value=completed( + line.replace("intel.icelake-sp.v1", "intel.raptorlake.v1") + ), + ): + result = doctor.check_openvmm_preflight(context_with_files()) + self.assertTrue(result.passed, result.detail) + + def test_rate_check_applies_the_spec_bounds_to_sleep_separated_samples(self): + def records( + rates: list[float], + *, + disciplined: list[float] | None = None, + interval_s: float = 10.0, + bracket: int = 40, + clock_hz: int = 1_000_000_000, + resolution_ns: float = 1.0, + names: tuple[str, str] = ("CLOCK_MONOTONIC", "CLOCK_MONOTONIC_RAW"), + ) -> list[tuple[str, dict[str, str]]]: + """Probe records whose consecutive samples measure ``rates`` against + the stability clock and ``disciplined`` (default ``rates``) against + the rate clock.""" + result: list[tuple[str, dict[str, str]]] = [] + for role, name in zip(("rate", "stability"), names, strict=True): + clock = {"role": role, "name": name, "hz": str(clock_hz)} + clock["resolution_ns"] = f"{resolution_ns:.3f}" + result.append(("clock", clock)) + series = {"rate": disciplined or rates, "stability": rates} + state = {role: (10**12, 5 * clock_hz) for role in series} + for index in range(len(rates) + 1): + sample = {"index": str(index)} + for role, role_rates in series.items(): + tsc, ticks = state[role] + if index: + tsc += round(role_rates[index - 1] * interval_s) + ticks += round(interval_s * clock_hz) + state[role] = (tsc, ticks) + sample[f"{role}_tsc"] = str(tsc) + sample[f"{role}_clock"] = str(ticks) + sample[f"{role}_bracket"] = str(bracket) + result.append(("sample", sample)) + return result + + def check( + context: doctor.DoctorContext, + probe_records: list[tuple[str, dict[str, str]]], + clocksource: str = "tsc", + windows: bool = False, + ) -> doctor.CheckResult: + with ( + patch.object(doctor, "run_probe", return_value=probe_records) as run, + patch.object(doctor, "host_clocksource", return_value=clocksource), + patch.object(doctor, "host_is_windows", return_value=windows), + ): + result = doctor.check_rate(context) + self.assertEqual( + run.call_args.args[1:], + ( + "rate", + "--samples", + str(context.schedule.rate_samples), + "--interval-ms", + str(context.schedule.rate_interval_ms), + ), + ) + # The probe sleeps through its sampling window, which the timeout + # allows on top of --timeout (120 s on the qualification schedule). + self.assertEqual( + run.call_args.kwargs, + { + "duration": (context.schedule.rate_samples - 1) + * context.schedule.rate_interval_ms + / 1000 + }, + ) + return result + + def ci_context(backend: str = "kvm") -> doctor.DoctorContext: + context = doctor_context(self.root, backend) + context.schedule = doctor.CI_SCHEDULE + return context + + steady = [2793437000.0 + jitter for jitter in (0, 200, -150, 100) * 3] + context = doctor_context(self.root) + context.facts["tsc_hz"] = "2793437000" + result = check(context, records(steady)) + self.assertTrue(result.passed, result.detail) + self.assertIn( + "against CLOCK_MONOTONIC from 13 samples 10 s apart; against " + "CLOCK_MONOTONIC_RAW (resolution 1 ns) the interval rates agree within " + "0.125 ppm, largest interval uncertainty 0.004 ppm", + result.detail, + ) + self.assertIn("0.0 ppm from the backend's 2793437000 Hz", result.detail) + self.assertIn("host clocksource tsc (evidence)", result.detail) + self.assertEqual(context.facts["rate_agreement_ppm"], "0.125") + self.assertEqual(context.facts["rate_deviation_ppm"], "0.0") + # chrony steers CLOCK_MONOTONIC's rate by ppm between seconds, which + # says nothing about the TSC: the undisciplined clock judges stability, + # and the disciplined one the rate. + steered = [2300000100.0 + jitter for jitter in (0, 6900, -4600)] * 4 + context = doctor_context(self.root, "mshv") + context.facts["native_tsc_hz"] = "2300000000" + result = check(context, records([2300000100.0] * 12, disciplined=steered)) + self.assertTrue(result.passed, result.detail) + self.assertIn("agree within 0.000 ppm", result.detail) + self.assertEqual(context.facts["measured_tsc_hz"], "2300000867") + self.assertIn("0.4 ppm from the backend's 2300000000 Hz", result.detail) + # CI's short schedule; Windows reads QueryPerformanceCounter, which time + # synchronization never steers, in both roles. Its 100 ns tick keeps a + # 1 s interval conclusive at 0.1 ppm. + qpc = records( + [2300000100.0, 2300000200.0], + interval_s=1.0, + bracket=0, + clock_hz=10_000_000, + resolution_ns=100.0, + names=("QueryPerformanceCounter", "QueryPerformanceCounter"), + ) + windows = check(ci_context("whp"), qpc, windows=True) + self.assertTrue(windows.passed, windows.detail) + self.assertIn( + "against QueryPerformanceCounter from 3 samples 1 s apart", windows.detail + ) + self.assertIn("uncertainty 0.100 ppm", windows.detail) + self.assertNotIn("clocksource", windows.detail) + # Hyper-V's reference TSC page advances CLOCK_MONOTONIC_RAW in 100 ns + # steps: the resolution, not the empty brackets, bounds the measurement. + tsc_page = records( + [2300000100.0, 2300000200.0], interval_s=1.0, bracket=0, resolution_ns=100.0 + ) + result = check(ci_context("mshv"), tsc_page, "hyperv_clocksource_tsc_page") + self.assertTrue(result.passed, result.detail) + self.assertIn("CLOCK_MONOTONIC_RAW (resolution 100 ns)", result.detail) + self.assertIn("uncertainty 0.100 ppm", result.detail) + coarse = records( + [2300000100.0, 2300000200.0], interval_s=1.0, bracket=0, resolution_ns=300.0 + ) + result = check(ci_context("mshv"), coarse) + self.assertFalse(result.passed) + self.assertIn( + "inconclusive: an interval's uncertainty is 0.300 ppm", result.detail + ) + for probe_records, message in ( + ( + records([2793437000.0, 2793440000.0] * 6), + "unstable: interval rates differ by 1.074 ppm > 1.0 ppm", + ), + ( + records([2793437000.0] * 2, interval_s=1.0, bracket=400), + "inconclusive: an interval's uncertainty is 0.401 ppm > 0.25 ppm", + ), + (records([2794000000.0] * 12), "201.5 ppm from the measured rate"), + (records([100_000_000.0] * 12), "[E_TSC_RATE_IMPLAUSIBLE]"), + ): + with self.subTest(message=message): + context = doctor_context(self.root) + if "inconclusive" in message: + context = ci_context() + context.facts["tsc_hz"] = "2793437000" + result = check(context, probe_records) + self.assertFalse(result.passed) + self.assertIn(message, result.detail) + with self.assertRaisesRegex(doctor.ScriptError, "printed 3 samples, not 13"): + check(doctor_context(self.root), records([2793437000.0] * 2)) + with self.assertRaisesRegex(doctor.ScriptError, "no stability clock"): + check(ci_context(), records([2793437000.0] * 2, interval_s=1.0)[:1]) + # The clocksource is evidence on every backend. + for backend, clocksource in ( + ("kvm", "hpet"), + ("mshv", "hyperv_clocksource_tsc_page"), + ): + with self.subTest(backend=backend, clocksource=clocksource): + context = doctor_context(self.root, backend) + context.facts["native_tsc_hz"] = "2300000000" + result = check(context, records([2300000100.0] * 12), clocksource) + self.assertTrue(result.passed, result.detail) + self.assertIn( + f"host clocksource {clocksource} (evidence)", result.detail + ) + self.assertEqual(context.facts["host_clocksource"], clocksource) + # Without H3 (validate-runner), H4 still checks stability. + standalone = check(ci_context(), records([2300000100.0] * 2, interval_s=1.0)) + self.assertTrue(standalone.passed, standalone.detail) + self.assertIn("not compared with the backend's rate", standalone.detail) + + def test_rate_check_continues_when_the_clocksource_is_unreadable(self): + # #507: the host clocksource is evidence only, so an unreadable + # sysfs file must not fail H4; the native probe still runs and the + # verdict depends on the measurements alone. + def records(gaps: tuple[int, int]) -> list[tuple[str, dict[str, str]]]: + result: list[tuple[str, dict[str, str]]] = [ + ( + "clock", + { + "role": role, + "name": name, + "hz": "1000000000", + "resolution_ns": "1.000", + }, + ) + for role, name in ( + ("rate", "CLOCK_MONOTONIC"), + ("stability", "CLOCK_MONOTONIC_RAW"), + ) + ] + tsc = 0 + for index in range(3): + sample = {"index": str(index)} + for role in ("rate", "stability"): + sample[f"{role}_tsc"] = str(tsc) + sample[f"{role}_clock"] = str(index * 1_000_000_000) + sample[f"{role}_bracket"] = "20" + tsc += gaps[min(index, 1)] + result.append(("sample", sample)) + return result + + error = FileNotFoundError(2, "No such file or directory", "/sys/fake") + for gaps, passed, marker in ( + ((3_000_000_000, 3_000_000_000), True, "host clocksource unknown"), + ((3_000_000_000, 3_100_000_000), False, "interval rates differ"), + ): + with self.subTest(passed=passed): + context = doctor_context(self.root) + context.schedule = doctor.CI_SCHEDULE + with ( + patch.object( + doctor, "run_probe", return_value=records(gaps) + ) as run, + patch.object(doctor, "host_clocksource", side_effect=error), + patch.object(doctor, "host_is_windows", return_value=False), + ): + result = doctor.check_rate(context) + self.assertTrue(run.called) + self.assertEqual(result.passed, passed, result.detail) + self.assertIn(marker, result.detail) + self.assertIn("evidence", result.detail) + self.assertEqual( + context.facts["host_clocksource"], + f"unknown ({error})", + ) + + def test_host_skew_check_applies_the_bound(self): + summary = { + "pairs": "28", + "cpus": "0-7", + "max_abs_offset_ns": "40", + "max_uncertainty_ns": "120", + "stalled_pairs": "0", + "conclusive": "1", + } + context = doctor_context(self.root) + context.facts["measured_tsc_hz"] = "2793437000" + with patch.object(doctor, "run_probe", return_value=[("skew", summary)]) as run: + self.assertTrue(doctor.check_host_skew(context).passed) + self.assertIn("--tsc-hz", run.call_args.args) + for changes, message in ( + ({"max_abs_offset_ns": "1500"}, "exceeds 1000"), + ({"stalled_pairs": "2", "conclusive": "0"}, "2 CPU pair(s) stalled"), + ({"conclusive": "0"}, "inconclusive"), + ): + with ( + self.subTest(message=message), + patch.object( + doctor, "run_probe", return_value=[("skew", summary | changes)] + ), + ): + result = doctor.check_host_skew(doctor_context(self.root)) + self.assertFalse(result.passed) + self.assertIn(message, result.detail) + + def test_guest_warp_check_runs_the_idle_schedules_and_applies_the_bound(self): + context = doctor_context(self.root) + for path in (context.openvmm, context.kernel, context.initrd): + path.write_bytes(b"") + + def boot(cpus: int) -> str: + line = BOOT_LINE.replace("cpus=4", f"cpus={cpus}") + return " ALPINE-MICROVM-BOOT-OK: 3.22.1\n" + line + STATUS_OK + + smp = boot(8) + warp_output(pairs=28) * 4 + warp_output(pairs=28, offset=60) + # A newer guest also reports the boot check's CPU time. + single = boot(1).replace("elapsed_us=1873", "elapsed_us=1873 cpu_us=1100") + single += warp_output(pairs=0, offset=0) + with ( + patch.object(doctor.os, "cpu_count", return_value=8), + patch.object( + doctor, + "run_guest_script", + side_effect=[{"text": smp}, {"text": single}], + ) as run, + ): + result = doctor.check_guest_warp(context) + self.assertTrue(result.passed, result.detail) + self.assertIn( + "vcpus=8 rounds=5 idle_gaps_s=0.1,1,5,1 boot_elapsed_us=1873;", + result.detail, + ) + self.assertIn( + "vcpus=1 rounds=1 idle_gaps_s=none boot_elapsed_us=1873 boot_cpu_us=1100", + result.detail, + ) + self.assertEqual(context.facts["guest_warp_ns"], "60") + (smp_call, single_call) = run.call_args_list + for call, processors, gaps in ( + (smp_call, "8", time_abi.QUALIFICATION_WARP_GAPS), + (single_call, "1", ()), + ): + command, script, marker = call.args + self.assertEqual(command[command.index("--processors") + 1], processors) + self.assertEqual(script, time_abi.warp_probe_script(gaps) + "nvx-exit 0\n") + self.assertEqual(marker, time_abi.WARP_PROBE_COMPLETION_MARKER) + self.assertIs(call.kwargs["time_abi_status"], True) + # CI's schedule runs the probe twice with a 1 s gap. + context.schedule = doctor.CI_SCHEDULE + ci = boot(8) + warp_output(pairs=28) * 2 + with ( + patch.object(doctor.os, "cpu_count", return_value=8), + patch.object( + doctor, + "run_guest_script", + side_effect=[{"text": ci}, {"text": single}], + ), + ): + result = doctor.check_guest_warp(context) + self.assertTrue(result.passed, result.detail) + self.assertIn("vcpus=8 rounds=2 idle_gaps_s=1 ", result.detail) + context.schedule = doctor.QUALIFICATION_SCHEDULE + cases = ( + ( + [boot(8) + warp_output(pairs=28) * 2], + "vcpus=8", + "ran 2 rounds instead of 5", + ), + ( + [ + boot(8) + + warp_output(pairs=28) * 4 + + warp_output(pairs=28, offset=1500) + ], + "vcpus=8", + "in round 5: max_abs_offset_ns=1500", + ), + ( + [STATUS_OK + warp_output(pairs=28) * 5], + "vcpus=8", + "NVX-TIME-ABI boot marker", + ), + ( + [BOOT_LINE.replace("cpus=4", "cpus=8") + warp_output(pairs=28) * 5], + "vcpus=8", + "nvx-time status did not finish before the warp probe", + ), + ( + [smp, boot(1) + warp_output(pairs=0, backward=2000, verdict="FAIL")], + "vcpus=1", + "max_backward_ns=2000", + ), + ) + for outputs, guest, message in cases: + with ( + self.subTest(message=message), + patch.object(doctor.os, "cpu_count", return_value=8), + patch.object( + doctor, + "run_guest_script", + side_effect=[{"text": output} for output in outputs], + ), + ): + result = doctor.check_guest_warp(context) + self.assertFalse(result.passed) + self.assertTrue(result.detail.startswith(f"{guest}: "), result.detail) + self.assertIn(message, result.detail) + # A 1-vCPU host runs only the 1-vCPU guest. + with ( + patch.object(doctor.os, "cpu_count", return_value=1), + patch.object( + doctor, "run_guest_script", return_value={"text": single} + ) as run, + ): + result = doctor.check_guest_warp(context) + self.assertTrue(result.passed, result.detail) + self.assertEqual(run.call_count, 1) + self.assertTrue(result.detail.startswith("vcpus=1 rounds=1 idle_gaps_s=none")) + missing = doctor.check_guest_warp(doctor_context(self.root / "missing")) + self.assertIn("OpenVMM was not found", missing.detail) + + def test_utc_check_reads_the_host_clock_discipline(self): + context = doctor_context(self.root) + with patch.object(doctor, "host_is_windows", return_value=False): + with patch.object(doctor, "_linux_clock_state", return_value=(0, 0x2001)): + self.assertTrue(doctor.check_utc(context).passed) + with patch.object(doctor, "_linux_clock_state", return_value=(5, 0x0041)): + self.assertIn("STA_UNSYNC", doctor.check_utc(context).detail) + status = ( + "Leap Indicator: 0(no warning)\nStratum: 2\n" + "Source: VM IC Time Synchronization Provider\n" + ) + with ( + patch.object(doctor, "host_is_windows", return_value=True), + patch.object(doctor.subprocess, "run", return_value=completed(status)), + ): + result = doctor.check_utc(context) + self.assertTrue(result.passed, result.detail) + self.assertIn("VM IC Time Synchronization Provider", result.detail) + for unsynchronized in ( + status.replace("0(no warning)", "3(not synchronized)"), + "Leap Indicator: 0\nSource: Local CMOS Clock\n", + ): + with ( + patch.object(doctor, "host_is_windows", return_value=True), + patch.object( + doctor.subprocess, "run", return_value=completed(unsynchronized) + ), + ): + self.assertFalse(doctor.check_utc(context).passed) + # Output H7 can't read, such as a localized w32tm's, fails it with a + # stable leading phrase instead of passing as an unknown source. + unverified = "cannot verify host UTC synchronization: " + for outcome in ( + completed( + "Sprungindikator: 0(keine Warnung)\nQuelle: time.windows.com,0x9\n" + ), + completed("Leap Indicator: 0(no warning)\nStratum: 2\n"), + completed("Stratum: 2\nSource: VM IC Time Synchronization Provider\n"), + completed(status.replace("0(no warning)", "unknown")), + completed("The service has not been started. (0x80070426)\n", 1), + subprocess.TimeoutExpired(["w32tm"], 30), + ): + with ( + patch.object(doctor, "host_is_windows", return_value=True), + patch.object(doctor.subprocess, "run", side_effect=[outcome]), + contextlib.redirect_stdout(io.StringIO()), + ): + (result,) = doctor.run_checks(context, ("H7",)) + self.assertFalse(result.passed, result.detail) + self.assertTrue(result.detail.startswith(unverified), result.detail) + # So does an adjtimex clock state Linux doesn't define, or no adjtimex. + with ( + patch.object(doctor, "host_is_windows", return_value=False), + contextlib.redirect_stdout(io.StringIO()), + ): + with patch.object(doctor, "_linux_clock_state", return_value=(6, 0)): + (unknown_state,) = doctor.run_checks(context, ("H7",)) + with patch.object(doctor.ctypes, "CDLL", side_effect=OSError("no libc")): + (no_adjtimex,) = doctor.run_checks(context, ("H7",)) + for result in (unknown_state, no_adjtimex): + self.assertFalse(result.passed, result.detail) + self.assertTrue(result.detail.startswith(unverified), result.detail) + + def test_run_prints_lines_writes_the_summary_and_fails_closed(self): + def passing(check: str): + def run_check(context: doctor.DoctorContext) -> doctor.CheckResult: + context.facts["generation"] = "emeraldrapids" + return doctor.CheckResult(check, True, f"{check} ok") + + return run_check + + def broken(context: doctor.DoctorContext) -> doctor.CheckResult: + del context + raise doctor.ScriptError("rustc is required to build the host time probe") + + summary = self.root / "summary.md" + arguments = doctor_parser().parse_args( + [ + "--backend", + "mshv", + "--checks", + "H5", + "H2", + "--summary", + str(summary), + "--probe-dir", + str(self.root), + ] + ) + stdout = io.StringIO() + with ( + patch.dict(doctor.CHECKS, {"H2": passing("H2"), "H5": passing("H5")}), + contextlib.redirect_stdout(stdout), + ): + self.assertEqual(doctor.run(arguments), 0) + lines = stdout.getvalue().splitlines() + self.assertEqual( + [line.split()[1] for line in lines if line.startswith("NVX-DOCTOR")], + ["check=H2", "check=H5"], + ) + self.assertIn("passed; generation emeraldrapids", lines[-1]) + self.assertIn("(mshv): passed", summary.read_text(encoding="utf-8")) + with ( + patch.dict(doctor.CHECKS, {"H2": passing("H2"), "H5": broken}), + contextlib.redirect_stdout(io.StringIO()), + ): + self.assertEqual(doctor.run(arguments), 1) + text = summary.read_text(encoding="utf-8") + self.assertIn("**fail** | rustc is required", text) + self.assertIn("Generation: `emeraldrapids`", text) + + # Unreadable values that break a computation, such as a zero rate, + # fail only their own check, and the later checks still run. + def arithmetic(context: doctor.DoctorContext) -> doctor.CheckResult: + del context + raise ZeroDivisionError("float division by zero") + + stdout = io.StringIO() + with ( + patch.dict(doctor.CHECKS, {"H2": arithmetic, "H5": passing("H5")}), + contextlib.redirect_stdout(stdout), + ): + self.assertEqual(doctor.run(arguments), 1) + self.assertIn( + 'check=H2 status=fail detail="float division by zero"', stdout.getvalue() + ) + self.assertIn("check=H5 status=pass", stdout.getvalue()) + + def test_run_selects_the_cpu_fingerprint_or_runs_without_openvmm(self): + seen: list[Path | None] = [] + schedules: list[doctor.Schedule] = [] + + def cpu(context: doctor.DoctorContext) -> doctor.CheckResult: + seen.append(context.fingerprint) + schedules.append(context.schedule) + context.facts["surface_digest"] = "sha256:62" + return doctor.CheckResult("H2", True, "ok") + + summary = self.root / "summary.md" + common = ["--backend", "kvm", "--checks", "H2", "--summary", str(summary)] + common += ["--probe-dir", str(self.root)] + with ( + patch.dict(doctor.CHECKS, {"H2": cpu}), + contextlib.redirect_stdout(io.StringIO()), + ): + for extra in ( + [], + ["--cpu-fingerprint", str(self.root / "fingerprint.json")], + ["--no-openvmm", "--ci-schedule"], + ): + arguments = doctor_parser().parse_args([*common, *extra]) + self.assertEqual(doctor.run(arguments), 0) + self.assertEqual( + seen, + [ + self.root / "nvx-cpu-fingerprint-kvm.json", + self.root / "fingerprint.json", + None, + ], + ) + self.assertEqual( + schedules, + [doctor.QUALIFICATION_SCHEDULE] * 2 + [doctor.CI_SCHEDULE], + ) + self.assertEqual(doctor.QUALIFICATION_SCHEDULE.rate_samples, 13) + self.assertEqual(doctor.QUALIFICATION_SCHEDULE.rate_interval_ms, 10_000) + self.assertEqual(doctor.CI_SCHEDULE.rate_samples, 3) + self.assertEqual(doctor.CI_SCHEDULE.rate_interval_ms, 1_000) + self.assertIn("CPU surface digest: `sha256:62`", summary.read_text("utf-8")) + arguments = doctor_parser().parse_args(["--backend", "kvm", "--no-openvmm"]) + with self.assertRaisesRegex(doctor.ScriptError, "cannot run H3 and H6"): + doctor.run(arguments) + with contextlib.redirect_stderr(io.StringIO()), self.assertRaises(SystemExit): + doctor_parser().parse_args( + ["--backend", "kvm", "--no-openvmm", "--cpu-fingerprint", "x"] + ) + + def test_builds_the_host_probe_once_per_source_version(self): + target = doctor.probe_path(self.root) + self.assertRegex(target.name, r"^nvx-host-time-probe-[0-9a-f]{16}(\.exe)?$") + + def rustc(command: list[str], **_kwargs: object): + Path(command[command.index("-o") + 1]).write_bytes(b"probe") + return completed("") + + with patch.object( + doctor, "require_tool", side_effect=doctor.ScriptError("rustc is required") + ) as require_tool: + with self.assertRaisesRegex(doctor.ScriptError, "rustc is required"): + doctor.build_probe(self.root) + require_tool.assert_called_once_with( + "rustc", "rustc is required to build the host time probe" + ) + with ( + patch.object(doctor, "require_tool", return_value="rustc"), + patch.object(doctor.subprocess, "run", side_effect=rustc) as run, + ): + self.assertEqual(doctor.build_probe(self.root), target) + self.assertEqual(doctor.build_probe(self.root), target) + run.assert_called_once() + self.assertEqual(target.read_bytes(), b"probe") + target.unlink() + with ( + patch.object(doctor, "require_tool", return_value="rustc"), + patch.object( + doctor.subprocess, "run", return_value=completed("", 1, "error[E0425]") + ), + ): + with self.assertRaisesRegex(doctor.ScriptError, "E0425"): + doctor.build_probe(self.root) + + def test_parses_probe_records(self): + context = doctor_context(self.root) + context.probe = self.root / "probe.exe" + output = ( + "noise\nNVX-HOST-TIME-PROBE rate index=1 tsc_hz=1.5\n" + "NVX-HOST-TIME-PROBE skew pairs=1 cpus=0-1\n" + ) + with patch.object( + doctor.subprocess, "run", return_value=completed(output) + ) as run: + records = doctor.run_probe(context, "skew") + self.assertEqual(run.call_args.kwargs["timeout"], context.timeout) + self.assertEqual( + records, + [ + ("rate", {"index": "1", "tsc_hz": "1.5"}), + ("skew", {"pairs": "1", "cpus": "0-1"}), + ], + ) + with patch.object( + doctor.subprocess, "run", return_value=completed(output) + ) as run: + doctor.run_probe(context, "rate", duration=120.0) + self.assertEqual(run.call_args.kwargs["timeout"], context.timeout + 120.0) + with patch.object( + doctor.subprocess, "run", return_value=completed("", 3, "no CPU") + ): + with self.assertRaisesRegex(doctor.ScriptError, "skew failed: no CPU"): + doctor.run_probe(context, "skew") + + +def doctor_parser(): + import argparse + + parser = argparse.ArgumentParser() + doctor.configure_parser(parser) + return parser + + +if __name__ == "__main__": + unittest.main() From 6626c13d1d4cd2711351d0f924974eb7cf494a42 Mon Sep 17 00:00:00 2001 From: Akhil Gupta Date: Mon, 12 Oct 2026 04:59:32 -0700 Subject: [PATCH 2/5] Delete scripts/nvx_tools/doctor.py --- scripts/nvx_tools/doctor.py | 1192 ----------------------------------- 1 file changed, 1192 deletions(-) delete mode 100644 scripts/nvx_tools/doctor.py diff --git a/scripts/nvx_tools/doctor.py b/scripts/nvx_tools/doctor.py deleted file mode 100644 index 42973e6fa..000000000 --- a/scripts/nvx_tools/doctor.py +++ /dev/null @@ -1,1192 +0,0 @@ -"""Host qualification for the NVX time ABI (``nvx.py doctor``). - -doc/design/time-abi.md ("Host qualification") defines checks H1 to H7. Each -check prints one ``NVX-DOCTOR: check= status= detail="..."`` -line, and qualification fails closed if any selected check fails. A failure -that matches a time ABI failure code starts its detail with that code in -brackets, as OpenVMM errors do. -""" - -from __future__ import annotations - -import argparse -import ctypes -import hashlib -import itertools -import os -import platform -import re -import subprocess -import sys -from collections.abc import Callable, Mapping, Sequence -from dataclasses import dataclass, field -from pathlib import Path - -from .benchmark import run_guest_script, workload_boot_command -from .build_constants import AlpineBuildConstants, BuildConstants, KernelBuildConstants -from .ci import OPENVMM_TEST_BACKENDS -from .common import ( - ScriptError, - artifact_path, - openvmm_binary_path, - require_tool, -) -from .time_abi import ( - CI_WARP_GAPS, - LAPIC_HZ, - MAX_TSC_HZ, - MIN_TSC_HZ, - QUALIFICATION_WARP_GAPS, - WARP_BOUND_NS, - WARP_PROBE_COMPLETION_MARKER, - HostCpu, - TimeAbiFailure, - TimeAbiMonitor, - check_warp_probe, - cpu_generation, - describe_cpu_generations, - is_catalog_profile_id, - is_host_profile_id, - parse_fields, - warp_probe_script, - warp_rounds, -) - -CHECK_IDS = ("H1", "H2", "H3", "H4", "H5", "H6", "H7") -CHECK_TITLES: Mapping[str, str] = { - "H1": "backend device or API", - "H2": "CPU fingerprint, generation, and profile", - "H3": "OpenVMM time ABI preflight", - "H4": "host TSC rate stability", - "H5": "host cross-CPU TSC skew", - "H6": "guest warp probe", - "H7": "host UTC synchronization", -} -# Checks that boot OpenVMM; H2 also runs it unless qualification has none. -OPENVMM_CHECKS = ("H3", "H6") -DOCTOR_PREFIX = "NVX-DOCTOR: " -PROBE_SOURCE = Path(__file__).with_name("host_time_probe.rs") -PROBE_PREFIX = "NVX-HOST-TIME-PROBE " -VERIFY_PREFIX = "NVX-TIME-ABI-VERIFY:" -CPU_PROFILE_PREFIX = "NVX-CPU-PROFILE:" -RUST_ESCAPE = re.compile(r"\\(u\{[0-9a-fA-F]{1,6}\}|.)") -# doc/design/time-abi.md, "Rate stability (H4)". -RATE_UNCERTAINTY_PPM = 0.25 -RATE_AGREEMENT_PPM = 1.0 -RATE_TOLERANCE_PPM = 100.0 -SKEW_PAIR_DURATION_MS = 5 -GUEST_VCPU_COUNTS = (8, 4, 2, 1) -GUEST_MEMORY_MIB = 128 - - -@dataclass(frozen=True) -class Schedule: - """How long H4 samples the TSC rate and how often H6 runs the warp probe.""" - - rate_samples: int - rate_interval_ms: int - warp_gaps: tuple[str, ...] - - -# doc/design/time-abi.md, "Rate stability (H4)" and "Warp schedules": doctor -# qualifies on the long schedules, and CI on the short ones. -QUALIFICATION_SCHEDULE = Schedule(13, 10_000, QUALIFICATION_WARP_GAPS) -CI_SCHEDULE = Schedule(3, 1_000, CI_WARP_GAPS) -# Qualification gates only on measured properties, alike on every backend: -# the guest warp probe, the TSC rate stability, and the CPU profile. The host -# OS's invariant-TSC flags and clocksource are recorded as evidence only, -# because they don't decide what a guest observes: Azure WHP hosts show the -# CPUID bit but cannot offer invariant TSC to partitions, and their guests -# measure tens of nanoseconds of skew. -LINUX_INVARIANT_TSC_FLAGS = ("constant_tsc", "nonstop_tsc") -CLOCKSOURCE_PATH = Path( - "/sys/devices/system/clocksource/clocksource0/current_clocksource" -) -CPUINFO_PATH = Path("/proc/cpuinfo") -# The processor that Windows reports, such as -# "Intel64 Family 6 Model 154 Stepping 3, GenuineIntel": the display family, -# model, and stepping, and the CPUID vendor. -WINDOWS_PROCESSOR = re.compile(r"Family (\d+) Model (\d+) Stepping (\d+), (\S+)") -# The registry key of the first processor's identity on Windows. -WINDOWS_PROCESSOR_KEY = r"HARDWARE\DESCRIPTION\System\CentralProcessor\0" -ADJTIMEX_TIME_ERROR = 5 -ADJTIMEX_STA_UNSYNC = 0x0040 -# Every H7 failure to read the host's synchronization state starts with this -# phrase; a clock that is not synchronized starts "host UTC is not -# synchronized" instead. -UTC_UNVERIFIED = "cannot verify host UTC synchronization" -WHP_CAPABILITY_HYPERVISOR_PRESENT = 0 - - -@dataclass -class CheckResult: - check: str - passed: bool - detail: str - - def line(self) -> str: - status = "pass" if self.passed else "fail" - detail = self.detail.replace("\\", "\\\\").replace('"', '\\"') - return f'{DOCTOR_PREFIX}check={self.check} status={status} detail="{detail}"' - - -@dataclass -class DoctorContext: - backend: str - openvmm: Path - kernel: Path - initrd: Path - openvmm_args: tuple[str, ...] - probe_directory: Path - timeout: float - facts: dict[str, str] = field(default_factory=dict[str, str]) - probe: Path | None = None - # Where H2 writes OpenVMM's CPU fingerprint; None skips the profile check - # when qualification runs without OpenVMM. - fingerprint: Path | None = None - schedule: Schedule = QUALIFICATION_SCHEDULE - - -def host_is_windows() -> bool: - return os.name == "nt" - - -def host_clocksource() -> str: - return CLOCKSOURCE_PATH.read_text(encoding="utf-8").strip() - - -def _escape_markdown(text: str) -> str: - return text.replace("|", "\\|").replace("\n", " ") - - -def probe_path(directory: Path) -> Path: - """Return the cached host probe binary for the current probe source.""" - digest = hashlib.sha256(PROBE_SOURCE.read_bytes()).hexdigest()[:16] - suffix = ".exe" if os.name == "nt" else "" - return directory / f"nvx-host-time-probe-{digest}{suffix}" - - -def build_probe(directory: Path) -> Path: - """Build the host probe once per source version with rustc.""" - target = probe_path(directory) - if target.is_file(): - return target - rustc = require_tool("rustc", "rustc is required to build the host time probe") - directory.mkdir(parents=True, exist_ok=True) - temporary = target.with_name(f".{target.stem}-{os.getpid()}{target.suffix}") - completed = subprocess.run( - [ - rustc, - "--edition=2021", - "-C", - "opt-level=2", - "-C", - "debuginfo=0", - "-o", - os.fspath(temporary), - os.fspath(PROBE_SOURCE), - ], - capture_output=True, - text=True, - check=False, - ) - if completed.returncode != 0: - temporary.unlink(missing_ok=True) - raise ScriptError( - f"rustc failed to build the host time probe:\n{completed.stderr.strip()}" - ) - try: - os.replace(temporary, target) - except OSError: - # Another job built the same version first and may be running it. - temporary.unlink(missing_ok=True) - if not target.is_file(): - raise - temporary.with_suffix(".pdb").unlink(missing_ok=True) - return target - - -def run_probe( - context: DoctorContext, *arguments: str, duration: float = 0.0 -) -> list[tuple[str, dict[str, str]]]: - """Run the host probe and parse its ``NVX-HOST-TIME-PROBE`` records. - - ``duration`` is the time the probe spends sleeping by design, which the - timeout allows on top of ``context.timeout``. - """ - if context.probe is None: - context.probe = build_probe(context.probe_directory) - completed = subprocess.run( - [os.fspath(context.probe), *arguments], - capture_output=True, - text=True, - timeout=context.timeout + duration, - check=False, - ) - if completed.returncode != 0: - message = completed.stderr.strip() or f"exit status {completed.returncode}" - raise ScriptError(f"host time probe {arguments[0]} failed: {message}") - records: list[tuple[str, dict[str, str]]] = [] - for line in completed.stdout.splitlines(): - if line.startswith(PROBE_PREFIX): - kind, _, fields = line.removeprefix(PROBE_PREFIX).partition(" ") - records.append((kind, parse_fields(fields))) - return records - - -def _probe_record( - records: Sequence[tuple[str, dict[str, str]]], kind: str -) -> dict[str, str]: - for record_kind, fields in records: - if record_kind == kind: - return fields - raise ScriptError(f"host time probe printed no {kind} record") - - -def _linux_cpuinfo(path: Path = CPUINFO_PATH) -> dict[str, str]: - """Return the first processor's /proc/cpuinfo fields.""" - fields: dict[str, str] = {} - for line in path.read_text(encoding="utf-8").splitlines(): - if not line.strip(): - if fields: - break - continue - name, separator, value = line.partition(":") - if separator: - fields.setdefault(name.strip(), value.strip()) - return fields - - -def _windows_registry_value(key: str, name: str) -> object: - if sys.platform != "win32": - raise ScriptError("the Windows registry is unavailable") - import winreg - - with winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, key) as handle: - value, _ = winreg.QueryValueEx(handle, name) - return value - - -def host_cpu_signature() -> HostCpu | None: - """Return the host CPU's vendor and display family, model, and stepping, as - the host OS reports them, with its brand string where the OS reports one, - or None where they cannot be read.""" - try: - if host_is_windows(): - match = WINDOWS_PROCESSOR.search(platform.processor()) - if match is None: - return None - family, model, stepping, vendor = match.groups() - try: - brand = str( - _windows_registry_value( - WINDOWS_PROCESSOR_KEY, "ProcessorNameString" - ) - ).strip() - except (OSError, ScriptError): - brand = "" - else: - cpuinfo = _linux_cpuinfo() - vendor = cpuinfo["vendor_id"] - family = cpuinfo["cpu family"] - model = cpuinfo["model"] - stepping = cpuinfo["stepping"] - brand = cpuinfo.get("model name", "") - return HostCpu(vendor, int(family), int(model), int(stepping), brand) - except (OSError, KeyError, ValueError): - return None - - -def host_cpu(context: DoctorContext) -> dict[str, str]: - """Collect the host CPU identity and the host OS's view of its TSC.""" - if host_is_windows(): - match = WINDOWS_PROCESSOR.search(platform.processor()) - if match is None: - raise ScriptError(f"cannot parse the processor {platform.processor()!r}") - family, model, stepping, vendor = match.groups() - info = { - "vendor": vendor, - "family": family, - "model": model, - "stepping": stepping, - "brand": str( - _windows_registry_value(WINDOWS_PROCESSOR_KEY, "ProcessorNameString") - ).strip(), - } - revision = _windows_registry_value(WINDOWS_PROCESSOR_KEY, "Update Revision") - if isinstance(revision, bytes) and len(revision) in (4, 8): - info["microcode"] = hex(int.from_bytes(revision[-4:], "little")) - build = _windows_registry_value( - r"SOFTWARE\Microsoft\Windows NT\CurrentVersion", "UBR" - ) - info["os"] = f"Windows {platform.version()}.{build}" - try: - probed = _probe_record(run_probe(context, "cpu"), "cpu") - info["invariant_tsc"] = "yes" if probed["invariant_tsc"] == "1" else "no" - except (ScriptError, KeyError, subprocess.TimeoutExpired) as error: - info["invariant_tsc"] = f"unknown ({error})" - return info - cpuinfo = _linux_cpuinfo() - flags = cpuinfo.get("flags", "").split() - missing = [flag for flag in LINUX_INVARIANT_TSC_FLAGS if flag not in flags] - return { - "vendor": cpuinfo.get("vendor_id", "unknown"), - "family": cpuinfo.get("cpu family", "?"), - "model": cpuinfo.get("model", "?"), - "stepping": cpuinfo.get("stepping", "?"), - "microcode": cpuinfo.get("microcode", "unknown"), - "brand": cpuinfo.get("model name", "unknown"), - "os": f"Linux {platform.release()}", - "invariant_tsc": "yes" if not missing else f"no (missing {' '.join(missing)})", - } - - -def _whp_hypervisor_present() -> bool: - if sys.platform != "win32": - raise ScriptError("WHP qualification requires Windows") - try: - library = ctypes.WinDLL("WinHvPlatform.dll") - except OSError as error: - raise ScriptError(f"WinHvPlatform.dll is unavailable: {error}") from error - present = ctypes.c_int32(0) - written = ctypes.c_uint32(0) - result = int( - library.WHvGetCapability( - WHP_CAPABILITY_HYPERVISOR_PRESENT, - ctypes.byref(present), - ctypes.sizeof(present), - ctypes.byref(written), - ) - ) - if result != 0: - raise ScriptError( - f"WHvGetCapability failed with HRESULT 0x{result & 0xFFFFFFFF:08x}" - ) - return bool(present.value) - - -def check_backend(context: DoctorContext) -> CheckResult: - if context.backend == "whp": - if not host_is_windows(): - return CheckResult("H1", False, "WHP qualification requires Windows") - if not _whp_hypervisor_present(): - return CheckResult( - "H1", False, "the Windows Hypervisor Platform reports no hypervisor" - ) - return CheckResult("H1", True, "the Windows Hypervisor Platform is present") - if host_is_windows(): - return CheckResult( - "H1", False, f"{context.backend} qualification requires Linux" - ) - device = Path("/dev") / context.backend - if not device.exists(): - return CheckResult("H1", False, f"{device} does not exist") - if not os.access(device, os.R_OK | os.W_OK): - return CheckResult("H1", False, f"{device} is not readable and writable") - if context.backend == "kvm" and Path("/dev/mshv").exists(): - return CheckResult( - "H1", False, "/dev/mshv exists, so OpenVMM would select MSHV, not KVM" - ) - return CheckResult("H1", True, f"{device} is readable and writable") - - -def check_cpu(context: DoctorContext) -> CheckResult: - info = host_cpu(context) - signature = f"{info['vendor']} {info['family']}/{info['model']}/{info['stepping']}" - context.facts.update( - { - "cpu": signature, - "brand": info["brand"], - "microcode": info.get("microcode", "unknown"), - "os": info["os"], - "invariant_tsc": info["invariant_tsc"], - } - ) - problems: list[str] = [] - if context.fingerprint is None: - # Without OpenVMM, NVX's copy of OpenVMM's catalog maps the host. - try: - generation = cpu_generation( - info["vendor"], - int(info["family"]), - int(info["model"]), - int(info["stepping"]), - ) - except ValueError: - generation = None - name = generation.name if generation else "unknown" - profile = generation.profile_id if generation else "none" - if generation is None: - problems.append( - f"[E_PROFILE_HOST_UNKNOWN] {signature} is not a time ABI generation " - f"({describe_cpu_generations()})" - ) - profile_detail = "CPU profile not checked: qualification runs without OpenVMM" - else: - name, profile, profile_detail, profile_problems = _check_cpu_profile(context) - problems.extend(profile_problems) - context.facts.update({"generation": name, "profile": profile}) - detail = ( - f"generation={name} profile={profile} cpu={signature} " - f"microcode={info.get('microcode', 'unknown')} os={info['os']} " - f"invariant_tsc={info['invariant_tsc']} brand={info['brand']}; " - f"{profile_detail}" - ) - if problems: - return CheckResult("H2", False, "; ".join(problems) + f"; {detail}") - return CheckResult("H2", True, detail) - - -def parse_cpu_profile_line(line: str) -> dict[str, str]: - """Parse OpenVMM's ``NVX-CPU-PROFILE:`` line; ``detail`` is a Rust string.""" - text = line.strip().removeprefix(CPU_PROFILE_PREFIX) - head, quoted, detail = text.partition(' detail="') - fields = parse_fields(head) - if quoted: - fields["detail"] = RUST_ESCAPE.sub(_rust_unescape, detail.removesuffix('"')) - return fields - - -def _rust_unescape(match: re.Match[str]) -> str: - escape = match.group(1) - if escape.startswith("u{"): - return chr(int(escape[2:-1], 16)) - return {"n": "\n", "r": "\r", "t": "\t", "0": "\0"}.get(escape, escape) - - -def _same_profile_lineage(selected: str, expected: str) -> bool: - """Whether ``selected`` is a revision of the profile ``expected`` names.""" - lineage = re.escape(expected.rpartition(".v")[0]) - return re.fullmatch(rf"{lineage}\.v\d+", selected) is not None - - -def _check_cpu_profile(context: DoctorContext) -> tuple[str, str, str, list[str]]: - """Fingerprint the host with OpenVMM and check the profile that it - selects. Return the generation and the profile as OpenVMM reports them, - the detail, and the problems: OpenVMM's catalog, not NVX's copy, decides - which hosts it serves.""" - assert context.fingerprint is not None - if not context.openvmm.is_file(): - return ( - "unknown", - "none", - "CPU profile not checked", - [ - f"OpenVMM was not found at {context.openvmm}; H2 checks the CPU " - "profile with its --cpu-fingerprint tool (or pass --no-openvmm)" - ], - ) - context.fingerprint.parent.mkdir(parents=True, exist_ok=True) - environment = os.environ.copy() - environment["OPENVMM_LOG"] = "off" - completed = subprocess.run( - [ - os.fspath(context.openvmm), - "--hypervisor", - context.backend, - "--cpu-fingerprint", - os.fspath(context.fingerprint), - ], - capture_output=True, - text=True, - timeout=context.timeout, - env=environment, - check=False, - ) - output = f"{completed.stdout}\n{completed.stderr}" - line = next( - ( - line.strip() - for line in output.splitlines() - if line.strip().startswith(CPU_PROFILE_PREFIX) - ), - None, - ) - if line is None: - last = next( - (line.strip() for line in reversed(output.splitlines()) if line.strip()), - "no output", - ) - if "--cpu-fingerprint" in output and "unexpected argument" in output: - last = "this OpenVMM predates the --cpu-fingerprint tool" - return ( - "unknown", - "none", - "CPU profile not checked", - [ - f"OpenVMM's CPU fingerprint exited {completed.returncode} without " - f"an {CPU_PROFILE_PREFIX} line: {last}" - ], - ) - fields = parse_cpu_profile_line(line) - generation = fields.get("generation") or "none" - profile = fields.get("profile") or "none" - for name in ("profile_digest", "surface_digest"): - if fields.get(name, "none") != "none": - context.facts[name] = fields[name] - context.facts["cpu_fingerprint"] = os.fspath(context.fingerprint) - detail = ( - f"CPU profile check status={fields.get('status')} " - f"profile={profile} " - f"profile_digest={fields.get('profile_digest', 'none')} " - f"surface_digest={fields.get('surface_digest')} " - f"fingerprint={context.fingerprint}" - ) - problems: list[str] = [] - if completed.returncode != 0 or fields.get("status") != "pass": - code = fields.get("code", "") - prefix = f"[{code}] " if code.startswith("E_") else "" - problems.append( - f"{prefix}OpenVMM's CPU profile check failed (exit " - f"{completed.returncode}): {fields.get('detail', line)}" - ) - return generation, profile, detail, problems - if fields.get("backend") != context.backend: - problems.append(f"OpenVMM fingerprinted the {fields.get('backend')} backend") - unreported = [name for name in ("generation", "profile") if not fields.get(name)] - if unreported: - problems.append( - "OpenVMM's CPU profile check passed without reporting its " - + " and ".join(unreported) - ) - elif is_host_profile_id(profile): - problems.append( - f"OpenVMM reported host profile {profile}, which qualification " - "never accepts" - ) - elif not is_catalog_profile_id(profile): - problems.append( - f"OpenVMM reported CPU profile {profile}, which is not a catalog " - "profile ID (vendor.generation.vN)" - ) - elif profile.split(".")[1] != generation: - problems.append( - f"OpenVMM reported CPU profile {profile} of generation " - f"{profile.split('.')[1]} for generation {generation}" - ) - return generation, profile, detail, problems - - -def _guest_vcpus() -> int: - host_cpus = os.cpu_count() or 1 - return next(count for count in GUEST_VCPU_COUNTS if count <= host_cpus) - - -def check_openvmm_preflight(context: DoctorContext) -> CheckResult: - for path, description in ( - (context.openvmm, "OpenVMM"), - (context.kernel, "guest kernel"), - (context.initrd, "guest initramfs"), - ): - if not path.is_file(): - return CheckResult("H3", False, f"{description} was not found at {path}") - command = [ - os.fspath(context.openvmm), - "--machine", - "microvm", - "--processors", - str(_guest_vcpus()), - "--hypervisor", - context.backend, - "--kernel", - os.fspath(context.kernel), - "--initrd", - os.fspath(context.initrd), - *context.openvmm_args, - "--x-time-abi-verify", - ] - environment = os.environ.copy() - environment["OPENVMM_LOG"] = "off" - completed = subprocess.run( - command, - capture_output=True, - text=True, - timeout=context.timeout, - env=environment, - check=False, - ) - output = f"{completed.stdout}\n{completed.stderr}" - verify = next( - ( - line.strip() - for line in output.splitlines() - if line.strip().startswith(VERIFY_PREFIX) - ), - None, - ) - if verify is None: - last = next( - (line.strip() for line in reversed(output.splitlines()) if line.strip()), - "no output", - ) - if "unexpected argument '--x-time-abi-verify'" in output: - last = "this OpenVMM predates the time ABI's --x-time-abi-verify mode" - return CheckResult( - "H3", - False, - f"OpenVMM verification exited {completed.returncode} without a " - f"verification line: {last}", - ) - fields = parse_fields(verify.removeprefix(VERIFY_PREFIX).strip()) - if fields.get("status") != "ok" or completed.returncode != 0: - code = fields.get("code", "none") - prefix = f"[{code}] " if code.startswith("E_") else "" - return CheckResult( - "H3", - False, - f"{prefix}OpenVMM verification failed (exit {completed.returncode}): " - f"{fields.get('detail', verify)}", - ) - problems: list[str] = [] - if fields.get("backend") != context.backend: - problems.append(f"OpenVMM verified backend {fields.get('backend')}") - for name in ("tsc_hz", "native_tsc_hz"): - try: - rate = int(fields.get(name, "")) - if not MIN_TSC_HZ <= rate <= MAX_TSC_HZ: - problems.append(f"[E_TSC_RATE_IMPLAUSIBLE] {name}={rate}") - except ValueError: - problems.append(f"{name}={fields.get(name)!r} is not an integer") - expected_lapic = LAPIC_HZ[context.backend] - if fields.get("lapic_hz") != str(expected_lapic): - problems.append( - f"[E_LAPIC_RATE_MISMATCH] lapic_hz={fields.get('lapic_hz')} is not " - f"{expected_lapic}" - ) - expected_profile = context.facts.get("profile", "none") - selected_profile = fields.get("cpu_profile") or "none" - # A missing, host, or malformed profile fails even when H2 doesn't run - # with H3. Which generations have profiles is OpenVMM's to decide. - if not is_catalog_profile_id(selected_profile): - problems.append( - "OpenVMM verified no catalog CPU profile: " - f"cpu_profile={selected_profile}" - + ( - " (a host profile, to which --cpu-profile auto falls back on a " - "CPU that no built-in profile serves, and which qualification " - "never accepts)" - if is_host_profile_id(selected_profile) - else "" - ) - ) - elif expected_profile != "none" and not _same_profile_lineage( - selected_profile, expected_profile - ): - problems.append( - f"OpenVMM selected CPU profile {selected_profile}, but H2 maps the " - f"host to {expected_profile}" - ) - detail = " ".join(f"{name}={value}" for name, value in fields.items()) - context.facts.update( - { - "tsc_hz": fields.get("tsc_hz", ""), - "native_tsc_hz": fields.get("native_tsc_hz", ""), - "lapic_hz": fields.get("lapic_hz", ""), - "profile": selected_profile, - "route": fields.get("msr_route", ""), - "sync": fields.get("sync", ""), - } - ) - if problems: - return CheckResult("H3", False, "; ".join(problems) + f"; {detail}") - return CheckResult("H3", True, detail) - - -def _rate_points( - samples: Sequence[Mapping[str, str]], clock: Mapping[str, str], role: str -) -> list[tuple[int, int, float]]: - """Return each sample's (TSC, bracket midpoint in half clock ticks, - uncertainty in clock ticks: half the bracket plus half the resolution).""" - half_resolution = float(clock["resolution_ns"]) * int(clock["hz"]) / 2e9 - return [ - ( - int(sample[f"{role}_tsc"]), - 2 * int(sample[f"{role}_clock"]) + int(sample[f"{role}_bracket"]), - int(sample[f"{role}_bracket"]) / 2 + half_resolution, - ) - for sample in samples - ] - - -def _window_rate(points: Sequence[tuple[int, int, float]], clock_hz: int) -> float: - seconds = (points[-1][1] - points[0][1]) / (2 * clock_hz) - if seconds <= 0: - raise ScriptError("the host clock did not advance between samples") - return (points[-1][0] - points[0][0]) / seconds - - -def check_rate(context: DoctorContext) -> CheckResult: - clocksource = "" - if not host_is_windows(): - clocksource = host_clocksource() - context.facts["host_clocksource"] = clocksource - schedule = context.schedule - # The probe sleeps between samples for the whole window, 120 s on the - # qualification schedule, so the timeout covers the window as well. - records = run_probe( - context, - "rate", - "--samples", - str(schedule.rate_samples), - "--interval-ms", - str(schedule.rate_interval_ms), - duration=(schedule.rate_samples - 1) * schedule.rate_interval_ms / 1000, - ) - clocks = {fields.get("role"): fields for kind, fields in records if kind == "clock"} - for role in ("rate", "stability"): - if role not in clocks: - raise ScriptError(f"host time probe printed no {role} clock") - samples = [fields for kind, fields in records if kind == "sample"] - if len(samples) != schedule.rate_samples: - raise ScriptError( - f"host time probe printed {len(samples)} samples, not " - f"{schedule.rate_samples}" - ) - # The rate clock, which time synchronization disciplines, gives the true - # rate. The stability clock, which it never steers, shows whether the TSC - # keeps its rate across idle: chrony's frequency updates move - # CLOCK_MONOTONIC's rate by several ppm between seconds. - rate_clock, stability_clock = clocks["rate"], clocks["stability"] - measured = _window_rate( - _rate_points(samples, rate_clock, "rate"), int(rate_clock["hz"]) - ) - clock_hz = int(stability_clock["hz"]) - points = _rate_points(samples, stability_clock, "stability") - rates: list[float] = [] - uncertainties_ppm: list[float] = [] - for (tsc0, mid0, error0), (tsc1, mid1, error1) in itertools.pairwise(points): - if mid1 <= mid0: - raise ScriptError("the host clock did not advance between samples") - seconds = (mid1 - mid0) / (2 * clock_hz) - rates.append((tsc1 - tsc0) / seconds) - uncertainties_ppm.append((error0 + error1) / clock_hz / seconds * 1e6) - stable_rate = _window_rate(points, clock_hz) - agreement_ppm = (max(rates) - min(rates)) / stable_rate * 1e6 - uncertainty_ppm = max(uncertainties_ppm) - context.facts["measured_tsc_hz"] = f"{measured:.0f}" - context.facts["rate_agreement_ppm"] = f"{agreement_ppm:.3f}" - context.facts["rate_uncertainty_ppm"] = f"{uncertainty_ppm:.3f}" - detail = ( - f"tsc_hz={measured:.0f} against {rate_clock['name']} from {len(samples)} " - f"samples {schedule.rate_interval_ms / 1000:g} s apart; against " - f"{stability_clock['name']} (resolution " - f"{float(stability_clock['resolution_ns']):g} ns) the interval rates agree " - f"within {agreement_ppm:.3f} ppm, largest interval uncertainty " - f"{uncertainty_ppm:.3f} ppm" - ) - problems: list[str] = [] - if not MIN_TSC_HZ <= measured <= MAX_TSC_HZ: - problems.append(f"[E_TSC_RATE_IMPLAUSIBLE] measured {measured:.0f} Hz") - if uncertainty_ppm > RATE_UNCERTAINTY_PPM: - problems.append( - f"the measurement is inconclusive: an interval's uncertainty is " - f"{uncertainty_ppm:.3f} ppm > {RATE_UNCERTAINTY_PPM} ppm" - ) - if agreement_ppm > RATE_AGREEMENT_PPM: - problems.append( - f"the TSC rate is unstable: interval rates differ by " - f"{agreement_ppm:.3f} ppm > {RATE_AGREEMENT_PPM} ppm" - ) - declared = context.facts.get("native_tsc_hz") or context.facts.get("tsc_hz") - if declared is None: - # validate-runner runs before the OpenVMM binary is available; the - # VM jobs run H3 with H4 to compare against the backend's rate. - detail += "; not compared with the backend's rate (H3 did not run)" - else: - deviation_ppm = abs(measured - int(declared)) / int(declared) * 1e6 - context.facts["rate_deviation_ppm"] = f"{deviation_ppm:.1f}" - detail += f"; {deviation_ppm:.1f} ppm from the backend's {declared} Hz" - if deviation_ppm > RATE_TOLERANCE_PPM: - problems.append( - f"the backend rate is {deviation_ppm:.1f} ppm from the measured " - f"rate (limit {RATE_TOLERANCE_PPM:g} ppm)" - ) - if clocksource: - detail += f"; host clocksource {clocksource} (evidence)" - if problems: - return CheckResult("H4", False, "; ".join(problems) + f"; {detail}") - return CheckResult("H4", True, detail) - - -def check_host_skew(context: DoctorContext) -> CheckResult: - arguments = ["skew", "--duration-ms", str(SKEW_PAIR_DURATION_MS)] - rate = context.facts.get("measured_tsc_hz") or context.facts.get("tsc_hz") - if rate is not None: - arguments.extend(("--tsc-hz", rate)) - summary = _probe_record(run_probe(context, *arguments), "skew") - offset = int(summary["max_abs_offset_ns"]) - uncertainty = int(summary["max_uncertainty_ns"]) - stalled = int(summary["stalled_pairs"]) - context.facts["host_skew_ns"] = str(offset) - detail = ( - f"pairs={summary['pairs']} cpus={summary['cpus']} max_abs_offset_ns={offset} " - f"max_uncertainty_ns={uncertainty}" - ) - problems: list[str] = [] - if offset > WARP_BOUND_NS: - problems.append(f"max_abs_offset_ns={offset} exceeds {WARP_BOUND_NS}") - if stalled: - problems.append(f"{stalled} CPU pair(s) stalled") - if summary.get("conclusive") != "1": - problems.append( - f"the measurement is inconclusive (uncertainty {uncertainty} ns)" - ) - if problems: - return CheckResult("H5", False, "; ".join(problems) + f"; {detail}") - return CheckResult("H5", True, detail) - - -def _warp_guest( - context: DoctorContext, vcpus: int, gaps: Sequence[str] -) -> tuple[dict[str, str], list[dict[str, str]]]: - """Boot a guest, check its boot marker, and run a warp schedule in it.""" - command = workload_boot_command( - context.openvmm, - context.backend, - context.kernel, - context.initrd, - GUEST_MEMORY_MIB, - "quiet loglevel=0", - processors=vcpus, - ) - command.extend(context.openvmm_args) - result = run_guest_script( - command, - warp_probe_script(gaps) + "nvx-exit 0\n", - WARP_PROBE_COMPLETION_MARKER, - timeout=context.timeout, - time_abi_status=True, - ) - monitor = TimeAbiMonitor(command) - monitor.feed(result["text"].encode()) - monitor.finish() - monitor.require_status("the warp probe") - boot = monitor.require_boot("the warp probe", online_cpus=vcpus) - rounds = check_warp_probe( - result["text"], cpus=vcpus, context="H6", rounds=warp_rounds(vcpus, gaps) - ) - return boot, rounds - - -def check_guest_warp(context: DoctorContext) -> CheckResult: - for path, description in ( - (context.openvmm, "OpenVMM"), - (context.kernel, "guest kernel"), - (context.initrd, "guest initramfs"), - ): - if not path.is_file(): - return CheckResult("H6", False, f"{description} was not found at {path}") - vcpus = _guest_vcpus() - # The largest guest runs the schedule, then a 1-vCPU guest runs the - # probe once. - guests: list[tuple[int, tuple[str, ...]]] = [] - if vcpus > 1: - guests.append((vcpus, context.schedule.warp_gaps)) - guests.append((1, ())) - boots: list[dict[str, str]] = [] - rounds: list[dict[str, str]] = [] - runs: list[str] = [] - for count, count_gaps in guests: - try: - boot, count_rounds = _warp_guest(context, count, count_gaps) - except (RuntimeError, TimeAbiFailure) as error: - first = str(error).splitlines()[0] - return CheckResult("H6", False, f"vcpus={count}: {first}") - boots.append(boot) - rounds.extend(count_rounds) - # The boot check's wall time and, where the guest reports it, its CPU - # time, which the spec budgets; both are evidence only. - cpu_us = f" boot_cpu_us={boot['cpu_us']}" if "cpu_us" in boot else "" - runs.append( - f"vcpus={count} rounds={len(count_rounds)} " - f"idle_gaps_s={','.join(count_gaps) or 'none'} " - f"boot_elapsed_us={boot.get('elapsed_us', '?')}{cpu_us}" - ) - - def worst(name: str) -> int: - return max(int(warp[name]) for warp in rounds) - - context.facts["guest_warp_ns"] = str(worst("max_abs_offset_ns")) - context.facts.setdefault("tsc_hz", boots[0]["tsc_hz"]) - context.facts.setdefault("lapic_hz", boots[0]["lapic_hz"]) - return CheckResult( - "H6", - True, - "; ".join(runs) + f"; max_backward_ns={worst('max_backward_ns')} " - f"max_abs_offset_ns={worst('max_abs_offset_ns')} " - f"max_uncertainty_ns={worst('max_uncertainty_ns')}", - ) - - -def _linux_clock_state() -> tuple[int, int]: - """Return adjtimex's clock state and status without changing the clock.""" - try: - libc = ctypes.CDLL(None, use_errno=True) - buffer = ctypes.create_string_buffer(256) - state = int(libc.adjtimex(buffer)) - except (AttributeError, OSError) as error: - raise ScriptError( - f"{UTC_UNVERIFIED}: adjtimex is unavailable: {error}" - ) from error - if state < 0: - raise ScriptError( - f"{UTC_UNVERIFIED}: adjtimex failed: {os.strerror(ctypes.get_errno())}" - ) - # struct timex: unsigned modes, then the long offset, freq, maxerror, and - # esterror fields, then the int status. - status = int.from_bytes(buffer.raw[40:44], sys.byteorder) - return state, status - - -def _windows_time_source() -> tuple[bool, str]: - try: - completed = subprocess.run( - ["w32tm", "/query", "/status"], - capture_output=True, - text=True, - timeout=30, - check=False, - ) - except (OSError, subprocess.SubprocessError, ValueError) as error: - raise ScriptError(f"{UTC_UNVERIFIED}: w32tm /query /status: {error}") from error - if completed.returncode != 0: - raise ScriptError( - f"{UTC_UNVERIFIED}: w32tm /query /status failed: " - + (completed.stdout.strip() or completed.stderr.strip()) - ) - fields: dict[str, str] = {} - for line in completed.stdout.splitlines(): - name, separator, value = line.partition(":") - if separator: - fields[name.strip()] = value.strip() - source = fields.get("Source", "") - leap = fields.get("Leap Indicator", "") - # w32tm prints its labels in the display language. Output without both - # English labels and a 0-3 leap indicator names no source, so the check - # fails rather than certify an unverified one. - if not source or not leap.startswith(("0", "1", "2", "3")): - raise ScriptError( - f"{UTC_UNVERIFIED}: w32tm /query /status reported " - f"source={source or 'none'} leap_indicator={leap or 'none'}" - ) - unsynchronized = ( - leap.startswith("3") - or "Local CMOS Clock" in source - or "Free-running System Clock" in source - ) - return not unsynchronized, f"source={source} leap_indicator={leap}" - - -def check_utc(context: DoctorContext) -> CheckResult: - del context - if host_is_windows(): - synchronized, detail = _windows_time_source() - if not synchronized: - return CheckResult("H7", False, f"host UTC is not synchronized: {detail}") - return CheckResult("H7", True, detail) - state, status = _linux_clock_state() - detail = f"adjtimex state={state} status=0x{status:04x}" - if state > ADJTIMEX_TIME_ERROR: - return CheckResult( - "H7", False, f"{UTC_UNVERIFIED}: unknown clock state; {detail}" - ) - if state == ADJTIMEX_TIME_ERROR or status & ADJTIMEX_STA_UNSYNC: - return CheckResult( - "H7", False, f"host UTC is not synchronized (STA_UNSYNC); {detail}" - ) - return CheckResult("H7", True, detail) - - -CHECKS: Mapping[str, Callable[[DoctorContext], CheckResult]] = { - "H1": check_backend, - "H2": check_cpu, - "H3": check_openvmm_preflight, - "H4": check_rate, - "H5": check_host_skew, - "H6": check_guest_warp, - "H7": check_utc, -} - - -def run_checks(context: DoctorContext, checks: Sequence[str]) -> list[CheckResult]: - """Run checks in spec order; an exception fails only its own check.""" - results: list[CheckResult] = [] - for check in CHECK_IDS: - if check not in checks: - continue - try: - result = CHECKS[check](context) - except ( - ArithmeticError, - IndexError, - KeyError, - OSError, - RuntimeError, - subprocess.SubprocessError, - TypeError, - ValueError, - ) as error: - first = str(error).splitlines()[0] if str(error) else type(error).__name__ - result = CheckResult(check, False, first) - results.append(result) - print(result.line(), flush=True) - return results - - -def summary_markdown( - backend: str, results: Sequence[CheckResult], facts: Mapping[str, str] -) -> str: - status = "passed" if all(result.passed for result in results) else "failed" - lines = [ - f"### Time ABI host qualification ({backend}): {status}", - "", - "| Check | Name | Status | Detail |", - "| --- | --- | --- | --- |", - ] - for result in results: - lines.append( - f"| {result.check} | {CHECK_TITLES[result.check]} | " - f"{'pass' if result.passed else '**fail**'} | " - f"{_escape_markdown(result.detail)} |" - ) - reported = ( - ("generation", "Generation"), - ("cpu", "CPU"), - ("microcode", "Microcode"), - ("os", "Host OS"), - ("profile", "Profile"), - ("profile_digest", "Profile digest"), - ("surface_digest", "CPU surface digest"), - ("tsc_hz", "Declared TSC rate (Hz)"), - ("lapic_hz", "LAPIC rate (Hz)"), - ("measured_tsc_hz", "Measured TSC rate (Hz)"), - ("rate_deviation_ppm", "Rate deviation from the backend (ppm)"), - ("rate_agreement_ppm", "Interval rate agreement (ppm)"), - ("rate_uncertainty_ppm", "Largest interval uncertainty (ppm)"), - ("host_skew_ns", "Host skew (ns)"), - ("guest_warp_ns", "Guest warp offset (ns)"), - ("invariant_tsc", "Host invariant TSC (evidence)"), - ("host_clocksource", "Host clocksource (evidence)"), - ) - facts_line = " ยท ".join( - f"{label}: `{facts[name]}`" for name, label in reported if name in facts - ) - if facts_line: - lines.extend(("", facts_line)) - return "\n".join(lines) + "\n\n" - - -def run(args: argparse.Namespace) -> int: - if args.backend not in OPENVMM_TEST_BACKENDS: - raise ScriptError(f"unsupported backend {args.backend!r}") - checks = tuple(dict.fromkeys(args.checks or CHECK_IDS)) - if args.no_openvmm: - booting = [check for check in checks if check in OPENVMM_CHECKS] - if booting: - raise ScriptError( - f"--no-openvmm cannot run {' and '.join(booting)}, which boot OpenVMM" - ) - probe_directory = args.probe_dir or default_probe_directory() - fingerprint: Path | None = None - if not args.no_openvmm: - fingerprint = args.cpu_fingerprint or ( - probe_directory / f"nvx-cpu-fingerprint-{args.backend}.json" - ) - context = DoctorContext( - backend=args.backend, - openvmm=args.openvmm or openvmm_binary_path(), - kernel=args.kernel or artifact_path(KernelBuildConstants.BINARY_NAME), - initrd=args.initrd or artifact_path(AlpineBuildConstants.INITRAMFS_NAME), - openvmm_args=tuple(args.openvmm_arg), - probe_directory=probe_directory, - timeout=args.timeout, - fingerprint=fingerprint, - schedule=CI_SCHEDULE if args.ci_schedule else QUALIFICATION_SCHEDULE, - ) - results = run_checks(context, checks) - failed = [result.check for result in results if not result.passed] - generation = context.facts.get("generation") - profile = context.facts.get("profile") - print( - f"Time ABI host qualification on {args.backend}: " - + ("passed" if not failed else f"failed ({', '.join(failed)})") - + (f"; generation {generation}" if generation else "") - + (f", profile {profile}" if profile else ""), - flush=True, - ) - if args.summary is not None: - with args.summary.open("a", encoding="utf-8") as summary: - summary.write(summary_markdown(args.backend, results, context.facts)) - return 1 if failed else 0 - - -def default_probe_directory() -> Path: - tool_cache = os.environ.get("RUNNER_TOOL_CACHE") - if tool_cache: - return Path(tool_cache) / "nvx-host-time-probe" - return BuildConstants.BUILD_DIR / "host-time-probe" - - -def configure_parser(parser: argparse.ArgumentParser) -> None: - parser.description = ( - "Qualify this host for the NVX time ABI (doc/design/time-abi.md, " - "'Host qualification')." - ) - parser.add_argument("--backend", choices=OPENVMM_TEST_BACKENDS, required=True) - parser.add_argument( - "--checks", - nargs="+", - choices=CHECK_IDS, - metavar="ID", - help="checks to run, in spec order (default: H1 to H7)", - ) - parser.add_argument( - "--openvmm", type=Path, help="OpenVMM binary for H2, H3, and H6" - ) - parser.add_argument("--kernel", type=Path, help="guest kernel for H3 and H6") - parser.add_argument("--initrd", type=Path, help="guest initramfs for H3 and H6") - openvmm = parser.add_mutually_exclusive_group() - openvmm.add_argument( - "--cpu-fingerprint", - type=Path, - help="where H2 writes OpenVMM's CPU fingerprint " - "(default: nvx-cpu-fingerprint-.json in the probe directory)", - ) - openvmm.add_argument( - "--no-openvmm", - action="store_true", - help="qualify without an OpenVMM binary: H2 checks the CPU identity and " - "generation but not the CPU profile, and H3 and H6 cannot run", - ) - parser.add_argument( - "--ci-schedule", - action="store_true", - help="run H4 and H6 on CI's short schedules: H4 takes 3 samples 1 s " - "apart instead of 13 samples 10 s apart, and H6 runs the warp probe " - "twice instead of five times", - ) - parser.add_argument( - "--probe-dir", - type=Path, - help="cache directory for the host probe binary " - "(default: $RUNNER_TOOL_CACHE or build/)", - ) - parser.add_argument( - "--summary", - type=Path, - help="append a Markdown summary, for example $GITHUB_STEP_SUMMARY", - ) - parser.add_argument( - "--timeout", - type=float, - default=120.0, - help="seconds allowed for each probe or guest, on top of H4's sampling " - "window (default: 120)", - ) - parser.add_argument( - "--openvmm-arg", - action="append", - default=[], - help=argparse.SUPPRESS, - ) - parser.set_defaults(handler=run) From d65381467e7ab13f0c632c5744aea78aa731be1f Mon Sep 17 00:00:00 2001 From: Akhil Gupta Date: Mon, 12 Oct 2026 05:00:11 -0700 Subject: [PATCH 3/5] Rename nvx507-doctor.py to scripts/nvx_tools/doctor.py --- nvx507-doctor.py => scripts/nvx_tools/doctor.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename nvx507-doctor.py => scripts/nvx_tools/doctor.py (100%) diff --git a/nvx507-doctor.py b/scripts/nvx_tools/doctor.py similarity index 100% rename from nvx507-doctor.py rename to scripts/nvx_tools/doctor.py From 3a7af3759aa4d72b737b72b58a00122152ff3262 Mon Sep 17 00:00:00 2001 From: Akhil Gupta Date: Mon, 12 Oct 2026 05:01:05 -0700 Subject: [PATCH 4/5] Delete scripts/test_time_abi.py --- scripts/test_time_abi.py | 3030 -------------------------------------- 1 file changed, 3030 deletions(-) delete mode 100644 scripts/test_time_abi.py diff --git a/scripts/test_time_abi.py b/scripts/test_time_abi.py deleted file mode 100644 index f4d58b5a5..000000000 --- a/scripts/test_time_abi.py +++ /dev/null @@ -1,3030 +0,0 @@ -"""Unit tests for the harness-side NVX time ABI checks and host qualification.""" - -from __future__ import annotations - -import contextlib -import io -import json -import queue -import re -import shutil -import socket -import subprocess -import sys -import tempfile -import threading -import unittest -from pathlib import Path -from unittest.mock import patch - -sys.path.insert(0, str(Path(__file__).resolve().parent)) - -from nvx_tools import benchmark, doctor, openvmm_process, time_abi # noqa: E402 -from nvx_tools.time_abi import TimeAbiFailure, TimeAbiMonitor # noqa: E402 - -BOOT_LINE = ( - "NVX-TIME-ABI: v=1 phase=boot status=ok cpus=4 tsc_hz=2194804000 " - "lapic_hz=1000000000 generation=0 elapsed_us=1873\n" -) -RESTORE_LINE = ( - "NVX-TIME-ABI: v=1 phase=restore status=ok cpus=4 tsc_hz=2194804000 " - "lapic_hz=1000000000 generation=2 elapsed_us=41\n" -) -# nvx-time status prints the last check's marker, then this line. -RUNTIME_LINE = ( - "NVX-TIME-ABI: v=1 phase=runtime status=unsynchronized generation=0 " - "discontinuities=0 offset_ns=0 uncertainty_ns=0 rejected_samples=0 " - "last_sample_error=none\n" -) -STATUS_OK = "NVX-TIME-STATUS-EXIT status=0\n" -KVM_BOOT = ["openvmm", "--hypervisor", "kvm", "--kernel", "vmlinux"] -MSHV_RESTORE = ["openvmm", "--hypervisor", "mshv", "--restore-snapshot", "snap"] -# OpenVMM's openvmm_entry fatal_error_message, after a guest prompt. -FATAL_DETAIL = "[E_TSC_SYNC_UNSUPPORTED] failed to launch vm worker" -FATAL = f"~ # fatal error: {FATAL_DETAIL}\r\n\r\nCaused by:\r\n".encode() - - -def warp_output( - *, - pairs: int = 6, - backward: int = 0, - offset: int = 15, - verdict: str = "PASS", - stalled: int = 0, - conclusive: int = 1, -) -> str: - return ( - "pair 0-1: warp_iterations=881154 warps=0 max_backward_ns=0\n" - f"NVX-TIME-PROBE warp max_backward_cycles=0 max_backward_ns={backward} " - f"max_abs_offset_ns={offset} pairs={pairs} verdict={verdict} bound_ns=1000\r\n" - "NVX-TIME-PROBE warp-detail max_abs_offset_cycles=33.0 max_uncertainty_ns=127 " - "max_skew_bound_ns=130 total_warps=0 inconsistent_pairs=0 " - f"stalled_pairs={stalled} cpus=0-3 duration_ms=100 tsc_hz=2194804000 " - f"tsc_hz_source=kmsg-detected-processor conclusive={conclusive}\n" - ) - - -def write_openvmm_profiles( - checkout: Path, *generations: time_abi.CpuGeneration, revision: int = 1 -) -> Path: - """Write ``generations`` as revision ``revision`` of the pinned CPU - profiles of an OpenVMM checkout at ``checkout``, with the fields that NVX - reads, and return the checkout.""" - directory = checkout / time_abi.OPENVMM_CPU_PROFILES - directory.mkdir(parents=True, exist_ok=True) - for generation in generations: - profile_id = ( - f"{time_abi.PROFILE_VENDORS[generation.vendor]}.{generation.name}" - f".v{revision}" - ) - document = { - "id": profile_id, - "vendor": generation.vendor, - "generation": { - "name": generation.name, - "cpus": [ - { - "family": cpu.family, - "model": cpu.model, - "steppings": [cpu.steppings.start, cpu.steppings.stop - 1], - } - for cpu in generation.cpus - ], - }, - } - (directory / f"{profile_id}.json").write_text( - json.dumps(document), encoding="utf-8" - ) - return checkout - - -class FieldParsingTests(unittest.TestCase): - def test_parses_plain_and_quoted_fields_with_guest_escapes(self): - fields = time_abi.parse_fields( - 'v=1 code=G_TSC_WARP detail="say \\"hi\\" \\\\ \\x07 end" phase=boot' - ) - self.assertEqual( - fields, - { - "v": "1", - "code": "G_TSC_WARP", - "detail": 'say "hi" \\ \x07 end', - "phase": "boot", - }, - ) - - def test_rejects_malformed_fields(self): - for text in ( - "novalue", - "=x", - 'detail="unterminated', - 'detail="bad \\q escape"', - 'detail="\\x4"', - "v=1 v=2", - ): - with self.subTest(text=text), self.assertRaises(ValueError): - time_abi.parse_fields(text) - - def test_parses_markers_and_violations(self): - marker = time_abi.parse_marker(BOOT_LINE) - self.assertIsNotNone(marker) - assert marker is not None - self.assertEqual(marker["phase"], "boot") - self.assertEqual(marker["tsc_hz"], "2194804000") - self.assertIsNone(time_abi.parse_marker("NVX-TIME-REPORT: v=1 phase=boot")) - with self.assertRaises(ValueError): - time_abi.parse_marker("NVX-TIME-ABI: v=1 status=ok") - violation = time_abi.parse_violation( - "noise NVX-TIME-ABI-VIOLATION: v=1 code=G_RCU_STALL source=watcher " - 'phase=runtime generation=1 boottime_ns=5 detail="rcu: INFO: x"\r' - ) - self.assertIsNotNone(violation) - assert violation is not None - self.assertEqual(violation["code"], "G_RCU_STALL") - self.assertEqual(violation["detail"], "rcu: INFO: x") - - def test_describes_only_time_abi_exit_statuses(self): - self.assertIn("conformance", time_abi.describe_exit_status(193) or "") - self.assertIn("runtime violation", time_abi.describe_exit_status(194) or "") - self.assertIn("restore repair", time_abi.describe_exit_status(195) or "") - for status in (None, 0, 1, 37, 192, 196, 255): - self.assertIsNone(time_abi.describe_exit_status(status)) - - def test_maps_cpu_generations(self): - def name(model: int, stepping: int, vendor: str = "GenuineIntel"): - generation = time_abi.cpu_generation(vendor, 6, model, stepping) - return generation.name if generation else None - - self.assertEqual(name(85, 4), "skylake-sp") - self.assertEqual(name(106, 6), "icelake-sp") - self.assertEqual(name(207, 2), "emeraldrapids") - # Alder Lake-S and Alder Lake-P and -H share one profile. - self.assertEqual(name(151, 2), "alderlake") - self.assertEqual(name(154, 3), "alderlake") - # Cascade Lake and Cooper Lake share model 85 but have no profile. - self.assertIsNone(name(85, 7)) - self.assertIsNone(name(85, 11)) - self.assertIsNone(name(143, 8)) - # Tiger Lake and Raptor Lake have no profile either. - self.assertIsNone(name(140, 1)) - self.assertIsNone(name(183, 1)) - self.assertIsNone(name(1, 1, "AuthenticAMD")) - # Milan's profile serves every stepping of AMD's family 25 model 1, - # Milan-X's 2 included, Genoa's every stepping of model 17, and - # Turin's every stepping of family 26 model 2, and none of them - # another vendor's CPU with its signature. - for family, model, expected in ( - (25, 1, "milan"), - (25, 17, "genoa"), - (26, 2, "turin"), - ): - for stepping in (0, 1, 2, 15): - generation = time_abi.cpu_generation( - "AuthenticAMD", family, model, stepping - ) - assert generation is not None - self.assertEqual(generation.name, expected) - # Vermeer and Raphael (Ryzen), Turin Dense, and Granite Ridge have no - # profile, and neither has another vendor's CPU with the signature of - # an AMD generation. - for vendor, family, model in ( - ("AuthenticAMD", 25, 33), - ("AuthenticAMD", 25, 97), - ("AuthenticAMD", 26, 17), - ("AuthenticAMD", 26, 68), - ("GenuineIntel", 25, 1), - ("HygonGenuine", 25, 1), - ("GenuineIntel", 26, 2), - ): - with self.subTest(vendor=vendor, family=family, model=model): - self.assertIsNone(time_abi.cpu_generation(vendor, family, model, 1)) - - def test_names_the_catalog_profiles(self): - # One profile per generation serves every backend. - self.assertEqual( - [generation.profile_id for generation in time_abi.CPU_GENERATIONS], - [ - "intel.skylake-sp.v1", - "intel.icelake-sp.v1", - "intel.emeraldrapids.v1", - "intel.alderlake.v1", - "amd.milan.v1", - "amd.genoa.v1", - "amd.turin.v1", - ], - ) - self.assertEqual( - time_abi.describe_cpu_generations(), - "skylake-sp 6/85 steppings 0-4, icelake-sp 6/106, emeraldrapids 6/207, " - "alderlake 6/151 and 6/154, milan 25/1, genoa 25/17, turin 26/2", - ) - - def test_the_catalog_copy_matches_openvmm_pinned_profiles(self): - # NVX's copy must not drift from the profiles that OpenVMM pins. They - # are read from the gitlink's commit, not from the submodule's working - # tree, which can hold another revision: a self-hosted CI runner keeps - # an earlier job's OpenVMM checkout until the job checks out its own. - root = Path(__file__).resolve().parents[1] - submodule = root / "openvmm" - if not (submodule / ".git").exists(): - self.skipTest("the OpenVMM submodule is not initialized") - - def git(directory: Path, *arguments: str) -> str | None: - result = subprocess.run( - ["git", "-C", str(directory), *arguments], - capture_output=True, - encoding="utf-8", - check=False, - ) - return result.stdout if result.returncode == 0 else None - - pin = (git(root, "rev-parse", ":openvmm") or "").strip() - if not pin or git(submodule, "cat-file", "-e", f"{pin}^{{commit}}") is None: - self.skipTest( - f"the OpenVMM submodule lacks its pinned revision {pin or '(unknown)'}" - ) - listing = git( - submodule, - "ls-tree", - "-z", - "--name-only", - pin, - "--", - "vmm_core/cpu_profile/profiles/", - ) - assert listing is not None - pinned: dict[str, tuple[str, str, tuple[tuple[int, int, range], ...]]] = {} - temporary = tempfile.TemporaryDirectory() - self.addCleanup(temporary.cleanup) - checkout = Path(temporary.name) - profiles = checkout / time_abi.OPENVMM_CPU_PROFILES - profiles.mkdir(parents=True) - for path in sorted(listing.split("\0")): - if not path.endswith(".json"): - continue - content = git(submodule, "show", f"{pin}:{path}") - assert content is not None - (profiles / Path(path).name).write_text(content, encoding="utf-8") - document = json.loads(content) - generation = document["generation"] - # Keyed by profile ID, so that a second pinned revision of a - # generation fails the comparison instead of replacing the first. - pinned[document["id"]] = ( - generation["name"], - document["vendor"], - tuple( - ( - cpu["family"], - cpu["model"], - range(cpu["steppings"][0], cpu["steppings"][1] + 1), - ) - for cpu in generation["cpus"] - ), - ) - self.assertTrue(pinned) - copy = { - generation.profile_id: ( - generation.name, - generation.vendor, - tuple( - (cpu.family, cpu.model, cpu.steppings) for cpu in generation.cpus - ), - ) - for generation in time_abi.CPU_GENERATIONS - } - self.assertEqual(copy, pinned) - # benchmark reads the same generations from these documents, as it - # reads them from the OpenVMM checkout that it builds. - self.assertEqual( - set(time_abi.openvmm_cpu_generations(checkout)), - set(time_abi.CPU_GENERATIONS), - ) - - def test_reads_the_cpu_profiles_of_an_openvmm_checkout(self): - temporary = tempfile.TemporaryDirectory() - self.addCleanup(temporary.cleanup) - directory = Path(temporary.name) - icelake, alderlake = time_abi.CPU_GENERATIONS[1], time_abi.CPU_GENERATIONS[3] - granite_rapids = time_abi.CpuGeneration( - "graniterapids", "GenuineIntel", (time_abi.CpuModel(6, 173),) - ) - checkout = write_openvmm_profiles( - directory / "openvmm", icelake, alderlake, granite_rapids - ) - # A second revision of a generation's profile adds no generation. - write_openvmm_profiles(checkout, icelake, revision=2) - generations = time_abi.openvmm_cpu_generations(checkout) - self.assertEqual(set(generations), {icelake, alderlake, granite_rapids}) - self.assertEqual(len(generations), 3) - # A CPU that only the checkout's profiles cover is benchmarked with - # them, and not with NVX's copy of the catalog. - granite_rapids_host = time_abi.HostCpu("GenuineIntel", 6, 173, 1) - self.assertIsNone( - time_abi.benchmark_cpu_profile_refusal( - granite_rapids_host, "openvmm", "kvm", generations - ) - ) - self.assertIsNotNone( - time_abi.benchmark_cpu_profile_refusal( - granite_rapids_host, "openvmm", "kvm" - ) - ) - # A checkout that pins no profile, and a malformed profile, are errors, - # naming the checkout or the profile. - with self.assertRaisesRegex(ValueError, "pins no CPU profile"): - time_abi.openvmm_cpu_generations(directory / "empty") - valid = { - "vendor": "GenuineIntel", - "generation": { - "name": "icelake-sp", - "cpus": [{"family": 6, "model": 106, "steppings": [0, 15]}], - }, - } - cases: tuple[tuple[str, object, str], ...] = ( - ("not-json", None, "cannot read the CPU profile .*not-json.json"), - ("list", [], "it is not a JSON object"), - ("no-generation", {"vendor": "GenuineIntel"}, "names no vendor"), - ( - "no-cpus", - {**valid, "generation": {"name": "icelake-sp", "cpus": []}}, - "names no CPU models", - ), - ( - "stepping-pair", - { - **valid, - "generation": { - "name": "icelake-sp", - "cpus": [{"family": 6, "model": 106, "steppings": [3]}], - }, - }, - "not a first and a last", - ), - ( - "stepping-order", - { - **valid, - "generation": { - "name": "icelake-sp", - "cpus": [{"family": 6, "model": 106, "steppings": [9, 3]}], - }, - }, - "steppings 9 to 3 are not a range", - ), - ( - "boolean-family", - { - **valid, - "generation": { - "name": "icelake-sp", - "cpus": [{"family": True, "model": 106, "steppings": [0, 1]}], - }, - }, - "its family is not an integer", - ), - ) - for name, document, message in cases: - with self.subTest(name=name): - malformed = directory / name - profiles = malformed / time_abi.OPENVMM_CPU_PROFILES - profiles.mkdir(parents=True) - (profiles / f"{name}.json").write_text( - "{" if document is None else json.dumps(document), - encoding="utf-8", - ) - with self.assertRaisesRegex(ValueError, message): - time_abi.openvmm_cpu_generations(malformed) - - def test_tells_catalog_profiles_from_host_profiles(self): - for profile_id, catalog, host in ( - ("intel.icelake-sp.v1", True, False), - ("intel.raptorlake.v2", True, False), - ("amd.milan.v1", True, False), - ("intel.host.v1", False, True), - ("amd.host.v1", False, True), - # Host profile IDs take the revision syntax of every profile ID. - ("intel.host.v0", False, False), - ("interim.host.kvm.v1", False, False), - ("intel.icelake-sp.v2-rc", False, False), - ("intel.icelake-sp.v0", False, False), - ("none", False, False), - ("", False, False), - ): - with self.subTest(profile_id=profile_id): - self.assertEqual(time_abi.is_catalog_profile_id(profile_id), catalog) - self.assertEqual(time_abi.is_host_profile_id(profile_id), host) - - def test_guides_hosts_that_no_built_in_profile_serves(self): - def guide(cpu_profile: str | None, host: time_abi.HostCpu | None) -> str | None: - return time_abi.host_cpu_profile_guidance( - cpu_profile, host, "/nvx/openvmm", "kvm" - ) - - tiger_lake = time_abi.HostCpu( - "GenuineIntel", 6, 140, 1, "11th Gen Intel(R) Core(TM) i7-1185G7" - ) - guidance = guide(None, tiger_lake) - assert guidance is not None - lines = guidance.splitlines() - self.assertTrue(all(line.startswith("nvx: ") for line in lines), guidance) - # auto falls back on the CPU, so the guidance says what auto does there, - # not which code ended this run, which run cannot read. - self.assertEqual( - lines[0], - "nvx: no built-in CPU profile serves this host's CPU, GenuineIntel " - "6/140/1 (11th Gen Intel(R) Core(TM) i7-1185G7), so --cpu-profile " - "auto, the default, falls back on it to a CPU profile derived from " - "this host, which OpenVMM warns about with NVX-CPU-PROFILE-FALLBACK:; " - f"the built-in profiles cover {time_abi.describe_cpu_generations()}.", - ) - self.assertIn("OpenVMM's error above names the cause", lines[1]) - self.assertNotIn("E_PROFILE_HOST_UNKNOWN", guidance) - self.assertNotIn("rerun with", guidance) - # It asks for the CPU's fingerprint through the CPU profile request - # form, which the link prefills with the CPU. - self.assertEqual( - lines[2:], - [ - "nvx: Help NVX add a built-in profile for this CPU: run", - "nvx: /nvx/openvmm --hypervisor kvm --cpu-fingerprint " - "fingerprint.json", - "nvx: and attach fingerprint.json to a CPU profile request:", - f"nvx: {time_abi.cpu_profile_request_url(tiger_lake)}", - "nvx: https://github.com/microsoft/nvx/issues/408 tracks CPU " - "profiles for more Intel CPUs.", - ], - ) - self.assertEqual(guidance, guide("auto", tiger_lake)) - # A host profile that fails on this CPU fails for a reason that - # OpenVMM's own error explains. - self.assertIsNone(guide("host", tiger_lake)) - - # Host profiles serve AMD CPUs too: an AMD CPU without a built-in - # profile, such as Raphael (Ryzen 7000), falls back as an Intel one - # does, with the issue that tracks profiles for more AMD CPUs. - raphael = time_abi.HostCpu("AuthenticAMD", 25, 97, 2) - guidance = guide(None, raphael) - assert guidance is not None - self.assertIn( - "this host's CPU, AuthenticAMD 25/97/2, so --cpu-profile auto, the " - "default, falls back on it", - guidance, - ) - self.assertIn(time_abi.cpu_profile_request_url(raphael), guidance) - self.assertIn( - "https://github.com/microsoft/nvx/issues/409 tracks CPU profiles for " - "more AMD CPUs.", - guidance, - ) - self.assertNotIn("issues/408", guidance) - self.assertIsNone(guide("host", raphael)) - # Another vendor's CPU fails with either request, and gets no request - # form and no issue: the time ABI does not serve its CPUs. - hygon = time_abi.HostCpu("HygonGenuine", 24, 0, 1) - guidance = guide("auto", hygon) - assert guidance is not None - self.assertIn( - "this host's CPU, HygonGenuine 24/0/1, so a cold boot with " - "--cpu-profile auto, the default, fails on it with " - "E_PROFILE_HOST_UNKNOWN;", - guidance, - ) - self.assertIn("host CPU profiles serve only Intel and AMD CPUs", guidance) - self.assertNotIn("falls back", guidance) - self.assertNotIn("github.com", guidance) - self.assertEqual(guidance, guide("host", hygon)) - - # A CPU that a built-in profile serves, an explicit profile, and an - # unknown CPU get none. - alder_lake = time_abi.HostCpu("GenuineIntel", 6, 154, 3) - milan = time_abi.HostCpu("AuthenticAMD", 25, 1, 1) - genoa = time_abi.HostCpu("AuthenticAMD", 25, 17, 1) - turin = time_abi.HostCpu("AuthenticAMD", 26, 2, 1) - for cpu_profile, host in ( - (None, alder_lake), - ("host", alder_lake), - (None, milan), - ("host", milan), - (None, genoa), - (None, turin), - ("host", turin), - ("intel.alderlake.v1", tiger_lake), - ("intel.alderlake.v1", raphael), - ("intel.skylake-sp.v1", milan), - ("intel.skylake-sp.v1", hygon), - (None, None), - ): - with self.subTest(cpu_profile=cpu_profile, host=host): - self.assertIsNone(guide(cpu_profile, host)) - - def test_names_cpus_and_prefills_cpu_profile_requests(self): - granite_rapids = time_abi.HostCpu( - "GenuineIntel", 6, 173, 1, "Intel(R) Xeon(R) 6973P-C" - ) - self.assertEqual(granite_rapids.signature(), "GenuineIntel 6/173/1") - self.assertEqual( - granite_rapids.describe(), "GenuineIntel 6/173/1 (Intel(R) Xeon(R) 6973P-C)" - ) - # The link that OpenVMM's fallback warning gives the same CPU - # (openvmm_core's time_abi tests). - self.assertEqual( - time_abi.cpu_profile_request_url(granite_rapids), - "https://github.com/microsoft/nvx/issues/new?template=cpu-profile.yml" - "&title=CPU%20profile%3A%20GenuineIntel%206%2F173%2F1%20%28Intel%28R%29" - "%20Xeon%28R%29%206973P-C%29&signature=GenuineIntel%206%2F173%2F1" - "&cpu=Intel%28R%29%20Xeon%28R%29%206973P-C", - ) - # A CPU without a brand string prefills only its signature. - brandless = time_abi.HostCpu("GenuineIntel", 6, 173, 1) - self.assertEqual(brandless.describe(), "GenuineIntel 6/173/1") - self.assertEqual( - time_abi.cpu_profile_request_url(brandless), - f"{time_abi.CPU_PROFILE_REQUEST_URL}&title=CPU%20profile%3A%20" - "GenuineIntel%206%2F173%2F1&signature=GenuineIntel%206%2F173%2F1", - ) - self.assertIn( - "%26%3D%3F%23%25%2B", - time_abi.cpu_profile_request_url( - time_abi.HostCpu("GenuineIntel", 6, 173, 1, "&=?#%+") - ), - ) - - def test_parses_the_cpu_profile_fallback_warning(self): - digest = "ab" * 32 - warning = ( - " 0.012345s WARN openvmm_core::worker::dispatch::time_abi: " - "NVX-CPU-PROFILE-FALLBACK: no built-in CPU profile serves this host's " - "CPU, GenuineIntel 6/173/1 (Intel(R) Xeon(R) 6973P-C), so --cpu-profile " - f"auto fell back to intel.host.v1 (sha256:{digest}), a profile derived " - "from this host's kvm backend for development.\n" - "It is not pinned: a microcode, firmware, hypervisor, or OS update can " - "change it.\n" - ) - self.assertEqual(time_abi.parse_cpu_profile_fallback(warning), "intel.host.v1") - self.assertEqual( - time_abi.parse_cpu_profile_fallback( - warning.replace("intel.host.v1", "amd.host.v1") - ), - "amd.host.v1", - ) - self.assertIsNone(time_abi.parse_cpu_profile_fallback("no warning\n")) - for output, message in ( - (warning + warning, "logged 2 NVX-CPU-PROFILE-FALLBACK: warnings"), - (warning.replace("intel.host.v1", "intel.icelake-sp.v1"), "malformed"), - (warning.replace(f"sha256:{digest}", "sha256:ab"), "malformed"), - (warning.replace("fell back to", "selected"), "malformed"), - ): - with self.subTest(message=message): - with self.assertRaisesRegex(ValueError, message): - time_abi.parse_cpu_profile_fallback(output) - - def test_benchmark_refuses_cpus_that_no_built_in_profile_serves(self): - granite_rapids = time_abi.HostCpu( - "GenuineIntel", 6, 173, 1, "Intel(R) Xeon(R) 6973P-C" - ) - refusal = time_abi.benchmark_cpu_profile_refusal( - granite_rapids, "openvmm", "kvm" - ) - assert refusal is not None - self.assertTrue( - refusal.startswith( - "benchmark refuses this host: no built-in CPU profile serves its CPU, " - "GenuineIntel 6/173/1 (Intel(R) Xeon(R) 6973P-C), so --cpu-profile " - "auto would fall back to a CPU profile derived from this host, whose " - "every cold boot fingerprints the hypervisor first" - ), - refusal, - ) - self.assertIn(time_abi.describe_cpu_generations(), refusal) - self.assertIn( - "\n openvmm --hypervisor kvm --cpu-fingerprint fingerprint.json\n", refusal - ) - self.assertTrue( - refusal.endswith(time_abi.cpu_profile_request_url(granite_rapids)), refusal - ) - # Another vendor's CPU cannot boot at all, and gets no request form. - hygon = time_abi.benchmark_cpu_profile_refusal( - time_abi.HostCpu("HygonGenuine", 24, 0, 1), "openvmm", "kvm" - ) - assert hygon is not None - self.assertIn("host CPU profiles serve only Intel and AMD CPUs", hygon) - self.assertIn("E_PROFILE_HOST_UNKNOWN", hygon) - self.assertNotIn("github.com", hygon) - # A CPU that cannot be identified is refused too: OpenVMM, which reads - # the CPU itself, could fall back on it. - unknown = time_abi.benchmark_cpu_profile_refusal(None, "openvmm", "kvm") - assert unknown is not None - self.assertTrue( - unknown.startswith( - "benchmark refuses this host: it cannot identify the host's CPU" - ), - unknown, - ) - # Its vendor is unknown too, so the refusal qualifies the fallback by - # vendor. - self.assertIn( - "On an Intel or AMD CPU that none serves, --cpu-profile auto falls " - "back to a CPU profile", - unknown, - ) - self.assertIn( - "on another vendor's CPU, every cold boot fails with " - f"{time_abi.PROFILE_HOST_UNKNOWN}", - unknown, - ) - # A CPU that a built-in profile serves is benchmarked. - for host in ( - time_abi.HostCpu("GenuineIntel", 6, 106, 6), - time_abi.HostCpu("AuthenticAMD", 25, 1, 1), - ): - with self.subTest(host=host): - self.assertIsNone( - time_abi.benchmark_cpu_profile_refusal(host, "openvmm", "whp") - ) - # The profiles that decide are those of the OpenVMM that benchmark - # builds: one that adds a profile for a CPU serves it, and one that - # lacks a profile refuses its CPU, listing its own profiles. - graniterapids = time_abi.CpuGeneration( - "graniterapids", "GenuineIntel", (time_abi.CpuModel(6, 173),) - ) - self.assertIsNone( - time_abi.benchmark_cpu_profile_refusal( - granite_rapids, - "openvmm", - "kvm", - (*time_abi.CPU_GENERATIONS, graniterapids), - ) - ) - icelake = time_abi.HostCpu("GenuineIntel", 6, 106, 6) - refusal = time_abi.benchmark_cpu_profile_refusal( - icelake, "openvmm", "kvm", (graniterapids,) - ) - assert refusal is not None - self.assertIn( - "no built-in CPU profile serves its CPU, GenuineIntel 6/106/6", refusal - ) - self.assertIn("The built-in profiles cover graniterapids 6/173.\n", refusal) - # A CPU in more than one generation fails every cold boot, because - # auto rejects it as a catalog defect without falling back, as OpenVMM - # does; revisions of one generation's profile do not make it ambiguous. - icelake_x = time_abi.CpuGeneration( - "icelake-x", "GenuineIntel", (time_abi.CpuModel(6, 106),) - ) - ambiguous = time_abi.benchmark_cpu_profile_refusal( - icelake, "openvmm", "kvm", (*time_abi.CPU_GENERATIONS, icelake_x) - ) - assert ambiguous is not None - self.assertTrue( - ambiguous.startswith( - "benchmark refuses this host: its CPU, GenuineIntel 6/106/6, is in " - "more than one generation of the built-in CPU profiles (icelake-sp, " - "icelake-x), so every cold boot with --cpu-profile auto fails with " - f"{time_abi.PROFILE_HOST_UNKNOWN}." - ), - ambiguous, - ) - self.assertNotIn("github.com", ambiguous) - revised = time_abi.CpuGeneration( - "icelake-sp", - "GenuineIntel", - (time_abi.CpuModel(6, 106), time_abi.CpuModel(6, 108)), - ) - self.assertIsNone( - time_abi.benchmark_cpu_profile_refusal( - icelake, "openvmm", "kvm", (*time_abi.CPU_GENERATIONS, revised) - ) - ) - - def test_cpu_profile_requests_match_openvmm_and_the_issue_form(self): - # OpenVMM's fallback warning and NVX link to the same form, which must - # define the text fields that the links prefill, and NVX matches the - # warning's marker. OpenVMM's constants are read from the gitlink's - # commit, as the catalog copy's comparison reads its profiles. - root = Path(__file__).resolve().parents[1] - form = (root / ".github" / "ISSUE_TEMPLATE" / "cpu-profile.yml").read_text( - encoding="utf-8" - ) - fields = re.findall(r"^ - type: (\w+)\n id: ([\w-]+)$", form, re.MULTILINE) - for prefilled in ("cpu", "signature"): - self.assertIn(("input", prefilled), fields) - self.assertRegex(form, r"(?m)^name: CPU profile request$") - self.assertEqual( - time_abi.CPU_PROFILE_REQUEST_URL.rpartition("template=")[2], - "cpu-profile.yml", - ) - - submodule = root / "openvmm" - if not (submodule / ".git").exists(): - self.skipTest("the OpenVMM submodule is not initialized") - pin = subprocess.run( - ["git", "-C", str(root), "rev-parse", ":openvmm"], - capture_output=True, - encoding="utf-8", - check=False, - ).stdout.strip() - source = subprocess.run( - [ - "git", - "-C", - str(submodule), - "show", - f"{pin}:openvmm/openvmm_core/src/worker/dispatch/time_abi.rs", - ], - capture_output=True, - encoding="utf-8", - check=False, - ) - if not pin or source.returncode != 0: - self.skipTest( - f"the OpenVMM submodule lacks its pinned revision {pin or '(unknown)'}" - ) - constants = dict( - re.findall( - r'const (FALLBACK_MARKER|CPU_PROFILE_REQUEST_URL): &str =\s*"([^"]*)";', - source.stdout, - ) - ) - self.assertEqual( - constants, - { - "FALLBACK_MARKER": time_abi.CPU_PROFILE_FALLBACK_MARKER, - "CPU_PROFILE_REQUEST_URL": time_abi.CPU_PROFILE_REQUEST_URL, - }, - ) - for key in ("&title=", "&signature=", "&cpu="): - self.assertIn(key, source.stdout) - - def test_computes_the_checks_cpu_time_budget(self): - # The spec's final budgets: a base plus an increment per additional - # CPU, one budget per backend and phase. - for backend in ("kvm", "mshv", "whp"): - self.assertEqual( - set(time_abi.CHECK_CPU_BUDGET_US[backend]), - {"boot", "capture", "restore"}, - ) - self.assertEqual(time_abi.check_cpu_budget_us("kvm", "boot", 1), 6_000) - self.assertEqual(time_abi.check_cpu_budget_us("kvm", "capture", 1), 1_500) - self.assertEqual(time_abi.check_cpu_budget_us("kvm", "capture", 8), 4_300) - self.assertEqual(time_abi.check_cpu_budget_us("kvm", "restore", 1), 7_000) - self.assertEqual(time_abi.check_cpu_budget_us("kvm", "restore", 8), 21_000) - # The spec's binding MSHV boot cell: 4 vCPUs (4.94 ms measured). - self.assertEqual(time_abi.check_cpu_budget_us("mshv", "boot", 4), 6_000) - self.assertEqual(time_abi.check_cpu_budget_us("mshv", "restore", 8), 6_000) - self.assertEqual(time_abi.check_cpu_budget_us("whp", "restore", 1), 35_000) - # The spec's binding WHP cells: boot at 2 vCPUs (21.84 ms measured on an - # 8573C runner), capture at 1 (0.986 ms), and restore at 4 (42.20 ms). - self.assertEqual(time_abi.check_cpu_budget_us("whp", "boot", 2), 26_500) - self.assertEqual(time_abi.check_cpu_budget_us("whp", "capture", 1), 1_200) - self.assertEqual(time_abi.check_cpu_budget_us("whp", "capture", 2), 1_600) - self.assertEqual(time_abi.check_cpu_budget_us("whp", "restore", 4), 53_000) - self.assertIsNone(time_abi.check_cpu_budget_us("hvf", "boot", 1)) - self.assertIsNone(time_abi.check_cpu_budget_us("kvm", "runtime", 1)) - self.assertIsNone(time_abi.check_cpu_budget_us("kvm", "boot", 0)) - - -class MonitorTests(unittest.TestCase): - def test_classifies_commands(self): - monitor = TimeAbiMonitor(KVM_BOOT) - self.assertEqual(monitor.backend, "kvm") - self.assertTrue(monitor.cold_boot) - restore = TimeAbiMonitor(MSHV_RESTORE) - self.assertEqual(restore.backend, "mshv") - self.assertFalse(restore.cold_boot) - - def test_records_markers_across_chunk_boundaries(self): - monitor = TimeAbiMonitor(KVM_BOOT) - data = ( - "[ 0.1] early\n" - + " ALPINE-MICROVM-BOOT-OK: 3.22.1\n" - + BOOT_LINE - + RUNTIME_LINE - + "NVX-TIME-ABI: v=1 phase=restore status=ok cpus=4 tsc_hz=2194804000 " - + "lapic_hz=1000000000 generation=1 elapsed_us=12\n" - ).encode() - for index in range(0, len(data), 7): - monitor.feed(data[index : index + 7]) - self.assertTrue(monitor.guest_booted) - self.assertEqual(monitor.require_boot("test", online_cpus=4)["cpus"], "4") - assert monitor.runtime is not None - self.assertEqual(monitor.runtime["status"], "unsynchronized") - self.assertEqual(monitor.restores[0]["generation"], "1") - # The runtime line is the discipline's state and never fails a run. - TimeAbiMonitor(KVM_BOOT).feed( - RUNTIME_LINE.replace("unsynchronized", "uncertain").encode() - ) - - def test_reports_a_check_that_stayed_pending(self): - pending = ( - "NVX-TIME-ABI: v=1 phase=restore status=pending cpus=4 " - "tsc_hz=2194804000 lapic_hz=200000000 generation=1\n" - ) - with self.assertRaisesRegex( - TimeAbiFailure, - "restore check was still pending when nvx-time status stopped " - "waiting after 30 s", - ): - TimeAbiMonitor(MSHV_RESTORE).feed(pending.encode() + STATUS_OK.encode()) - - def test_fails_fast_on_violation_events_and_failed_checks(self): - monitor = TimeAbiMonitor(KVM_BOOT) - with self.assertRaisesRegex(TimeAbiFailure, "violation G_TSC_UNSTABLE"): - monitor.feed( - b"NVX-TIME-ABI-VIOLATION: v=1 code=G_TSC_UNSTABLE source=watcher " - b'phase=runtime generation=0 boottime_ns=1 detail="Marking"\n' - ) - self.assertIn("G_TSC_UNSTABLE", monitor.violation or "") - with self.assertRaisesRegex(TimeAbiFailure, "boot check C9 failed: token"): - TimeAbiMonitor(KVM_BOOT).feed( - b'NVX-TIME-ABI: v=1 phase=boot status=fail check=C9 detail="token"\n' - ) - with self.assertRaisesRegex(TimeAbiFailure, "malformed"): - TimeAbiMonitor(KVM_BOOT).feed(b"NVX-TIME-ABI: v=1 phase=boot\n") - with self.assertRaisesRegex(TimeAbiFailure, "unknown time ABI marker status"): - TimeAbiMonitor(KVM_BOOT).feed( - b"NVX-TIME-ABI: v=1 phase=boot status=maybe\n" - ) - - def test_rejects_a_report_only_boot_when_a_status_query_exits(self): - monitor = TimeAbiMonitor(KVM_BOOT) - monitor.feed( - b"NVX-TIME-REPORT-VIOLATION: v=1 code=G_TSC_WARP source=watcher " - b'phase=boot generation=0 boottime_ns=1 detail="x"\n' - b"NVX-GUEST-BOOT-OK: alpine\n" - ) - self.assertTrue(monitor.report_only) - # A report-only guest whose check failed exits 1; the failure names - # report-only mode, not the exit status. - report = ( - b"NVX-TIME-REPORT: v=1 phase=boot status=fail cpus=1 tsc_hz=2194804000 " - b"lapic_hz=1000000000 generation=0 elapsed_us=9 failures=3\n" - ) - with self.assertRaisesRegex(TimeAbiFailure, "report-only"): - monitor.feed(report + b"NVX-TIME-STATUS-EXIT status=1\n") - restored = TimeAbiMonitor(MSHV_RESTORE) - with self.assertRaisesRegex( - TimeAbiFailure, - "restore marker before nvx-time status exited; .*report-only", - ): - restored.feed( - report.replace(b"phase=boot", b"phase=restore") - + b"NVX-TIME-STATUS-EXIT status=0\n" - ) - - def test_status_queries_require_the_line_of_the_launch_kind(self): - # A quiet guest reaches its shell without printing any time ABI line. - quiet = TimeAbiMonitor(KVM_BOOT) - quiet.feed(b" ALPINE-MICROVM-BOOT-OK: 3.22.1\nNVX-GUEST-BOOT-OK: alpine\n") - self.assertTrue(quiet.guest_booted) - with self.assertRaisesRegex( - TimeAbiFailure, - "nvx-time status did not finish before the scenario; the guest never " - "reported its time ABI state", - ): - quiet.require_status("the scenario") - # The shell's echo of the query is not its exit line. - quiet.feed( - b'/ # /sbin/nvx-time status; echo "NVX-TIME-STATUS-EXIT status=$?"\n' - ) - self.assertEqual(quiet.status_queries, 0) - with self.assertRaisesRegex( - TimeAbiFailure, - "did not report the NVX-TIME-ABI boot marker before nvx-time status " - "exited; the guest image or OpenVMM does not implement time ABI v1", - ): - quiet.feed(b"NVX-TIME-STATUS-EXIT status=0\r\n") - - booted = TimeAbiMonitor(KVM_BOOT) - booted.feed((BOOT_LINE + RUNTIME_LINE + STATUS_OK).encode()) - self.assertEqual(booted.status_queries, 1) - booted.require_status("the scenario") - self.assertEqual(booted.require_boot("x", online_cpus=4)["cpus"], "4") - mismatched = BOOT_LINE.replace("lapic_hz=1000000000", "lapic_hz=200000000") - monitor = TimeAbiMonitor(KVM_BOOT) - monitor.feed(mismatched.encode()) - with self.assertRaisesRegex(TimeAbiFailure, "is not the kvm rate"): - monitor.feed(b"NVX-TIME-STATUS-EXIT status=0\n") - - with self.assertRaisesRegex( - TimeAbiFailure, - "did not report an NVX-TIME-ABI restore marker before nvx-time status " - "exited", - ): - TimeAbiMonitor(MSHV_RESTORE).feed(b"NVX-TIME-STATUS-EXIT status=0\n") - restored = TimeAbiMonitor(MSHV_RESTORE) - restored.feed( - RESTORE_LINE.replace("lapic_hz=1000000000", "lapic_hz=200000000").encode() - + RUNTIME_LINE.replace("generation=0", "generation=2").encode() - + b"NVX-TIME-STATUS-EXIT status=0\n" - ) - self.assertEqual( - restored.require_restore("x", online_cpus=4)["generation"], "2" - ) - # After a restore, status prints every recorded phase, oldest first, - # each with the values from when its check ran: the boot and capture - # lines keep their boot-time CPU count, and only the newest line counts. - activated = TimeAbiMonitor(MSHV_RESTORE) - mshv = "lapic_hz=200000000" - activated.feed( - ( - BOOT_LINE.replace("cpus=4", "cpus=1").replace( - "lapic_hz=1000000000", mshv - ) - + BOOT_LINE.replace("phase=boot", "phase=capture") - .replace("cpus=4", "cpus=1") - .replace("lapic_hz=1000000000", mshv) - + RESTORE_LINE.replace("generation=2", "generation=1").replace( - "lapic_hz=1000000000", mshv - ) - + RUNTIME_LINE.replace("generation=0", "generation=1") - + STATUS_OK - ).encode() - ) - self.assertEqual( - activated.require_restore("x", online_cpus=4, generation=1)["cpus"], "4" - ) - assert activated.boot is not None - self.assertEqual(activated.boot["cpus"], "1") - # A launch that neither boots a kernel nor restores has no line to require. - TimeAbiMonitor(["openvmm"]).feed(b"NVX-TIME-STATUS-EXIT status=0\n") - - with self.assertRaisesRegex( - TimeAbiFailure, "exited 2: the guest image has no status subcommand" - ): - # An old nvx-time's usage text can end without a newline. - TimeAbiMonitor(KVM_BOOT).feed( - b"usage: nvx-time boot|exhaustiveNVX-TIME-STATUS-EXIT status=2\n" - ) - with self.assertRaisesRegex(TimeAbiFailure, "nvx-time status exited 1$"): - TimeAbiMonitor(KVM_BOOT).feed( - BOOT_LINE.encode() + b"NVX-TIME-STATUS-EXIT status=1\n" - ) - - def test_builds_the_status_query(self): - self.assertEqual( - time_abi.status_script(), - '/sbin/nvx-time status; echo "NVX-TIME-STATUS-EXIT status=$?"\n', - ) - # The harness waits longer than the guest, which reports pending checks. - self.assertGreater( - time_abi.STATUS_TIMEOUT_SECONDS, time_abi.STATUS_WAIT_SECONDS - ) - - def test_validates_boot_marker_fields_against_the_backend(self): - cases = ( - ("v=1", "v=2", "version"), - ("lapic_hz=1000000000", "lapic_hz=200000000", "kvm rate"), - ("tsc_hz=2194804000", "tsc_hz=400000000", "outside 500 MHz"), - ("tsc_hz=2194804000", "tsc_hz=fast", "not an integer"), - ("generation=0", "generation=3", "not 0 at cold boot"), - ("cpus=4", "cpus=2", "cpus=2 is not 4"), - ) - for valid, invalid, message in cases: - fields = time_abi.parse_marker(BOOT_LINE.replace(valid, invalid)) - assert fields is not None - with self.subTest(invalid=invalid): - with self.assertRaisesRegex(TimeAbiFailure, message): - time_abi.validate_boot_marker(fields, backend="kvm", online_cpus=4) - mshv = time_abi.parse_marker( - BOOT_LINE.replace("lapic_hz=1000000000", "lapic_hz=200000000") - ) - assert mshv is not None - time_abi.validate_boot_marker(mshv, backend="mshv") - - def test_validates_restore_marker_fields_against_the_backend(self): - restore = time_abi.parse_marker(RESTORE_LINE) - assert restore is not None - time_abi.validate_restore_marker(restore, backend="kvm", online_cpus=4) - cases = ( - ("generation=2", "generation=0", "generation=0 is not 1 or later"), - ("generation=2", "generation=x", "generation=x is not 1 or later"), - ("phase=restore", "phase=boot", "not a passing restore check"), - ("lapic_hz=1000000000", "lapic_hz=200000000", "kvm rate"), - ("cpus=4", "cpus=8", "cpus=8 is not 4"), - ) - for valid, invalid, message in cases: - fields = time_abi.parse_marker(RESTORE_LINE.replace(valid, invalid)) - assert fields is not None - with self.subTest(invalid=invalid): - with self.assertRaisesRegex( - TimeAbiFailure, f"restore marker is invalid: .*{message}" - ): - time_abi.validate_restore_marker( - fields, backend="kvm", online_cpus=4 - ) - # A restore of a snapshot captured at generation 0 carries generation 1. - with self.assertRaisesRegex(TimeAbiFailure, "generation=2 is not 1$"): - time_abi.validate_restore_marker(restore, backend="kvm", generation=1) - time_abi.validate_restore_marker(restore, backend="kvm", generation=2) - - def test_classifies_time_abi_exit_statuses_with_the_violation_event(self): - monitor = TimeAbiMonitor(MSHV_RESTORE) - monitor.check_exit(0) - monitor.check_exit(None) - with self.assertRaisesRegex( - TimeAbiFailure, "status 195 \\(restore repair\\): no NVX-TIME-ABI" - ): - monitor.check_exit(195) - try: - monitor.feed( - b"NVX-TIME-ABI-VIOLATION: v=1 code=G_REPAIR_SAMPLE source=repair " - b'phase=restore generation=1 boottime_ns=9 detail="epsilon"' - ) - monitor.finish() - except TimeAbiFailure: - pass - with self.assertRaisesRegex(TimeAbiFailure, "status 195.*G_REPAIR_SAMPLE"): - monitor.check_exit(195) - - def test_bounds_an_unterminated_line(self): - monitor = TimeAbiMonitor(KVM_BOOT) - monitor.feed(b"x" * (time_abi._MAX_PENDING_LINE + 10)) # pyright: ignore[reportPrivateUsage] - monitor.feed(b"\n" + BOOT_LINE.encode()) - self.assertIsNotNone(monitor.boot) - - def test_records_the_first_fatal_time_abi_error_for_exit_errors(self): - monitor = TimeAbiMonitor(KVM_BOOT) - monitor.feed(b"fatal error: failed to open the kernel\r\n") - self.assertIsNone(monitor.fatal) - self.assertEqual(str(monitor.exit_error(1)), "OpenVMM exited with status 1") - # A guest prompt without a newline can precede OpenVMM's line. - monitor.feed(FATAL + b" 0: [E_TSC_SYNC_UNSUPPORTED] no synchronized set\r\n") - monitor.feed(b"fatal error: [E_TEST_HOOK] later\r\n") - self.assertEqual(monitor.fatal, FATAL_DETAIL) - # The line only explains the exit; it never changes its classification. - monitor.check_exit(1) - self.assertEqual( - str(monitor.exit_error(1, "snapshot source", "during teardown")), - f"snapshot source exited with status 1 during teardown: {FATAL_DETAIL}", - ) - - -class WarpProbeTests(unittest.TestCase): - def test_accepts_conclusive_passing_measurements(self): - results = time_abi.check_warp_probe(warp_output(), cpus=4, context="test") - self.assertEqual(results[0]["max_abs_offset_ns"], "15") - single = time_abi.check_warp_probe( - warp_output(pairs=0, offset=0), cpus=1, context="one vCPU" - ) - self.assertEqual(single[0]["pairs"], "0") - self.assertEqual( - time_abi.warp_probe_command(), - "/sbin/nvx-time-probe warp --bound-ns 1000", - ) - self.assertTrue(time_abi.warp_probe_command(cpus="0-3").endswith("--cpus 0-3")) - - def test_rejects_skew_stalls_and_inconclusive_measurements(self): - cases = ( - (warp_output(backward=1001), "max_backward_ns=1001"), - (warp_output(offset=1500, verdict="FAIL"), "max_abs_offset_ns=1500"), - (warp_output(stalled=1, verdict="FAIL"), "1 CPU pair"), - (warp_output(conclusive=0), "inconclusive"), - (warp_output(pairs=3), "measured 3 CPU pairs instead of 6"), - (warp_output(verdict="FAIL"), "verdict=FAIL"), - ("nothing here\n", "printed no summary"), - ( - warp_output().split("NVX-TIME-PROBE warp-detail")[0], - "1 summaries and 0 detail", - ), - ( - "NVX-TIME-PROBE warp pairs=6\nNVX-TIME-PROBE warp-detail x=1\n", - "incomplete", - ), - ('NVX-TIME-PROBE warp detail="\n', "malformed"), - ) - for text, message in cases: - with self.subTest(message=message): - with self.assertRaisesRegex(TimeAbiFailure, message): - time_abi.check_warp_probe(text, cpus=4, context="restore 4") - - def test_counts_rounds_and_names_the_failing_round(self): - self.assertEqual( - [time_abi.warp_rounds(cpus) for cpus in (1, 2, 8)], - [1, time_abi.WARP_PROBE_ROUNDS, time_abi.WARP_PROBE_ROUNDS], - ) - self.assertEqual(time_abi.CI_WARP_GAPS, ("1",)) - self.assertEqual( - [ - time_abi.warp_rounds(cpus, time_abi.QUALIFICATION_WARP_GAPS) - for cpus in (1, 8) - ], - [1, 5], - ) - with self.assertRaisesRegex(ValueError, "invalid warp probe idle gaps"): - time_abi.warp_probe_script(("1; reboot",)) - two = warp_output() + warp_output(offset=30) - results = time_abi.check_warp_probe(two, cpus=4, context="boot", rounds=2) - self.assertEqual( - [result["max_abs_offset_ns"] for result in results], ["15", "30"] - ) - with self.assertRaisesRegex(TimeAbiFailure, "ran 1 rounds instead of 2"): - time_abi.check_warp_probe(warp_output(), cpus=4, context="boot", rounds=2) - with self.assertRaisesRegex( - TimeAbiFailure, "failed in round 2: max_backward_ns=2000" - ): - time_abi.check_warp_probe( - warp_output() + warp_output(backward=2000, verdict="FAIL"), - cpus=4, - context="boot", - ) - - def test_warp_fragment_idles_between_rounds_and_powers_off_on_failure(self): - shell = shutil.which("sh") - if shell is None and (git := shutil.which("git")) is not None: - candidate = Path(git).parent.parent / "bin" / "sh.exe" - shell = str(candidate) if candidate.is_file() else None - if shell is None: - self.skipTest("POSIX shell is unavailable") - fragment = ( - time_abi.warp_probe_script() - .replace(time_abi.WARP_PROBE_PATH, "probe") - .replace("nvx-exit", "nvx_exit") - ) - - def run_fragment( - fragment: str, online: str, failing_round: int = 0 - ) -> subprocess.CompletedProcess[str]: - assert shell is not None - return subprocess.run( - [shell], - input=( - f"cat() {{ printf '%s\\n' '{online}'; }}\n" - "calls=0\n" - "probe() {\n" - " calls=$((calls + 1))\n" - ' printf "probe %s\\n" "$*"\n' - f' [ "$calls" -ne {failing_round} ]\n' - "}\n" - 'sleep() { printf "SLEEP %s\\n" "$1"; }\n' - 'nvx_exit() { printf "NVX-EXIT %s\\n" "$1"; }\n' - + fragment - + "echo after\n" - ), - text=True, - capture_output=True, - timeout=10, - check=False, - ) - - for online, failing_round, calls, sleeps in ( - ("0-3", 0, 2, 1), - ("0-3", 2, 2, 1), - ("0-3", 1, 1, 0), - ("0", 0, 1, 0), - ): - with self.subTest(online=online, failing_round=failing_round): - result = run_fragment(fragment, online, failing_round) - lines = result.stdout.splitlines() - self.assertEqual( - lines.count(f"probe warp --bound-ns 1000 --cpus {online}"), - calls, - result.stdout + result.stderr, - ) - self.assertEqual( - lines.count(f"SLEEP {time_abi.WARP_PROBE_IDLE_SECONDS}"), sleeps - ) - if failing_round: - self.assertEqual(result.returncode, 97) - self.assertIn( - f"NVX-WARP-PROBE-FAIL status=1 round={failing_round}", lines - ) - self.assertIn("NVX-EXIT 97", lines) - self.assertNotIn("NVX-WARP-PROBE-OK", lines) - self.assertNotIn("after", lines) - else: - self.assertEqual(result.returncode, 0, result.stderr) - self.assertIn("NVX-WARP-PROBE-OK", lines) - self.assertIn("after", lines) - # H6's qualification schedule idles for each gap in order. - qualification = fragment.replace( - 'warp_gaps="1"', - f'warp_gaps="{" ".join(time_abi.QUALIFICATION_WARP_GAPS)}"', - ) - self.assertEqual( - qualification, - time_abi.warp_probe_script(time_abi.QUALIFICATION_WARP_GAPS) - .replace(time_abi.WARP_PROBE_PATH, "probe") - .replace("nvx-exit", "nvx_exit"), - ) - result = run_fragment(qualification, "0-7") - probe = "probe warp --bound-ns 1000 --cpus 0-7" - self.assertEqual( - result.stdout.splitlines(), - [ - probe, - "SLEEP 0.1", - probe, - "SLEEP 1", - probe, - "SLEEP 5", - probe, - "SLEEP 1", - probe, - "NVX-WARP-PROBE-OK", - "after", - ], - result.stderr, - ) - - -VIOLATION = ( - b"NVX-TIME-ABI-VIOLATION: v=1 code=G_RCU_STALL source=watcher phase=runtime " - b'generation=1 boottime_ns=99 detail="rcu: INFO: rcu_preempt self-detected stall"\n' -) - - -class FakeProcess: - pid = 123 - - def __init__(self, status: int) -> None: - self.status = status - self.exited = False - self.returncode: int | None = None - - def poll(self) -> int | None: - if self.exited: - self.returncode = self.status - return self.returncode - - def wait(self, timeout: float | None = None) -> int: - del timeout - self.exited = True - self.returncode = self.status - return self.status - - def terminate(self) -> None: - self.exited = True - - def kill(self) -> None: - self.exited = True - - -class FakeInteraction: - def __init__(self, chunks: list[bytes], status: int) -> None: - self.process = FakeProcess(status) - self.chunks = chunks - self.writes: list[bytes] = [] - - def read_output(self, chunks: queue.Queue[bytes | None]) -> None: - for chunk in self.chunks: - chunks.put(chunk) - self.process.exited = True - chunks.put(None) - - def read_stderr(self, chunks: queue.Queue[bytes | None]) -> None: - chunks.put(None) - - def write_input(self, data: bytes) -> None: - self.writes.append(data) - - def close(self) -> None: - pass - - -class ScriptedInteraction(FakeInteraction): - """A guest that answers each console write with the next scripted reply - and exits after the last one.""" - - def __init__( - self, chunks: list[bytes], status: int, replies: list[list[bytes]] - ) -> None: - super().__init__(chunks, status) - self.replies = list(replies) - self._sink: queue.Queue[bytes | None] | None = None - - def read_output(self, chunks: queue.Queue[bytes | None]) -> None: - self._sink = chunks - for chunk in self.chunks: - chunks.put(chunk) - if not self.replies: - self._exit() - - def write_input(self, data: bytes) -> None: - self.writes.append(data) - assert self._sink is not None - for chunk in self.replies.pop(0): - self._sink.put(chunk) - if not self.replies: - self._exit() - - def _exit(self) -> None: - assert self._sink is not None - self.process.exited = True - self._sink.put(None) - - -class RunnerWiringTests(unittest.TestCase): - @staticmethod - def _fake_exit(process: FakeProcess, _timeout: float) -> int: - """Wait for a fake OpenVMM, whose PID is not a real process.""" - return process.wait() - - @staticmethod - def _teardown_timeout(process: FakeProcess, timeout: float) -> int: - raise subprocess.TimeoutExpired(str(process.pid), timeout) - - def _measure( - self, - chunks: list[bytes], - status: int, - command: list[str], - *, - teardown_mode: str = "guest-exit", - teardown_timeout: bool = False, - log_path: Path | None = None, - ): - interaction = FakeInteraction(chunks, status) - with ( - patch.object(benchmark, "InteractiveProcess", return_value=interaction), - patch.object(benchmark, "live_peak_rss_bytes", return_value=1024), - # Never pidfd_open the fake's PID. - patch.object( - benchmark, - "wait_for_process_exit", - side_effect=( - self._teardown_timeout if teardown_timeout else self._fake_exit - ), - ), - ): - return benchmark.measure_once( - command, - environment={}, - timeout=5, - marker=benchmark.RESTORE_MARKER, - marker_must_be_line=True, - teardown_mode=teardown_mode, - guest_exit_prequeued=True, - log_path=log_path, - ) - - def test_measure_once_fails_fast_on_a_violation_event(self): - with self.assertRaisesRegex(RuntimeError, "violation G_RCU_STALL") as raised: - self._measure( - [b"restored\n", VIOLATION, benchmark.RESTORE_MARKER + b"\n"], - 194, - MSHV_RESTORE, - ) - self.assertIn("--- OpenVMM output ---", str(raised.exception)) - - def test_measure_once_classifies_a_time_abi_power_off(self): - with self.assertRaisesRegex( - RuntimeError, "status 195 \\(restore repair\\): no NVX-TIME-ABI" - ): - self._measure([b"restoring\n"], 195, MSHV_RESTORE) - with self.assertRaisesRegex(RuntimeError, "OpenVMM exited with status 1"): - self._measure([b"restoring\n"], 1, MSHV_RESTORE) - - def test_exit_errors_lead_with_openvmm_fatal_time_abi_code(self): - expected = f"exited with status 1: {re.escape(FATAL_DETAIL)}\n" - with self.assertRaisesRegex(RuntimeError, f"^OpenVMM {expected}"): - self._measure([b"restoring\n", FATAL], 1, MSHV_RESTORE) - interaction = FakeInteraction([b"booting\n", FATAL], 1) - with tempfile.TemporaryDirectory() as temporary: - with patch.object( - benchmark, "InteractiveProcess", return_value=interaction - ): - with self.assertRaisesRegex( - RuntimeError, f"^snapshot source {expected}" - ): - benchmark.capture_snapshot( - KVM_BOOT, Path(temporary) / "snapshot", timeout=5 - ) - log_path = Path(temporary) / "process.log" - with patch.object( - openvmm_process, - "InteractiveProcess", - return_value=FakeInteraction([FATAL], 1), - ): - with openvmm_process.OpenvmmProcess(KVM_BOOT, log_path) as process: - with self.assertRaisesRegex( - RuntimeError, - f"^OpenVMM exited with status 1 before .*: " - f"{re.escape(FATAL_DETAIL)}\n", - ): - process.wait_for(b"NEVER", 1) - # An expected failure still returns its status and output. - with patch.object( - openvmm_process, - "InteractiveProcess", - return_value=FakeInteraction([FATAL], 1), - ): - with openvmm_process.OpenvmmProcess(KVM_BOOT, log_path) as process: - self.assertEqual(process.wait(1).returncode, 1) - - def test_measure_once_classifies_a_power_off_during_teardown(self): - interaction = FakeInteraction([benchmark.RESTORE_MARKER + b"\n"], 194) - with ( - patch.object(benchmark, "InteractiveProcess", return_value=interaction), - patch.object(benchmark, "live_peak_rss_bytes", return_value=1024), - patch.object(benchmark, "wait_for_process_exit", return_value=194), - ): - with self.assertRaisesRegex(RuntimeError, "status 194 \\(runtime"): - benchmark.measure_once( - MSHV_RESTORE, - environment={}, - timeout=5, - marker=benchmark.RESTORE_MARKER, - marker_must_be_line=True, - guest_exit_prequeued=True, - ) - - def test_measure_once_scans_the_console_after_the_marker(self): - # The guest can print a violation during teardown and still exit 0, - # when its power-off loses the race against the queued guest exit. - marker = benchmark.RESTORE_MARKER + b"\n" - with tempfile.TemporaryDirectory() as temporary: - log_path = Path(temporary) / "restore.log" - for logged in (False, True): - with self.subTest(logged=logged): - with self.assertRaisesRegex( - RuntimeError, "violation G_RCU_STALL" - ) as raised: - self._measure( - [marker, VIOLATION], - 0, - MSHV_RESTORE, - log_path=log_path if logged else None, - ) - self.assertIn("--- OpenVMM output ---", str(raised.exception)) - self.assertIn(VIOLATION, log_path.read_bytes()) - _elapsed, peak, teardown, _wall = self._measure( - [marker, b"~ # "], 0, MSHV_RESTORE - ) - self.assertEqual(peak, 1024) - self.assertIsNotNone(teardown) - - def test_measure_once_scans_an_unterminated_last_line(self): - marker = benchmark.RESTORE_MARKER + b"\n" - failed_check = ( - b'NVX-TIME-ABI: v=1 phase=boot status=fail check=C4 detail="kvm-clock"' - ) - for line, error in ( - (VIOLATION.rstrip(b"\n"), "violation G_RCU_STALL"), - (failed_check, "boot check C4 failed: kvm-clock"), - ): - with self.subTest(error=error): - with self.assertRaisesRegex(RuntimeError, error): - self._measure([marker, line], 0, MSHV_RESTORE) - - def test_measure_once_scans_the_console_after_a_teardown_timeout(self): - # The harness kills OpenVMM and returns the sample without a teardown - # time, unless the console or a power-off that beat the kill fails. - marker = benchmark.RESTORE_MARKER + b"\n" - _elapsed, peak, teardown, _wall = self._measure( - [marker], -9, MSHV_RESTORE, teardown_timeout=True - ) - self.assertEqual(peak, 1024) - self.assertIsNone(teardown) - with self.assertRaisesRegex(RuntimeError, "violation G_RCU_STALL"): - self._measure([marker, VIOLATION], -9, MSHV_RESTORE, teardown_timeout=True) - with self.assertRaisesRegex(RuntimeError, "status 194 \\(runtime violation"): - self._measure([marker], 194, MSHV_RESTORE, teardown_timeout=True) - - def test_measure_once_scans_the_console_of_a_terminated_openvmm(self): - # The status of the termination, -15 on Linux and 1 on Windows, never - # fails the launch, but the console and a power-off that beat the - # termination do. - marker = benchmark.RESTORE_MARKER + b"\n" - for status in (-15, 1): - with self.subTest(status=status): - result = self._measure( - [marker], status, MSHV_RESTORE, teardown_mode="host-terminate" - ) - self.assertEqual(result[1], 1024) - with self.assertRaisesRegex(RuntimeError, "violation G_RCU_STALL"): - self._measure( - [marker, VIOLATION], - status, - MSHV_RESTORE, - teardown_mode="host-terminate", - ) - with self.assertRaisesRegex(RuntimeError, "status 194 \\(runtime violation"): - self._measure([marker], 194, MSHV_RESTORE, teardown_mode="host-terminate") - - def test_measure_once_failure_paths_scan_without_masking_their_error(self): - # The rest of the console still reaches the monitor and the log, but - # no time ABI failure is raised over the error being handled. - with tempfile.TemporaryDirectory() as temporary: - log_path = Path(temporary) / "restore.log" - interaction = FakeInteraction([b"NVX-TEST-FAIL broken\n", VIOLATION], 1) - with patch.object( - benchmark, "InteractiveProcess", return_value=interaction - ): - with self.assertRaises(benchmark.GuestFailureReported) as reported: - benchmark.measure_once( - MSHV_RESTORE, - environment={}, - timeout=5, - marker=benchmark.RESTORE_MARKER, - marker_must_be_line=True, - guest_exit_prequeued=True, - log_path=log_path, - failure_marker=b"NVX-TEST-FAIL", - ) - self.assertEqual(reported.exception.line, "NVX-TEST-FAIL broken") - self.assertIn(VIOLATION, log_path.read_bytes()) - - killed = threading.Event() - - class KilledProcess(FakeProcess): - def kill(self) -> None: - super().kill() - killed.set() - - class LateEventInteraction(FakeInteraction): - """A guest whose event reaches the console as OpenVMM stops.""" - - def read_output(self, chunks: queue.Queue[bytes | None]) -> None: - chunks.put(b"booting\n") - killed.wait(5) - chunks.put(VIOLATION) - chunks.put(None) - - interaction = LateEventInteraction([], -9) - interaction.process = KilledProcess(-9) - with patch.object(benchmark, "InteractiveProcess", return_value=interaction): - with self.assertRaisesRegex( - RuntimeError, "^guest marker was not observed within 0.2s" - ) as timed_out: - benchmark.measure_once( - MSHV_RESTORE, - environment={}, - timeout=0.2, - marker=benchmark.RESTORE_MARKER, - marker_must_be_line=True, - guest_exit_prequeued=True, - ) - self.assertIn("G_RCU_STALL", str(timed_out.exception)) - - def test_run_guest_script_classifies_a_conformance_power_off(self): - interaction = FakeInteraction( - [ - b'NVX-TIME-ABI: v=1 phase=boot status=fail check=C4 detail="kvm-clock"', - ], - 193, - ) - with patch.object(benchmark, "InteractiveProcess", return_value=interaction): - with self.assertRaisesRegex( - RuntimeError, "boot check C4 failed: kvm-clock" - ): - benchmark.run_guest_script( - KVM_BOOT, "true\n", b"DONE", timeout=5, boot_marker=b"BOOT" - ) - - def test_cold_boot_runners_query_the_status_before_other_input(self): - status = time_abi.status_script().encode() - boot = benchmark.BOOT_MARKER + b": 3.22.1\n" - # A quiet guest that answers the query without a boot line fails, and - # never receives the script. - quiet = ScriptedInteraction([boot], 0, [[STATUS_OK.encode()], [b"DONE\n"]]) - with patch.object(benchmark, "InteractiveProcess", return_value=quiet): - with self.assertRaisesRegex(RuntimeError, "NVX-TIME-ABI boot marker"): - benchmark.run_guest_script( - KVM_BOOT, "true\n", b"DONE", timeout=5, time_abi_status=True - ) - self.assertEqual(quiet.writes, [status]) - answered = [BOOT_LINE.encode(), STATUS_OK.encode()] - conformant = ScriptedInteraction([boot], 0, [answered, [b"DONE\n"]]) - with patch.object(benchmark, "InteractiveProcess", return_value=conformant): - result = benchmark.run_guest_script( - KVM_BOOT, "true\n", b"DONE", timeout=5, time_abi_status=True - ) - self.assertEqual(conformant.writes, [status, b"true\n"]) - self.assertIn("phase=boot status=ok", result["text"]) - # A guest that exits during the query names the query. - exiting = ScriptedInteraction([boot], 0, [[]]) - with patch.object(benchmark, "InteractiveProcess", return_value=exiting): - with self.assertRaisesRegex( - RuntimeError, "nvx-time status did not finish before the guest exited" - ): - benchmark.run_guest_script( - KVM_BOOT, "true\n", b"DONE", timeout=5, time_abi_status=True - ) - # Benchmarks and restores never query: the console stays quiet. - for command in (KVM_BOOT, MSHV_RESTORE): - silent = ScriptedInteraction([boot], 0, [[b"DONE\n"]]) - with patch.object(benchmark, "InteractiveProcess", return_value=silent): - benchmark.run_guest_script( - command, - "true\n", - b"DONE", - timeout=5, - time_abi_status=command is MSHV_RESTORE, - ) - self.assertEqual(silent.writes, [b"true\n"]) - self._measure([boot, benchmark.RESTORE_MARKER + b"\n"], 0, KVM_BOOT) - - def test_capture_snapshot_queries_the_status_before_the_capture_script(self): - boot = benchmark.BOOT_MARKER + b": 3.22.1\n" - answered = [BOOT_LINE.encode(), STATUS_OK.encode()] - for queried in (False, True): - replies = [[b"capture script exited\n"]] - if queried: - replies.insert(0, answered) - interaction = ScriptedInteraction([boot], 0, replies) - with tempfile.TemporaryDirectory() as temporary: - with patch.object( - benchmark, "InteractiveProcess", return_value=interaction - ): - with self.assertRaisesRegex( - RuntimeError, "exited before its snapshot request" - ): - benchmark.capture_snapshot( - KVM_BOOT, - Path(temporary) / "snapshot", - timeout=5, - processors=1, - time_abi_status=queried, - ) - with self.subTest(queried=queried): - capture = benchmark.prepare_snapshot_capture_script( - 1, teardown_mode="guest-exit" - ).encode() - status = time_abi.status_script().encode() - expected = [status, capture] if queried else [capture] - self.assertEqual(interaction.writes, expected) - quiet = ScriptedInteraction([boot], 0, [[STATUS_OK.encode()]]) - with tempfile.TemporaryDirectory() as temporary: - with patch.object(benchmark, "InteractiveProcess", return_value=quiet): - with self.assertRaisesRegex(RuntimeError, "NVX-TIME-ABI boot marker"): - benchmark.capture_snapshot( - KVM_BOOT, - Path(temporary) / "snapshot", - timeout=5, - processors=1, - time_abi_status=True, - ) - self.assertEqual(len(quiet.writes), 1) - - def test_openvmm_process_queries_the_status_before_the_first_input(self): - status = time_abi.status_script().encode() - answered = [BOOT_LINE.encode(), STATUS_OK.encode()] - with tempfile.TemporaryDirectory() as temporary: - log_path = Path(temporary) / "process.log" - # The scenario's substring wait can return before the boot line - # ends; its input still follows the finished query. - guest = ScriptedInteraction( - [b"NVX-GUEST-BOOT-OK:"], 0, [[b" alpine\n", *answered], [b"OK\n"]] - ) - with patch.object( - openvmm_process, "InteractiveProcess", return_value=guest - ): - with openvmm_process.OpenvmmProcess(KVM_BOOT, log_path) as process: - process.wait_for(b"NVX-GUEST-BOOT-OK:", 1) - process.send_line("echo OK") - self.assertEqual(process.time_abi.status_queries, 1) - process.wait_for_line(b"OK", 1) - self.assertEqual(process.wait(1).returncode, 0) - self.assertEqual(guest.writes, [status, b"echo OK\n"]) - - quiet = ScriptedInteraction( - [b"NVX-GUEST-BOOT-OK: alpine\n"], 0, [[STATUS_OK.encode()], []] - ) - with patch.object( - openvmm_process, "InteractiveProcess", return_value=quiet - ): - with openvmm_process.OpenvmmProcess(KVM_BOOT, log_path) as process: - process.wait_for(b"NVX-GUEST-BOOT-OK:", 1) - with self.assertRaisesRegex( - RuntimeError, "did not report the NVX-TIME-ABI boot marker" - ): - process.send_line("echo OK") - self.assertEqual(quiet.writes, [status]) - - # Restores, guests whose shell is on another console, and opted-out - # processes receive only the scenario's input. - for command, output, enabled in ( - (MSHV_RESTORE, b"NVX-GUEST-BOOT-OK: alpine\n", True), - (KVM_BOOT, b"booting\n", True), - (KVM_BOOT, b"NVX-GUEST-BOOT-OK: alpine\n", False), - ): - other = ScriptedInteraction([output], 0, [[]]) - with patch.object( - openvmm_process, "InteractiveProcess", return_value=other - ): - with openvmm_process.OpenvmmProcess( - command, log_path, time_abi_status=enabled - ) as process: - process.wait_for(output.rstrip(b"\n"), 1) - process.send_bytes(b"\x01") - self.assertEqual(process.wait(1).returncode, 0) - with self.subTest(command=command, output=output, enabled=enabled): - self.assertEqual(other.writes, [b"\x01"]) - - def test_capture_snapshot_classifies_a_failed_capture_check(self): - interaction = FakeInteraction([b"booting\n"], 193) - with tempfile.TemporaryDirectory() as temporary: - with patch.object( - benchmark, "InteractiveProcess", return_value=interaction - ): - with self.assertRaisesRegex(RuntimeError, "status 193 \\(conformance"): - benchmark.capture_snapshot( - KVM_BOOT, - Path(temporary) / "snapshot", - timeout=5, - ) - - def test_openvmm_process_reports_violations_and_time_abi_statuses(self): - with tempfile.TemporaryDirectory() as temporary: - log_path = Path(temporary) / "process.log" - with patch.object( - openvmm_process, - "InteractiveProcess", - return_value=FakeInteraction([b"FIRST\n", VIOLATION], 194), - ): - with openvmm_process.OpenvmmProcess(KVM_BOOT, log_path) as process: - process.wait_for(b"FIRST", 1) - with self.assertRaisesRegex(RuntimeError, "G_RCU_STALL"): - process.wait_for(b"NEVER", 1) - self.assertIn(b"G_RCU_STALL", log_path.read_bytes()) - with patch.object( - openvmm_process, - "InteractiveProcess", - return_value=FakeInteraction([b"working\n"], 194), - ): - with openvmm_process.OpenvmmProcess(KVM_BOOT, log_path) as process: - with self.assertRaisesRegex(RuntimeError, "status 194"): - process.wait(1) - with patch.object( - openvmm_process, - "InteractiveProcess", - return_value=FakeInteraction([b"working\n"], 195), - ): - with openvmm_process.OpenvmmProcess(KVM_BOOT, log_path) as process: - with self.assertRaisesRegex(RuntimeError, "status 195"): - process.wait_for_line(b"NEVER", 1) - - def test_openvmm_process_waits_for_time_abi_phase_markers(self): - restore = ( - b"NVX-TIME-ABI: v=1 phase=restore status=ok cpus=2 tsc_hz=2300000000 " - b"lapic_hz=200000000 generation=1 elapsed_us=310\n" - ) - with tempfile.TemporaryDirectory() as temporary: - log_path = Path(temporary) / "process.log" - with patch.object( - openvmm_process, - "InteractiveProcess", - return_value=FakeInteraction([b"x\n", restore], 0), - ): - with openvmm_process.OpenvmmProcess(MSHV_RESTORE, log_path) as process: - marker = process.wait_for_time_abi("restore", 1) - self.assertEqual(marker["generation"], "1") - self.assertEqual(process.wait(1).returncode, 0) - with patch.object( - openvmm_process, - "InteractiveProcess", - return_value=FakeInteraction([b"x\n"], 0), - ): - with openvmm_process.OpenvmmProcess(MSHV_RESTORE, log_path) as process: - with self.assertRaisesRegex(RuntimeError, "restore marker"): - process.wait_for_time_abi("restore", 1) - - def test_tcp_console_monitor_reports_violations(self): - connection, peer = socket.socketpair() - console = openvmm_process.TcpConsole(connection, TimeAbiMonitor(KVM_BOOT)) - peer.sendall(VIOLATION) - with self.assertRaisesRegex(TimeAbiFailure, "G_RCU_STALL"): - console.wait_for(b"NEVER", 1.0) - console.close() - peer.close() - - def test_tcp_console_finish_scans_the_tail(self): - # A violation after the last awaited marker still fails the scenario, - # including an unterminated final line. - connection, peer = socket.socketpair() - console = openvmm_process.TcpConsole(connection, TimeAbiMonitor(KVM_BOOT)) - peer.sendall(b"READY\n") - console.wait_for(b"READY", 1.0) - peer.sendall(VIOLATION.rstrip(b"\n")) - peer.close() - with self.assertRaisesRegex(TimeAbiFailure, "G_RCU_STALL"): - console.finish() - self.assertIn(b"G_RCU_STALL", console.finish(check=False)) - with self.assertRaisesRegex(TimeAbiFailure, "G_RCU_STALL"): - console.finish() - - # Error paths keep the tail in the log without raising over the error. - connection, peer = socket.socketpair() - console = openvmm_process.TcpConsole(connection, TimeAbiMonitor(KVM_BOOT)) - peer.sendall(VIOLATION) - peer.close() - self.assertEqual(console.finish(check=False), VIOLATION) - - def test_tcp_console_queries_a_cold_boot_before_the_first_input(self): - connection, peer = socket.socketpair() - peer.settimeout(1.0) - monitor = TimeAbiMonitor(KVM_BOOT) - console = openvmm_process.TcpConsole(connection, monitor, time_abi_status=True) - peer.sendall(b"NVX-GUEST-BOOT-OK: alpine\n") - console.wait_for(b"NVX-GUEST-BOOT-OK", 1.0) - peer.sendall((BOOT_LINE + RUNTIME_LINE + STATUS_OK).encode()) - console.send_bytes(b"INPUT\n") - console.send_bytes(b"MORE\n") - expected = time_abi.status_script().encode() + b"INPUT\nMORE\n" - sent = b"" - while len(sent) < len(expected): - sent += peer.recv(4096) - self.assertEqual(sent, expected) - self.assertEqual(monitor.status_queries, 1) - self.assertIsNotNone(monitor.boot) - console.close() - peer.close() - - # A query without a passing boot line fails before the input is sent. - connection, peer = socket.socketpair() - console = openvmm_process.TcpConsole( - connection, TimeAbiMonitor(KVM_BOOT), time_abi_status=True - ) - peer.sendall(b"NVX-GUEST-BOOT-OK: alpine\n") - console.wait_for(b"NVX-GUEST-BOOT-OK", 1.0) - peer.sendall(STATUS_OK.encode()) - with self.assertRaisesRegex(TimeAbiFailure, "boot marker"): - console.send_bytes(b"INPUT\n") - console.close() - peer.close() - - def test_tcp_console_waits_for_a_status_query_to_exit(self): - connection, peer = socket.socketpair() - monitor = TimeAbiMonitor(MSHV_RESTORE) - console = openvmm_process.TcpConsole(connection, monitor) - restore = RESTORE_LINE.replace("lapic_hz=1000000000", "lapic_hz=200000000") - peer.sendall((restore + RUNTIME_LINE + STATUS_OK).encode()) - console.wait_for_time_abi_status(1.0) - self.assertEqual(monitor.status_queries, 1) - self.assertEqual(len(monitor.restores), 1) - # A query that reports no restore line fails when it exits. - peer.sendall(STATUS_OK.encode()) - monitor.restores.clear() - with self.assertRaisesRegex(TimeAbiFailure, "restore marker"): - console.wait_for_time_abi_status(1.0) - with self.assertRaisesRegex(TimeoutError, "did not exit"): - console.wait_for_time_abi_status(0.2) - console.close() - peer.close() - unmonitored = socket.socket() - self.addCleanup(unmonitored.close) - with self.assertRaisesRegex(ValueError, "no time ABI monitor"): - openvmm_process.TcpConsole(unmonitored).wait_for_time_abi_status(1.0) - - def test_tcp_console_queries_only_cold_boots_that_ask(self): - for command, enabled in ( - (MSHV_RESTORE, True), - (KVM_BOOT, False), - ): - with self.subTest(command=command, enabled=enabled): - connection, peer = socket.socketpair() - peer.settimeout(1.0) - monitor = TimeAbiMonitor(command) - console = openvmm_process.TcpConsole( - connection, monitor, time_abi_status=enabled - ) - peer.sendall(b"NVX-GUEST-BOOT-OK: alpine\n") - console.wait_for(b"NVX-GUEST-BOOT-OK", 1.0) - console.send_bytes(b"Z") - self.assertEqual(peer.recv(4096), b"Z") - self.assertEqual(monitor.status_queries, 0) - console.close() - peer.close() - - -def doctor_context(root: Path, backend: str = "kvm") -> doctor.DoctorContext: - return doctor.DoctorContext( - backend=backend, - openvmm=root / "openvmm", - kernel=root / "vmlinux", - initrd=root / "initramfs.cpio.gz", - openvmm_args=(), - probe_directory=root / "probe", - timeout=30, - ) - - -def completed(stdout: str, returncode: int = 0, stderr: str = ""): - return subprocess.CompletedProcess(["x"], returncode, stdout, stderr) - - -CPU = { - "vendor": "GenuineIntel", - "family": "6", - "model": "106", - "stepping": "6", - "microcode": "0xffffffff", - "brand": "Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz", - "os": "Linux 6.6.150.1-1.azl3", - "invariant_tsc": "yes", -} -# OpenVMM's cpu_profile FingerprintCheck::summary_line, printed on stderr. -PROFILE_PASS = ( - "NVX-CPU-PROFILE: status=pass backend=kvm generation=icelake-sp " - f"profile=intel.icelake-sp.v1 profile_digest=sha256:{'a3' * 32} " - "surface_digest=sha256:6286956acbef host_invariant_tsc=yes\n" -) - - -class DoctorTests(unittest.TestCase): - def setUp(self): - self.temporary = tempfile.TemporaryDirectory() - self.root = Path(self.temporary.name) - - def tearDown(self): - self.temporary.cleanup() - - def test_check_lines_escape_details_for_field_parsers(self): - line = doctor.CheckResult("H5", False, 'say "x" \\ y').line() - self.assertTrue(line.startswith("NVX-DOCTOR: check=H5 status=fail ")) - fields = time_abi.parse_fields(line.removeprefix(doctor.DOCTOR_PREFIX)) - self.assertEqual(fields["detail"], 'say "x" \\ y') - - def test_backend_check_requires_a_usable_device(self): - context = doctor_context(self.root) - existing: set[str] = set() - - def exists(path: Path) -> bool: - return path.as_posix() in existing - - with ( - patch.object(doctor, "host_is_windows", return_value=False), - patch.object(doctor.Path, "exists", exists), - patch.object(doctor.os, "access", return_value=True) as access, - ): - self.assertIn("does not exist", doctor.check_backend(context).detail) - existing.add("/dev/kvm") - self.assertTrue(doctor.check_backend(context).passed) - existing.add("/dev/mshv") - self.assertIn("select MSHV", doctor.check_backend(context).detail) - existing.discard("/dev/mshv") - access.return_value = False - self.assertIn("not readable", doctor.check_backend(context).detail) - with patch.object(doctor, "host_is_windows", return_value=False): - whp = doctor.check_backend(doctor_context(self.root, "whp")) - self.assertFalse(whp.passed) - self.assertIn("requires Windows", whp.detail) - - def test_cpu_check_names_the_generation_and_records_invariant_tsc_as_evidence( - self, - ): - context = doctor_context(self.root) - with patch.object(doctor, "host_cpu", return_value=dict(CPU)): - result = doctor.check_cpu(context) - self.assertTrue(result.passed) - self.assertIn("generation=icelake-sp", result.detail) - self.assertIn("profile=intel.icelake-sp.v1", result.detail) - self.assertEqual(context.facts["generation"], "icelake-sp") - for unknown in (dict(CPU, model="143"), dict(CPU, model="85", stepping="7")): - with patch.object(doctor, "host_cpu", return_value=unknown): - result = doctor.check_cpu(doctor_context(self.root)) - self.assertFalse(result.passed) - self.assertTrue(result.detail.startswith("[E_PROFILE_HOST_UNKNOWN] ")) - self.assertIn(time_abi.describe_cpu_generations(), result.detail) - alder_lake = dict(CPU, model="154", stepping="3") - with patch.object(doctor, "host_cpu", return_value=alder_lake): - result = doctor.check_cpu(doctor_context(self.root, "whp")) - self.assertTrue(result.passed, result.detail) - self.assertIn("generation=alderlake profile=intel.alderlake.v1", result.detail) - emerald = dict(CPU, model="207", stepping="2") - with patch.object(doctor, "host_cpu", return_value=emerald): - for backend in ("kvm", "mshv", "whp"): - result = doctor.check_cpu(doctor_context(self.root, backend)) - self.assertTrue(result.passed, result.detail) - self.assertIn("profile=intel.emeraldrapids.v1", result.detail) - # The host OS's invariant-TSC flags are evidence, never a gate. - variant = dict(CPU, invariant_tsc="no (missing nonstop_tsc)") - with patch.object(doctor, "host_cpu", return_value=variant): - for backend in ("kvm", "mshv", "whp"): - context = doctor_context(self.root, backend) - result = doctor.check_cpu(context) - self.assertTrue(result.passed, result.detail) - self.assertIn("invariant_tsc=no (missing nonstop_tsc)", result.detail) - self.assertEqual( - context.facts["invariant_tsc"], "no (missing nonstop_tsc)" - ) - - def test_cpu_check_verifies_the_profile_with_openvmm_fingerprint(self): - fingerprint = self.root / "out" / "fingerprint.json" - - def context_with_openvmm(backend: str = "kvm") -> doctor.DoctorContext: - context = doctor_context(self.root, backend) - context.openvmm.write_bytes(b"") - context.fingerprint = fingerprint - return context - - def check( - result: subprocess.CompletedProcess[str], - cpu: dict[str, str] = CPU, - backend: str = "kvm", - ) -> tuple[doctor.CheckResult, doctor.DoctorContext]: - context = context_with_openvmm(backend) - with ( - patch.object(doctor, "host_cpu", return_value=dict(cpu)), - patch.object(doctor.subprocess, "run", return_value=result), - ): - return doctor.check_cpu(context), context - - context = context_with_openvmm() - with ( - patch.object(doctor, "host_cpu", return_value=dict(CPU)), - patch.object( - doctor.subprocess, "run", return_value=completed("", 0, PROFILE_PASS) - ) as run, - ): - result = doctor.check_cpu(context) - self.assertTrue(result.passed, result.detail) - self.assertEqual( - run.call_args.args[0], - [ - str(context.openvmm), - "--hypervisor", - "kvm", - "--cpu-fingerprint", - str(fingerprint), - ], - ) - self.assertEqual(run.call_args.kwargs["env"]["OPENVMM_LOG"], "off") - self.assertTrue(fingerprint.parent.is_dir()) - self.assertIn("surface_digest=sha256:6286956acbef", result.detail) - self.assertEqual(context.facts["profile"], "intel.icelake-sp.v1") - self.assertEqual(context.facts["profile_digest"], f"sha256:{'a3' * 32}") - self.assertEqual(context.facts["surface_digest"], "sha256:6286956acbef") - # A later revision of the generation's profile is fine. - revised = PROFILE_PASS.replace("icelake-sp.v1", "icelake-sp.v2") - self.assertTrue(check(completed("", 0, revised))[0].passed) - # OpenVMM's catalog, not NVX's copy, decides which hosts it serves: a - # generation that only OpenVMM pins passes. - raptor_lake = PROFILE_PASS.replace("icelake-sp", "raptorlake") - result, context = check( - completed("", 0, raptor_lake), dict(CPU, model="183", stepping="1") - ) - self.assertTrue(result.passed, result.detail) - self.assertIn( - "generation=raptorlake profile=intel.raptorlake.v1", result.detail - ) - self.assertEqual(context.facts["generation"], "raptorlake") - self.assertEqual(context.facts["profile"], "intel.raptorlake.v1") - - unsupported = ( - "NVX-CPU-PROFILE: status=fail backend=whp generation=icelake-sp " - "profile=intel.icelake-sp.v1 profile_digest=sha256:a3 " - "surface_digest=sha256:d67f host_invariant_tsc=no " - 'code=E_PROFILE_UNSUPPORTED detail="CPUID 0x7.0 EDX bits 26, 27 are ' - 'not supported; leaf \\"7\\"\\u{a0}subleaf"\n' - ) - result, context = check(completed("", 1, unsupported), backend="whp") - self.assertFalse(result.passed) - self.assertTrue( - result.detail.startswith( - "[E_PROFILE_UNSUPPORTED] OpenVMM's CPU profile check failed (exit 1): " - 'CPUID 0x7.0 EDX bits 26, 27 are not supported; leaf "7"\xa0subleaf;' - ), - result.detail, - ) - unknown = ( - "NVX-CPU-PROFILE: status=fail backend=kvm generation=none profile=none " - "surface_digest=sha256:1 host_invariant_tsc=yes " - 'code=E_PROFILE_HOST_UNKNOWN detail="GenuineIntel family 6 model 143"\n' - ) - result, context = check(completed("", 1, unknown), dict(CPU, model="143")) - self.assertFalse(result.passed) - self.assertTrue(result.detail.startswith("[E_PROFILE_HOST_UNKNOWN] ")) - self.assertIn("[E_PROFILE_HOST_UNKNOWN] OpenVMM's CPU profile", result.detail) - self.assertEqual(context.facts["profile"], "none") - for output, message in ( - ( - completed("", 2, "error: unexpected argument '--cpu-fingerprint'"), - "predates the --cpu-fingerprint tool", - ), - (completed("", 1, "fatal error: no backend"), "fatal error: no backend"), - ( - completed("", 0, PROFILE_PASS.replace("=kvm", "=mshv")), - "fingerprinted the mshv backend", - ), - ( - completed( - "", - 0, - PROFILE_PASS.replace( - "generation=icelake-sp", "generation=emeraldrapids" - ), - ), - "CPU profile intel.icelake-sp.v1 of generation icelake-sp for " - "generation emeraldrapids", - ), - ( - completed( - "", - 0, - PROFILE_PASS.replace( - "profile=intel.icelake-sp.v1", "profile=intel.icelake-sp.v2-rc" - ), - ), - "CPU profile intel.icelake-sp.v2-rc, which is not a catalog profile ID", - ), - ( - completed( - "", - 0, - PROFILE_PASS.replace( - "generation=icelake-sp profile=intel.icelake-sp.v1", - "generation=host profile=intel.host.v1", - ), - ), - "host profile intel.host.v1, which qualification never accepts", - ), - ): - with self.subTest(message=message): - result = check(output)[0] - self.assertFalse(result.passed) - self.assertIn(message, result.detail) - # A passing line that lacks the generation or the profile, or leaves - # one empty, says so instead of classifying a profile that OpenVMM did - # not report. - for output_line, unreported in ( - (PROFILE_PASS.replace("generation=icelake-sp ", ""), "generation"), - ( - PROFILE_PASS.replace("profile=intel.icelake-sp.v1 ", "profile= "), - "profile", - ), - ( - PROFILE_PASS.replace( - "generation=icelake-sp profile=intel.icelake-sp.v1 ", "" - ), - "generation and profile", - ), - ): - with self.subTest(unreported=unreported): - result, context = check(completed("", 0, output_line)) - self.assertFalse(result.passed) - self.assertIn( - "OpenVMM's CPU profile check passed without reporting its " - f"{unreported};", - result.detail, - ) - self.assertNotIn("OpenVMM reported", result.detail) - self.assertEqual( - context.facts["generation"], - "none" if "generation" in unreported else "icelake-sp", - ) - self.assertEqual( - context.facts["profile"], - "none" if "profile" in unreported else "intel.icelake-sp.v1", - ) - missing = doctor_context(self.root / "none") - missing.fingerprint = fingerprint - with patch.object(doctor, "host_cpu", return_value=dict(CPU)): - result = doctor.check_cpu(missing) - self.assertFalse(result.passed) - self.assertIn("was not found", result.detail) - self.assertIn("--no-openvmm", result.detail) - # Without OpenVMM, H2 checks the identity and generation only. - with ( - patch.object(doctor, "host_cpu", return_value=dict(CPU)), - patch.object(doctor.subprocess, "run") as run, - ): - result = doctor.check_cpu(doctor_context(self.root)) - self.assertTrue(result.passed) - self.assertIn("CPU profile not checked", result.detail) - run.assert_not_called() - - def test_reads_the_first_processor_from_cpuinfo(self): - path = self.root / "cpuinfo" - path.write_text( - "processor\t: 0\nvendor_id\t: GenuineIntel\ncpu family\t: 6\n" - "model\t\t: 85\nstepping\t: 4\nmicrocode\t: 0x2007108\n" - "flags\t\t: fpu tsc constant_tsc nonstop_tsc\n\n" - "processor\t: 1\nvendor_id\t: Other\n", - encoding="utf-8", - ) - fields = doctor._linux_cpuinfo(path) # pyright: ignore[reportPrivateUsage] - self.assertEqual(fields["vendor_id"], "GenuineIntel") - self.assertEqual(fields["model"], "85") - self.assertIn("nonstop_tsc", fields["flags"]) - - def test_qualification_rejects_a_host_on_which_auto_falls_back(self): - """auto falls back to a host profile on a CPU that no built-in profile - serves, but qualification does not: H2's fingerprint check never falls - back, and H3 rejects the host profile that OpenVMM's preflight selects, - so CI's microVM jobs still require a built-in profile.""" - granite_rapids = dict( - CPU, model="173", stepping="1", brand="Intel(R) Xeon(R) 6973P-C" - ) - fingerprint = self.root / "fingerprint.json" - unknown = ( - "NVX-CPU-PROFILE: status=fail backend=kvm generation=none profile=none " - "surface_digest=sha256:1 host_invariant_tsc=yes " - 'code=E_PROFILE_HOST_UNKNOWN detail="no pinned CPU profile serves ' - 'the host CPU, GenuineIntel family 6 model 173 stepping 1"\n' - ) - fallback = ( - "NVX-TIME-ABI-VERIFY: v=1 status=ok backend=kvm cpu_profile=intel.host.v1 " - "tsc_hz=2793437000 native_tsc_hz=2793437000 lapic_hz=1000000000 " - "msr_route=ExitToVmm sync=CommonOffset\n" - ) - - def run(command: list[str], **_: object) -> subprocess.CompletedProcess[str]: - if "--cpu-fingerprint" in command: - return completed("", 1, unknown) - return completed(fallback) - - context = doctor_context(self.root) - for path in (context.openvmm, context.kernel, context.initrd): - path.write_bytes(b"") - context.fingerprint = fingerprint - with ( - patch.object(doctor, "host_cpu", return_value=granite_rapids), - patch.object(doctor.subprocess, "run", side_effect=run), - contextlib.redirect_stdout(io.StringIO()) as lines, - ): - cpu, preflight = doctor.run_checks(context, ("H2", "H3")) - self.assertIn("NVX-DOCTOR: check=H2 status=fail", lines.getvalue()) - self.assertIn("NVX-DOCTOR: check=H3 status=fail", lines.getvalue()) - self.assertEqual((cpu.check, cpu.passed), ("H2", False)) - self.assertTrue(cpu.detail.startswith("[E_PROFILE_HOST_UNKNOWN] "), cpu.detail) - self.assertEqual((preflight.check, preflight.passed), ("H3", False)) - self.assertIn( - "cpu_profile=intel.host.v1 (a host profile, to which --cpu-profile auto " - "falls back", - preflight.detail, - ) - # Without OpenVMM, NVX's copy of the catalog fails the CPU too. - context.fingerprint = None - with patch.object(doctor, "host_cpu", return_value=granite_rapids): - result = doctor.check_cpu(context) - self.assertFalse(result.passed) - self.assertTrue(result.detail.startswith("[E_PROFILE_HOST_UNKNOWN] ")) - - def test_identifies_the_host_cpu_signature(self): - HostCpu = time_abi.HostCpu - brand = "12th Gen Intel(R) Core(TM) i9-12900H" - - def registry(key: str, name: str) -> object: - self.assertEqual( - (key, name), (doctor.WINDOWS_PROCESSOR_KEY, "ProcessorNameString") - ) - return f"{brand} " - - with ( - patch.object(doctor, "host_is_windows", return_value=True), - patch.object(doctor, "_windows_registry_value", side_effect=registry), - ): - for processor, expected in ( - ( - "Intel64 Family 6 Model 154 Stepping 3, GenuineIntel", - HostCpu("GenuineIntel", 6, 154, 3, brand), - ), - ( - "AMD64 Family 25 Model 33 Stepping 0, AuthenticAMD", - HostCpu("AuthenticAMD", 25, 33, 0, brand), - ), - ("", None), - ): - with ( - self.subTest(processor=processor), - patch.object(doctor.platform, "processor", return_value=processor), - ): - self.assertEqual(doctor.host_cpu_signature(), expected) - # A brand string that the registry lacks leaves the signature. - with ( - patch.object(doctor, "host_is_windows", return_value=True), - patch.object( - doctor.platform, - "processor", - return_value="Intel64 Family 6 Model 154 Stepping 3, GenuineIntel", - ), - patch.object( - doctor, "_windows_registry_value", side_effect=OSError("no value") - ), - ): - self.assertEqual( - doctor.host_cpu_signature(), HostCpu("GenuineIntel", 6, 154, 3) - ) - cpuinfo = { - "vendor_id": "AuthenticAMD", - "cpu family": "25", - "model": "17", - "stepping": "1", - } - with patch.object(doctor, "host_is_windows", return_value=False): - for fields, expected in ( - (cpuinfo, HostCpu("AuthenticAMD", 25, 17, 1)), - ( - {**cpuinfo, "model name": "AMD EPYC 9V74 80-Core Processor"}, - HostCpu( - "AuthenticAMD", 25, 17, 1, "AMD EPYC 9V74 80-Core Processor" - ), - ), - ({**cpuinfo, "model": "?"}, None), - ({"vendor_id": "AuthenticAMD"}, None), - ): - with ( - self.subTest(fields=fields), - patch.object(doctor, "_linux_cpuinfo", return_value=fields), - ): - self.assertEqual(doctor.host_cpu_signature(), expected) - with patch.object( - doctor, "_linux_cpuinfo", side_effect=FileNotFoundError("cpuinfo") - ): - self.assertIsNone(doctor.host_cpu_signature()) - - def test_preflight_parses_openvmm_verification(self): - # The line format of OpenVMM's openvmm_entry verify.rs. - line = ( - "NVX-TIME-ABI-VERIFY: v=1 status=ok backend=kvm " - "cpu_profile=intel.icelake-sp.v1 tsc_hz=2793437000 " - "native_tsc_hz=2793437000 lapic_hz=1000000000 msr_route=ExitToVmm " - "sync=CommonOffset\n" - ) - - def context_with_files(backend: str = "kvm") -> doctor.DoctorContext: - context = doctor_context(self.root, backend) - for path in (context.openvmm, context.kernel, context.initrd): - path.write_bytes(b"") - return context - - context = context_with_files() - with patch.object( - doctor.subprocess, "run", return_value=completed(line) - ) as run: - result = doctor.check_openvmm_preflight(context) - self.assertTrue(result.passed, result.detail) - self.assertEqual(context.facts["native_tsc_hz"], "2793437000") - self.assertEqual(context.facts["profile"], "intel.icelake-sp.v1") - command = run.call_args.args[0] - self.assertEqual(command[-1], "--x-time-abi-verify") - self.assertEqual(command[command.index("--machine") + 1], "microvm") - self.assertEqual(command[command.index("--kernel") + 1], str(context.kernel)) - # --openvmm-arg passes development options, such as a test hook. - hooked = context_with_files() - hooked.openvmm_args = ("--x-time-abi-test-hook", "force-utc-downtime") - with patch.object( - doctor.subprocess, "run", return_value=completed(line) - ) as run: - self.assertTrue(doctor.check_openvmm_preflight(hooked).passed) - self.assertEqual( - run.call_args.args[0][-3:], - ["--x-time-abi-test-hook", "force-utc-downtime", "--x-time-abi-verify"], - ) - failure = ( - "NVX-TIME-ABI-VERIFY: v=1 status=fail backend=kvm " - 'code=E_TSC_SYNC_UNSUPPORTED detail="failed: \\"sync\\" unsupported"\n' - ) - cases = ( - ( - completed(failure, 1), - '[E_TSC_SYNC_UNSUPPORTED] OpenVMM verification failed (exit 1): failed: "sync" unsupported', - ), - ( - completed( - "", 2, "error: unexpected argument '--x-time-abi-verify' found" - ), - "predates the time ABI", - ), - # The flip removed --x-time-abi-v1; passing it is reported as is. - ( - completed("", 2, "error: unexpected argument '--x-time-abi-v1' found"), - "unexpected argument '--x-time-abi-v1' found", - ), - (completed(line.replace("1000000000", "200000000")), "[E_LAPIC_RATE_"), - ( - completed(line.replace("tsc_hz=2793437000", "tsc_hz=4000", 1)), - "[E_TSC_RATE_", - ), - ( - completed(line.replace("backend=kvm", "backend=mshv")), - "verified backend mshv", - ), - # Without H2 in the same run, H3 still requires a catalog profile. - ( - completed(line.replace(" cpu_profile=intel.icelake-sp.v1", "")), - "verified no catalog CPU profile: cpu_profile=none", - ), - ( - completed(line.replace("intel.icelake-sp.v1", "interim.host.kvm.v1")), - "verified no catalog CPU profile: cpu_profile=interim.host.kvm.v1", - ), - # Qualification never accepts a host profile, such as the one to - # which auto falls back on a CPU that no built-in profile serves. - ( - completed(line.replace("intel.icelake-sp.v1", "intel.host.v1")), - "verified no catalog CPU profile: cpu_profile=intel.host.v1 (a host " - "profile, to which --cpu-profile auto falls back on a CPU that no " - "built-in profile serves, and which qualification never accepts)", - ), - ) - for result_value, message in cases: - with self.subTest(message=message): - with patch.object(doctor.subprocess, "run", return_value=result_value): - result = doctor.check_openvmm_preflight(context_with_files()) - self.assertFalse(result.passed) - self.assertIn(message, result.detail) - # H3 records a missing profile as none, as H2 does. - context = context_with_files() - unprofiled = line.replace(" cpu_profile=intel.icelake-sp.v1", "") - with patch.object(doctor.subprocess, "run", return_value=completed(unprofiled)): - self.assertFalse(doctor.check_openvmm_preflight(context).passed) - self.assertEqual(context.facts["profile"], "none") - for selected, passed in ( - ("intel.icelake-sp.v2", True), - ("intel.skylake-sp.v1", False), - # A revision is the lineage plus .v and a number. - ("intel.icelake-sp.v2-rc", False), - # OpenVMM selects catalog profiles, so an interim one is a regression. - ("interim.host.kvm.v1", False), - ): - with self.subTest(selected=selected): - context = context_with_files() - context.facts["profile"] = "intel.icelake-sp.v1" - output = line.replace("intel.icelake-sp.v1", selected) - with patch.object( - doctor.subprocess, "run", return_value=completed(output) - ): - result = doctor.check_openvmm_preflight(context) - self.assertEqual(result.passed, passed, result.detail) - self.assertEqual(context.facts["profile"], selected) - missing = doctor.check_openvmm_preflight(doctor_context(self.root / "none")) - self.assertFalse(missing.passed) - self.assertIn("was not found", missing.detail) - # Without H2, a catalog profile of a generation that only OpenVMM pins - # passes. - with patch.object( - doctor.subprocess, - "run", - return_value=completed( - line.replace("intel.icelake-sp.v1", "intel.raptorlake.v1") - ), - ): - result = doctor.check_openvmm_preflight(context_with_files()) - self.assertTrue(result.passed, result.detail) - - def test_rate_check_applies_the_spec_bounds_to_sleep_separated_samples(self): - def records( - rates: list[float], - *, - disciplined: list[float] | None = None, - interval_s: float = 10.0, - bracket: int = 40, - clock_hz: int = 1_000_000_000, - resolution_ns: float = 1.0, - names: tuple[str, str] = ("CLOCK_MONOTONIC", "CLOCK_MONOTONIC_RAW"), - ) -> list[tuple[str, dict[str, str]]]: - """Probe records whose consecutive samples measure ``rates`` against - the stability clock and ``disciplined`` (default ``rates``) against - the rate clock.""" - result: list[tuple[str, dict[str, str]]] = [] - for role, name in zip(("rate", "stability"), names, strict=True): - clock = {"role": role, "name": name, "hz": str(clock_hz)} - clock["resolution_ns"] = f"{resolution_ns:.3f}" - result.append(("clock", clock)) - series = {"rate": disciplined or rates, "stability": rates} - state = {role: (10**12, 5 * clock_hz) for role in series} - for index in range(len(rates) + 1): - sample = {"index": str(index)} - for role, role_rates in series.items(): - tsc, ticks = state[role] - if index: - tsc += round(role_rates[index - 1] * interval_s) - ticks += round(interval_s * clock_hz) - state[role] = (tsc, ticks) - sample[f"{role}_tsc"] = str(tsc) - sample[f"{role}_clock"] = str(ticks) - sample[f"{role}_bracket"] = str(bracket) - result.append(("sample", sample)) - return result - - def check( - context: doctor.DoctorContext, - probe_records: list[tuple[str, dict[str, str]]], - clocksource: str = "tsc", - windows: bool = False, - ) -> doctor.CheckResult: - with ( - patch.object(doctor, "run_probe", return_value=probe_records) as run, - patch.object(doctor, "host_clocksource", return_value=clocksource), - patch.object(doctor, "host_is_windows", return_value=windows), - ): - result = doctor.check_rate(context) - self.assertEqual( - run.call_args.args[1:], - ( - "rate", - "--samples", - str(context.schedule.rate_samples), - "--interval-ms", - str(context.schedule.rate_interval_ms), - ), - ) - # The probe sleeps through its sampling window, which the timeout - # allows on top of --timeout (120 s on the qualification schedule). - self.assertEqual( - run.call_args.kwargs, - { - "duration": (context.schedule.rate_samples - 1) - * context.schedule.rate_interval_ms - / 1000 - }, - ) - return result - - def ci_context(backend: str = "kvm") -> doctor.DoctorContext: - context = doctor_context(self.root, backend) - context.schedule = doctor.CI_SCHEDULE - return context - - steady = [2793437000.0 + jitter for jitter in (0, 200, -150, 100) * 3] - context = doctor_context(self.root) - context.facts["tsc_hz"] = "2793437000" - result = check(context, records(steady)) - self.assertTrue(result.passed, result.detail) - self.assertIn( - "against CLOCK_MONOTONIC from 13 samples 10 s apart; against " - "CLOCK_MONOTONIC_RAW (resolution 1 ns) the interval rates agree within " - "0.125 ppm, largest interval uncertainty 0.004 ppm", - result.detail, - ) - self.assertIn("0.0 ppm from the backend's 2793437000 Hz", result.detail) - self.assertIn("host clocksource tsc (evidence)", result.detail) - self.assertEqual(context.facts["rate_agreement_ppm"], "0.125") - self.assertEqual(context.facts["rate_deviation_ppm"], "0.0") - # chrony steers CLOCK_MONOTONIC's rate by ppm between seconds, which - # says nothing about the TSC: the undisciplined clock judges stability, - # and the disciplined one the rate. - steered = [2300000100.0 + jitter for jitter in (0, 6900, -4600)] * 4 - context = doctor_context(self.root, "mshv") - context.facts["native_tsc_hz"] = "2300000000" - result = check(context, records([2300000100.0] * 12, disciplined=steered)) - self.assertTrue(result.passed, result.detail) - self.assertIn("agree within 0.000 ppm", result.detail) - self.assertEqual(context.facts["measured_tsc_hz"], "2300000867") - self.assertIn("0.4 ppm from the backend's 2300000000 Hz", result.detail) - # CI's short schedule; Windows reads QueryPerformanceCounter, which time - # synchronization never steers, in both roles. Its 100 ns tick keeps a - # 1 s interval conclusive at 0.1 ppm. - qpc = records( - [2300000100.0, 2300000200.0], - interval_s=1.0, - bracket=0, - clock_hz=10_000_000, - resolution_ns=100.0, - names=("QueryPerformanceCounter", "QueryPerformanceCounter"), - ) - windows = check(ci_context("whp"), qpc, windows=True) - self.assertTrue(windows.passed, windows.detail) - self.assertIn( - "against QueryPerformanceCounter from 3 samples 1 s apart", windows.detail - ) - self.assertIn("uncertainty 0.100 ppm", windows.detail) - self.assertNotIn("clocksource", windows.detail) - # Hyper-V's reference TSC page advances CLOCK_MONOTONIC_RAW in 100 ns - # steps: the resolution, not the empty brackets, bounds the measurement. - tsc_page = records( - [2300000100.0, 2300000200.0], interval_s=1.0, bracket=0, resolution_ns=100.0 - ) - result = check(ci_context("mshv"), tsc_page, "hyperv_clocksource_tsc_page") - self.assertTrue(result.passed, result.detail) - self.assertIn("CLOCK_MONOTONIC_RAW (resolution 100 ns)", result.detail) - self.assertIn("uncertainty 0.100 ppm", result.detail) - coarse = records( - [2300000100.0, 2300000200.0], interval_s=1.0, bracket=0, resolution_ns=300.0 - ) - result = check(ci_context("mshv"), coarse) - self.assertFalse(result.passed) - self.assertIn( - "inconclusive: an interval's uncertainty is 0.300 ppm", result.detail - ) - for probe_records, message in ( - ( - records([2793437000.0, 2793440000.0] * 6), - "unstable: interval rates differ by 1.074 ppm > 1.0 ppm", - ), - ( - records([2793437000.0] * 2, interval_s=1.0, bracket=400), - "inconclusive: an interval's uncertainty is 0.401 ppm > 0.25 ppm", - ), - (records([2794000000.0] * 12), "201.5 ppm from the measured rate"), - (records([100_000_000.0] * 12), "[E_TSC_RATE_IMPLAUSIBLE]"), - ): - with self.subTest(message=message): - context = doctor_context(self.root) - if "inconclusive" in message: - context = ci_context() - context.facts["tsc_hz"] = "2793437000" - result = check(context, probe_records) - self.assertFalse(result.passed) - self.assertIn(message, result.detail) - with self.assertRaisesRegex(doctor.ScriptError, "printed 3 samples, not 13"): - check(doctor_context(self.root), records([2793437000.0] * 2)) - with self.assertRaisesRegex(doctor.ScriptError, "no stability clock"): - check(ci_context(), records([2793437000.0] * 2, interval_s=1.0)[:1]) - # The clocksource is evidence on every backend. - for backend, clocksource in ( - ("kvm", "hpet"), - ("mshv", "hyperv_clocksource_tsc_page"), - ): - with self.subTest(backend=backend, clocksource=clocksource): - context = doctor_context(self.root, backend) - context.facts["native_tsc_hz"] = "2300000000" - result = check(context, records([2300000100.0] * 12), clocksource) - self.assertTrue(result.passed, result.detail) - self.assertIn( - f"host clocksource {clocksource} (evidence)", result.detail - ) - self.assertEqual(context.facts["host_clocksource"], clocksource) - # Without H3 (validate-runner), H4 still checks stability. - standalone = check(ci_context(), records([2300000100.0] * 2, interval_s=1.0)) - self.assertTrue(standalone.passed, standalone.detail) - self.assertIn("not compared with the backend's rate", standalone.detail) - - def test_host_skew_check_applies_the_bound(self): - summary = { - "pairs": "28", - "cpus": "0-7", - "max_abs_offset_ns": "40", - "max_uncertainty_ns": "120", - "stalled_pairs": "0", - "conclusive": "1", - } - context = doctor_context(self.root) - context.facts["measured_tsc_hz"] = "2793437000" - with patch.object(doctor, "run_probe", return_value=[("skew", summary)]) as run: - self.assertTrue(doctor.check_host_skew(context).passed) - self.assertIn("--tsc-hz", run.call_args.args) - for changes, message in ( - ({"max_abs_offset_ns": "1500"}, "exceeds 1000"), - ({"stalled_pairs": "2", "conclusive": "0"}, "2 CPU pair(s) stalled"), - ({"conclusive": "0"}, "inconclusive"), - ): - with ( - self.subTest(message=message), - patch.object( - doctor, "run_probe", return_value=[("skew", summary | changes)] - ), - ): - result = doctor.check_host_skew(doctor_context(self.root)) - self.assertFalse(result.passed) - self.assertIn(message, result.detail) - - def test_guest_warp_check_runs_the_idle_schedules_and_applies_the_bound(self): - context = doctor_context(self.root) - for path in (context.openvmm, context.kernel, context.initrd): - path.write_bytes(b"") - - def boot(cpus: int) -> str: - line = BOOT_LINE.replace("cpus=4", f"cpus={cpus}") - return " ALPINE-MICROVM-BOOT-OK: 3.22.1\n" + line + STATUS_OK - - smp = boot(8) + warp_output(pairs=28) * 4 + warp_output(pairs=28, offset=60) - # A newer guest also reports the boot check's CPU time. - single = boot(1).replace("elapsed_us=1873", "elapsed_us=1873 cpu_us=1100") - single += warp_output(pairs=0, offset=0) - with ( - patch.object(doctor.os, "cpu_count", return_value=8), - patch.object( - doctor, - "run_guest_script", - side_effect=[{"text": smp}, {"text": single}], - ) as run, - ): - result = doctor.check_guest_warp(context) - self.assertTrue(result.passed, result.detail) - self.assertIn( - "vcpus=8 rounds=5 idle_gaps_s=0.1,1,5,1 boot_elapsed_us=1873;", - result.detail, - ) - self.assertIn( - "vcpus=1 rounds=1 idle_gaps_s=none boot_elapsed_us=1873 boot_cpu_us=1100", - result.detail, - ) - self.assertEqual(context.facts["guest_warp_ns"], "60") - (smp_call, single_call) = run.call_args_list - for call, processors, gaps in ( - (smp_call, "8", time_abi.QUALIFICATION_WARP_GAPS), - (single_call, "1", ()), - ): - command, script, marker = call.args - self.assertEqual(command[command.index("--processors") + 1], processors) - self.assertEqual(script, time_abi.warp_probe_script(gaps) + "nvx-exit 0\n") - self.assertEqual(marker, time_abi.WARP_PROBE_COMPLETION_MARKER) - self.assertIs(call.kwargs["time_abi_status"], True) - # CI's schedule runs the probe twice with a 1 s gap. - context.schedule = doctor.CI_SCHEDULE - ci = boot(8) + warp_output(pairs=28) * 2 - with ( - patch.object(doctor.os, "cpu_count", return_value=8), - patch.object( - doctor, - "run_guest_script", - side_effect=[{"text": ci}, {"text": single}], - ), - ): - result = doctor.check_guest_warp(context) - self.assertTrue(result.passed, result.detail) - self.assertIn("vcpus=8 rounds=2 idle_gaps_s=1 ", result.detail) - context.schedule = doctor.QUALIFICATION_SCHEDULE - cases = ( - ( - [boot(8) + warp_output(pairs=28) * 2], - "vcpus=8", - "ran 2 rounds instead of 5", - ), - ( - [ - boot(8) - + warp_output(pairs=28) * 4 - + warp_output(pairs=28, offset=1500) - ], - "vcpus=8", - "in round 5: max_abs_offset_ns=1500", - ), - ( - [STATUS_OK + warp_output(pairs=28) * 5], - "vcpus=8", - "NVX-TIME-ABI boot marker", - ), - ( - [BOOT_LINE.replace("cpus=4", "cpus=8") + warp_output(pairs=28) * 5], - "vcpus=8", - "nvx-time status did not finish before the warp probe", - ), - ( - [smp, boot(1) + warp_output(pairs=0, backward=2000, verdict="FAIL")], - "vcpus=1", - "max_backward_ns=2000", - ), - ) - for outputs, guest, message in cases: - with ( - self.subTest(message=message), - patch.object(doctor.os, "cpu_count", return_value=8), - patch.object( - doctor, - "run_guest_script", - side_effect=[{"text": output} for output in outputs], - ), - ): - result = doctor.check_guest_warp(context) - self.assertFalse(result.passed) - self.assertTrue(result.detail.startswith(f"{guest}: "), result.detail) - self.assertIn(message, result.detail) - # A 1-vCPU host runs only the 1-vCPU guest. - with ( - patch.object(doctor.os, "cpu_count", return_value=1), - patch.object( - doctor, "run_guest_script", return_value={"text": single} - ) as run, - ): - result = doctor.check_guest_warp(context) - self.assertTrue(result.passed, result.detail) - self.assertEqual(run.call_count, 1) - self.assertTrue(result.detail.startswith("vcpus=1 rounds=1 idle_gaps_s=none")) - missing = doctor.check_guest_warp(doctor_context(self.root / "missing")) - self.assertIn("OpenVMM was not found", missing.detail) - - def test_utc_check_reads_the_host_clock_discipline(self): - context = doctor_context(self.root) - with patch.object(doctor, "host_is_windows", return_value=False): - with patch.object(doctor, "_linux_clock_state", return_value=(0, 0x2001)): - self.assertTrue(doctor.check_utc(context).passed) - with patch.object(doctor, "_linux_clock_state", return_value=(5, 0x0041)): - self.assertIn("STA_UNSYNC", doctor.check_utc(context).detail) - status = ( - "Leap Indicator: 0(no warning)\nStratum: 2\n" - "Source: VM IC Time Synchronization Provider\n" - ) - with ( - patch.object(doctor, "host_is_windows", return_value=True), - patch.object(doctor.subprocess, "run", return_value=completed(status)), - ): - result = doctor.check_utc(context) - self.assertTrue(result.passed, result.detail) - self.assertIn("VM IC Time Synchronization Provider", result.detail) - for unsynchronized in ( - status.replace("0(no warning)", "3(not synchronized)"), - "Leap Indicator: 0\nSource: Local CMOS Clock\n", - ): - with ( - patch.object(doctor, "host_is_windows", return_value=True), - patch.object( - doctor.subprocess, "run", return_value=completed(unsynchronized) - ), - ): - self.assertFalse(doctor.check_utc(context).passed) - # Output H7 can't read, such as a localized w32tm's, fails it with a - # stable leading phrase instead of passing as an unknown source. - unverified = "cannot verify host UTC synchronization: " - for outcome in ( - completed( - "Sprungindikator: 0(keine Warnung)\nQuelle: time.windows.com,0x9\n" - ), - completed("Leap Indicator: 0(no warning)\nStratum: 2\n"), - completed("Stratum: 2\nSource: VM IC Time Synchronization Provider\n"), - completed(status.replace("0(no warning)", "unknown")), - completed("The service has not been started. (0x80070426)\n", 1), - subprocess.TimeoutExpired(["w32tm"], 30), - ): - with ( - patch.object(doctor, "host_is_windows", return_value=True), - patch.object(doctor.subprocess, "run", side_effect=[outcome]), - contextlib.redirect_stdout(io.StringIO()), - ): - (result,) = doctor.run_checks(context, ("H7",)) - self.assertFalse(result.passed, result.detail) - self.assertTrue(result.detail.startswith(unverified), result.detail) - # So does an adjtimex clock state Linux doesn't define, or no adjtimex. - with ( - patch.object(doctor, "host_is_windows", return_value=False), - contextlib.redirect_stdout(io.StringIO()), - ): - with patch.object(doctor, "_linux_clock_state", return_value=(6, 0)): - (unknown_state,) = doctor.run_checks(context, ("H7",)) - with patch.object(doctor.ctypes, "CDLL", side_effect=OSError("no libc")): - (no_adjtimex,) = doctor.run_checks(context, ("H7",)) - for result in (unknown_state, no_adjtimex): - self.assertFalse(result.passed, result.detail) - self.assertTrue(result.detail.startswith(unverified), result.detail) - - def test_run_prints_lines_writes_the_summary_and_fails_closed(self): - def passing(check: str): - def run_check(context: doctor.DoctorContext) -> doctor.CheckResult: - context.facts["generation"] = "emeraldrapids" - return doctor.CheckResult(check, True, f"{check} ok") - - return run_check - - def broken(context: doctor.DoctorContext) -> doctor.CheckResult: - del context - raise doctor.ScriptError("rustc is required to build the host time probe") - - summary = self.root / "summary.md" - arguments = doctor_parser().parse_args( - [ - "--backend", - "mshv", - "--checks", - "H5", - "H2", - "--summary", - str(summary), - "--probe-dir", - str(self.root), - ] - ) - stdout = io.StringIO() - with ( - patch.dict(doctor.CHECKS, {"H2": passing("H2"), "H5": passing("H5")}), - contextlib.redirect_stdout(stdout), - ): - self.assertEqual(doctor.run(arguments), 0) - lines = stdout.getvalue().splitlines() - self.assertEqual( - [line.split()[1] for line in lines if line.startswith("NVX-DOCTOR")], - ["check=H2", "check=H5"], - ) - self.assertIn("passed; generation emeraldrapids", lines[-1]) - self.assertIn("(mshv): passed", summary.read_text(encoding="utf-8")) - with ( - patch.dict(doctor.CHECKS, {"H2": passing("H2"), "H5": broken}), - contextlib.redirect_stdout(io.StringIO()), - ): - self.assertEqual(doctor.run(arguments), 1) - text = summary.read_text(encoding="utf-8") - self.assertIn("**fail** | rustc is required", text) - self.assertIn("Generation: `emeraldrapids`", text) - - # Unreadable values that break a computation, such as a zero rate, - # fail only their own check, and the later checks still run. - def arithmetic(context: doctor.DoctorContext) -> doctor.CheckResult: - del context - raise ZeroDivisionError("float division by zero") - - stdout = io.StringIO() - with ( - patch.dict(doctor.CHECKS, {"H2": arithmetic, "H5": passing("H5")}), - contextlib.redirect_stdout(stdout), - ): - self.assertEqual(doctor.run(arguments), 1) - self.assertIn( - 'check=H2 status=fail detail="float division by zero"', stdout.getvalue() - ) - self.assertIn("check=H5 status=pass", stdout.getvalue()) - - def test_run_selects_the_cpu_fingerprint_or_runs_without_openvmm(self): - seen: list[Path | None] = [] - schedules: list[doctor.Schedule] = [] - - def cpu(context: doctor.DoctorContext) -> doctor.CheckResult: - seen.append(context.fingerprint) - schedules.append(context.schedule) - context.facts["surface_digest"] = "sha256:62" - return doctor.CheckResult("H2", True, "ok") - - summary = self.root / "summary.md" - common = ["--backend", "kvm", "--checks", "H2", "--summary", str(summary)] - common += ["--probe-dir", str(self.root)] - with ( - patch.dict(doctor.CHECKS, {"H2": cpu}), - contextlib.redirect_stdout(io.StringIO()), - ): - for extra in ( - [], - ["--cpu-fingerprint", str(self.root / "fingerprint.json")], - ["--no-openvmm", "--ci-schedule"], - ): - arguments = doctor_parser().parse_args([*common, *extra]) - self.assertEqual(doctor.run(arguments), 0) - self.assertEqual( - seen, - [ - self.root / "nvx-cpu-fingerprint-kvm.json", - self.root / "fingerprint.json", - None, - ], - ) - self.assertEqual( - schedules, - [doctor.QUALIFICATION_SCHEDULE] * 2 + [doctor.CI_SCHEDULE], - ) - self.assertEqual(doctor.QUALIFICATION_SCHEDULE.rate_samples, 13) - self.assertEqual(doctor.QUALIFICATION_SCHEDULE.rate_interval_ms, 10_000) - self.assertEqual(doctor.CI_SCHEDULE.rate_samples, 3) - self.assertEqual(doctor.CI_SCHEDULE.rate_interval_ms, 1_000) - self.assertIn("CPU surface digest: `sha256:62`", summary.read_text("utf-8")) - arguments = doctor_parser().parse_args(["--backend", "kvm", "--no-openvmm"]) - with self.assertRaisesRegex(doctor.ScriptError, "cannot run H3 and H6"): - doctor.run(arguments) - with contextlib.redirect_stderr(io.StringIO()), self.assertRaises(SystemExit): - doctor_parser().parse_args( - ["--backend", "kvm", "--no-openvmm", "--cpu-fingerprint", "x"] - ) - - def test_builds_the_host_probe_once_per_source_version(self): - target = doctor.probe_path(self.root) - self.assertRegex(target.name, r"^nvx-host-time-probe-[0-9a-f]{16}(\.exe)?$") - - def rustc(command: list[str], **_kwargs: object): - Path(command[command.index("-o") + 1]).write_bytes(b"probe") - return completed("") - - with patch.object( - doctor, "require_tool", side_effect=doctor.ScriptError("rustc is required") - ) as require_tool: - with self.assertRaisesRegex(doctor.ScriptError, "rustc is required"): - doctor.build_probe(self.root) - require_tool.assert_called_once_with( - "rustc", "rustc is required to build the host time probe" - ) - with ( - patch.object(doctor, "require_tool", return_value="rustc"), - patch.object(doctor.subprocess, "run", side_effect=rustc) as run, - ): - self.assertEqual(doctor.build_probe(self.root), target) - self.assertEqual(doctor.build_probe(self.root), target) - run.assert_called_once() - self.assertEqual(target.read_bytes(), b"probe") - target.unlink() - with ( - patch.object(doctor, "require_tool", return_value="rustc"), - patch.object( - doctor.subprocess, "run", return_value=completed("", 1, "error[E0425]") - ), - ): - with self.assertRaisesRegex(doctor.ScriptError, "E0425"): - doctor.build_probe(self.root) - - def test_parses_probe_records(self): - context = doctor_context(self.root) - context.probe = self.root / "probe.exe" - output = ( - "noise\nNVX-HOST-TIME-PROBE rate index=1 tsc_hz=1.5\n" - "NVX-HOST-TIME-PROBE skew pairs=1 cpus=0-1\n" - ) - with patch.object( - doctor.subprocess, "run", return_value=completed(output) - ) as run: - records = doctor.run_probe(context, "skew") - self.assertEqual(run.call_args.kwargs["timeout"], context.timeout) - self.assertEqual( - records, - [ - ("rate", {"index": "1", "tsc_hz": "1.5"}), - ("skew", {"pairs": "1", "cpus": "0-1"}), - ], - ) - with patch.object( - doctor.subprocess, "run", return_value=completed(output) - ) as run: - doctor.run_probe(context, "rate", duration=120.0) - self.assertEqual(run.call_args.kwargs["timeout"], context.timeout + 120.0) - with patch.object( - doctor.subprocess, "run", return_value=completed("", 3, "no CPU") - ): - with self.assertRaisesRegex(doctor.ScriptError, "skew failed: no CPU"): - doctor.run_probe(context, "skew") - - -def doctor_parser(): - import argparse - - parser = argparse.ArgumentParser() - doctor.configure_parser(parser) - return parser - - -if __name__ == "__main__": - unittest.main() From 31b5434df399a9f5f6ee13cc783e76d8ef37829e Mon Sep 17 00:00:00 2001 From: Akhil Gupta Date: Mon, 12 Oct 2026 05:01:51 -0700 Subject: [PATCH 5/5] Rename nvx507-test-time-abi.py to scripts/test_time_abi.py --- nvx507-test-time-abi.py => scripts/test_time_abi.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename nvx507-test-time-abi.py => scripts/test_time_abi.py (100%) diff --git a/nvx507-test-time-abi.py b/scripts/test_time_abi.py similarity index 100% rename from nvx507-test-time-abi.py rename to scripts/test_time_abi.py