test-microvm: pause and resume a microVM through OpenVMM state control

.github/actions/validate-nvx/action.yml

@@ -10,7 +10,7 @@ runs:
       run: |
         set -euo pipefail
         python3 -m compileall -q scripts
-        python3 -m unittest scripts/test_performance.py scripts/test_nvx_tools.py scripts/test_microvm_tests.py scripts/test_development_release.py scripts/test_egress_policy.py scripts/test_control_session.py scripts/test_managed_agent.py scripts/test_guest_time.py scripts/test_time_abi.py -v
+        python3 -m unittest scripts/test_performance.py scripts/test_nvx_tools.py scripts/test_microvm_tests.py scripts/test_development_release.py scripts/test_egress_policy.py scripts/test_control_session.py scripts/test_state_control.py scripts/test_managed_agent.py scripts/test_guest_time.py scripts/test_time_abi.py -v
         python3 scripts/test_adversarial.py -v
         python3 .github/skills/nvx-host-connect/scripts/test_hosts.py -v
         python3 scripts/nvx.py --help
@@ -29,7 +29,7 @@ runs:
         $ErrorActionPreference = "Stop"
         python -m compileall -q scripts
         if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
-        python -m unittest scripts/test_performance.py scripts/test_nvx_tools.py scripts/test_microvm_tests.py scripts/test_development_release.py scripts/test_egress_policy.py scripts/test_control_session.py scripts/test_managed_agent.py scripts/test_guest_time.py scripts/test_time_abi.py -v
+        python -m unittest scripts/test_performance.py scripts/test_nvx_tools.py scripts/test_microvm_tests.py scripts/test_development_release.py scripts/test_egress_policy.py scripts/test_control_session.py scripts/test_state_control.py scripts/test_managed_agent.py scripts/test_guest_time.py scripts/test_time_abi.py -v
         if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
         python scripts\test_adversarial.py -v
         if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }

doc/ci.md

@@ -40,8 +40,8 @@ pending checks. The test runners ask after every cold boot whose shell is on
 a console the harness reads, the OpenVMM console or a virtio console over
 TCP, before any other input, and wait for the query to exit, so the console's
 echo of later input cannot split its lines. Cold boots with no shell there are
-only scanned: one-shot workloads, and the managed lifecycle, where init starts
-the managed agent instead of a shell. A failing boot check still powers them
+only scanned: one-shot workloads, and the managed lifecycle and `pause-resume`
+scenarios, where init starts the managed agent instead of a shell. A failing boot check still powers them
 off with status 193, which fails the run. The query must
 exit 0 with a passing `NVX-TIME-ABI` boot line, which reports ABI version 1,
 generation 0, a plausible TSC rate, and the backend's LAPIC rate; a missing
@@ -99,6 +99,30 @@ harness stages its check, report `/sys/kernel/rcu_stall_count` as 0 two
 seconds later, and show an uptime of at least 30 s, which proves that
 monotonic time advanced by the downtime. The captures share one downtime
 window, so the scenario adds about a minute per backend.
+The `pause-resume` scenario covers [host pause](design/time-abi.md#host-pause).
+It boots the largest requested vCPU count with the managed lifecycle and
+OpenVMM's state-control endpoint (`--microvm-state-control`), which
+authenticates hosts with the control console's capability. A host that does
+not authenticate, or presents the wrong capability, must get no response.
+While a managed workload spins on one vCPU, the harness pauses the VM for
+30 s, longer than the guest's 21 s RCU stall timeout and the debug kernel's
+20 s soft-lockup threshold, and then twice for 2 s. It repeats every pause and
+resume request, which must change nothing the second time, queries the run
+state while paused, and requires the transition count to rise by one on every
+change. On Linux hosts, the OpenVMM process may use at most a tenth of a
+pause, plus 0.2 s, in CPU time while paused. Afterwards the workload must
+finish, having started at least 0.5 s of guest uptime before the long pause
+and stopped after it, guest uptime must have advanced by the time the VM ran to
+within 1 s, and `/sys/kernel/rcu_stall_count` must read 0. Host UTC kept
+running, so the guest's wall clock must return to within 1.5 s of host UTC
+within 150 s, through at least one wall-clock discipline step that
+`/run/nvx/time/state` records, with no new violation there. A second host,
+connected after the first one disconnects, must see the same broker instance
+and transition count, and pauses the VM again; a third host must then find it
+still paused. Finally the harness terminates OpenVMM, which must still be
+running and must exit with the termination status. The scenario prints one
+`NVX-PAUSE-RESUME:` line with its measurements and adds about a minute per
+backend.
 The `time-abi-conformance` scenario boots the largest requested vCPU count and
 runs the guest's exhaustive CI check, `/sbin/nvx-time exhaustive`, which the
 boot check leaves to CI. On every online CPU it checks every leaf
@@ -144,7 +168,8 @@ on the CI debug kernel (`build/vmlinux-debug`, built from
 `kernel/config-microvm-debug`), whose soft-lockup and hung-task detectors
 production kernels leave out. It selects the same-host restore scenarios
 `smp`, `smp-snapshot`, `restore-processors`, `restore-downtime`, and
-`snapshot-tiers`. The jobs skip every other test step, because those boot the
+`snapshot-tiers`, and the host pause scenario `pause-resume`. The jobs skip
+every other test step, because those boot the
 production kernel, which the `nvx-microvm-tests-{kvm,mshv,whp}` jobs already
 cover: the public managed execution configuration, the Ubuntu and Azure Linux
 guest tests, the Ubuntu sandbox layer and live-share smoke tests, the
@@ -159,7 +184,7 @@ whose guest OpenVMM stops at the restore gate on purpose. The harness refuses
 a kernel whose `vmlinux-debug.config` lacks the detectors, because the guest's
 `C11` check passes vacuously without them. To bound the cost, pull requests
 run the debug kernel on KVM only and `dev` pushes run it on every backend;
-each job takes about five minutes on its own runner, in parallel with the
+each job takes about six minutes on its own runner, in parallel with the
 other microVM jobs. The jobs gate the required status check, the development
 release, and performance persistence. The GitHub-hosted `debug-kernel` job
 builds the debug kernel beside the shared `artifacts` job

doc/design/concurrency-and-trust-boundaries.md

@@ -23,7 +23,11 @@ the OpenVMM user, requires a launcher-provided capability before any host data
 reaches the guest, and bounds every record, queue, and credit window in both
 directions; malformed, stale, or out-of-order records fail closed. The
 capability never appears in arguments, environment variables, logs, snapshots,
-or attachment identities.
+or attachment identities. The state-control endpoint, which pauses and resumes
+the VM without reaching the guest, admits the same local peer and requires the
+same capability before it answers. It serves one host and one request at a
+time; a failed or stalled authentication, a malformed request, or 60 s without
+a request closes the connection without a response.
 
 Host-facing outputs are bounded as well. The optional outcome report is written
 after teardown to a previously absent local path and records only a schema

doc/design/host-attachment-model.md

@@ -14,6 +14,7 @@ Each external resource has a stable ID and a declarative reconstruction policy.
 | filesystem | FUSE namespace, handles, cookies, root/object identity, access mode, and denied paths, or explicit dormant state | Fresh host-directory attachment with the same canonical path, target, mode, and denied paths; a dormant slot may stay unattached or bind a new attachment |
 | sandbox block | Queue/device state, fixed roles, access, geometry, read-only layer identities, and scratch policy | Matching read-only layers plus a verified private paired scratch copy, or a new same-geometry scratch file |
 | Readiness endpoint | Nothing | Optional single-use Unix socket or named pipe supplied for one restore |
+| State-control endpoint | Nothing | Optional same-user Unix socket or Windows named pipe, supplied with a live control console and authenticated with its fresh capability |
 
 Attachment resolution happens before vCPU start. A supplied attachment must
 reproduce the saved identity except that an eligible listener may replace only

doc/design/machine-and-device-abi.md

@@ -299,6 +299,16 @@ The workload protocol carried inside the brokered data stream belongs to the
 guest agent; see
 [Control protocol and checkpoint handoff](sandbox-filesystem-and-agent-architecture.md#control-protocol-and-checkpoint-handoff).
 
+A machine with a live, authenticated control console may also bind a
+state-control endpoint (`--microvm-state-control listen=<endpoint>`), through
+which the host pauses, resumes, and queries the VM without a snapshot;
+[Host pause](time-abi.md#host-pause) defines guest time across a pause. The
+endpoint follows the private-endpoint and peer-identity rules above, must
+differ from both console endpoints, and answers a host only after it presents
+the control console's capability. OpenVMM serves it outside the VM's devices,
+so it keeps answering while the VM is paused. Snapshots never record it; a
+restore may supply a fresh one.
+
 ### Network
 
 The optional NIC has one RX/TX queue pair and an exact feature mask:

doc/design/time-abi.md

@@ -37,10 +37,12 @@ Goals:
   rejected with a stable error code.
 - Restore within one backend across hosts of the same CPU generation and CPU
   profile, including restores after a host reboot.
-- Guest monotonic time advances by the snapshot downtime. Wall clock is set
+- Guest monotonic time advances by the snapshot downtime and stands still
+  while the host pauses the VM ([Host pause](#host-pause)). Wall clock is set
   on every restore and disciplined afterwards.
 - RCU-stall, soft-lockup, hung-task, clocksource, and TSC-warp detectors never
-  trip across capture and restore. A detected violation fails fast.
+  trip across capture, restore, and host pause. A detected violation fails
+  fast.
 - No performance regression beyond the gate in
   [Performance expectations and acceptance gate](#performance-expectations-and-acceptance-gate).
 
@@ -878,6 +880,40 @@ round trip on Azure takes about 12 µs, so nested MSHV pairs at a median of
 about 1.5× margin over the median and fail restores at random. Downtime needs
 only millisecond accuracy, so a 100 µs pairing error is negligible.
 
+## Host pause
+
+The host can pause a running microVM and later resume it in the same VM
+process, without a snapshot, through OpenVMM's state-control endpoint
+(`--microvm-state-control`; see the [appendix](#appendix-openvmm-guide)).
+Where a restore advances guest time by the downtime, a host pause holds it:
+
+- **Monotonic time.** Once the vCPUs stop, OpenVMM takes VP 0's guest TSC and
+  every vCPU's LAPIC state. Resume writes that TSC to every vCPU with the
+  [synchronized TSC set](#backend-obligations), writes the LAPIC states back
+  before any vCPU runs, and then starts the VM. The TSC, the LAPIC timers, and
+  the VM time behind the emulated timers continue from where they stopped, so
+  guest `CLOCK_MONOTONIC`, `CLOCK_BOOTTIME`, and jiffies count only the time
+  the VM ran, and no RCU-stall, soft-lockup, or hung-task detector sees a
+  pause of any length. The generation counter does not change, and no restore
+  repair runs.
+- **Rejection.** As at capture, a pause is rejected when a vCPU has an armed
+  periodic LAPIC timer, which a guest under this ABI arms at most before its
+  first tick, or a TSC-deadline timer, which it never arms (see
+  [Clocksource, tick, and PIT](#clocksource-tick-and-pit)). The vCPUs then
+  start again, and the guest sees only a brief stall.
+- **Wall clock.** Host UTC keeps running, and so do the CMOS RTC's date and
+  time and the [time sample](#time-sample). After a resume, guest
+  `CLOCK_REALTIME` is behind host UTC by the paused time until the
+  [wall-clock discipline](#wall-clock-discipline)'s next poll, at most 64 s
+  of guest time later, steps it and counts a discontinuity
+  (`last_discontinuity=step`); a lag under 128 ms is slewed instead. The
+  discipline follows it as it follows any host wall-clock step, without a
+  violation.
+- **Snapshots.** A guest snapshot request still pending when a pause stops
+  the vCPUs is cancelled, as when the VM stops for any other reason. A pause
+  at a snapshot boundary or during the post-restore gate is refused as busy
+  and changes nothing.
+
 ## Restore algorithm
 
 Restore runs these steps in order. Any failure rejects the restore before a
@@ -2180,8 +2216,8 @@ harness reads, and after every restore in the `smp-snapshot`,
 `restore-processors`, `restore-downtime`, and `snapshot-tiers` scenarios,
 except `snapshot-tiers`' gate-timeout check, whose guest OpenVMM stops at the
 restore gate on purpose. CI's other restores and its guests without a shell,
-the managed lifecycle and one-shot workloads, are only scanned for violation
-events; a failed check still powers the guest off with status 193. Each query
+the managed lifecycle, host pause, and one-shot workloads, are only scanned
+for violation events; a failed check still powers the guest off with status 193. Each query
 requires exit status 0 and, after a restore,
 a `phase=restore` line with `status=ok`, the restored `generation`, and
 `cpus` equal to the online CPUs; after a cold boot, a `phase=boot` line with
@@ -2234,6 +2270,13 @@ watcher active, report no RCU, soft-lockup, hung-task, clocksource, or warp
 message. The CI debug-kernel variant runs the same-host cases with the
 soft-lockup and hung-task detectors enabled.
 
+**Host pause.** CI's [`pause-resume`](../ci.md) scenario pauses a managed VM
+for 30 s, longer than the 21 s RCU stall timeout, while a workload spins on
+one vCPU, and then twice for 2 s, on every backend and on the debug kernel.
+It checks the rules of [Host pause](#host-pause): guest uptime counts only the
+time the VM ran, to within 1 s; `rcu_stall_count` stays 0, with no violation;
+and the guest's wall clock returns to host UTC through a discipline step.
+
 **Performance.** The gate above, computed by the performance agent from the
 CI benchmark matrix.
 
@@ -2356,3 +2399,8 @@ The OpenVMM Guide documents the user-facing contract in
 
 The CLI reference documents `--cpu-profile` and `--x-time-abi-verify`. The
 test hooks stay undocumented in the Guide; this document describes them.
+
+`Guide/src/reference/openvmm/management/state_control_protocol.md` documents
+[host pause](#host-pause): the state-control endpoint and its authentication,
+the protocol, and guest time across a pause. The CLI reference documents
+`--microvm-state-control`.

doc/usage.md

@@ -263,7 +263,8 @@ the Alpine-prompt-specific `console-snapshot` scenario. `--debug-kernel` boots
 the CI debug kernel, `build/vmlinux-debug`, whose soft-lockup and hung-task
 detectors the guest's time ABI watcher reports; without `--scenario`, it runs
 only the same-host restore scenarios `smp`, `smp-snapshot`,
-`restore-processors`, `restore-downtime`, and `snapshot-tiers`. The command
+`restore-processors`, `restore-downtime`, and `snapshot-tiers`, and the host
+pause scenario `pause-resume`. The command
 requires `build/vmlinux` (with `--debug-kernel`, `build/vmlinux-debug` and its
 `build/vmlinux-debug.config`), the selected initramfs, and
 `openvmm/target/release/openvmm[.exe]`.

scripts/nvx_tools/control_session.py

@@ -10,7 +10,7 @@
 import time
 from dataclasses import dataclass
 from pathlib import Path
-from typing import Any, cast
+from typing import Any, Protocol, cast
 
 from .common import ScriptError, remaining_timeout
 
@@ -242,8 +242,26 @@ def close(self) -> None:
         os.close(self._fd)
 
 
+class EndpointStream(Protocol):
+    """A connected byte stream to one of OpenVMM's ``listen=`` endpoints."""
+
+    def read_exact(self, length: int, deadline: float) -> bytes: ...
+
+    def write_all(self, data: bytes) -> None: ...
+
+    def close(self) -> None: ...
+
+
+def connect_endpoint(endpoint: Path, timeout: float) -> EndpointStream:
+    """Connects to an OpenVMM endpoint: a Unix-domain socket, or on Windows a
+    named pipe."""
+    if os.name == "nt":
+        return _NamedPipeStream.connect(endpoint, timeout)
+    return _SocketStream.connect(endpoint, timeout)
+
+
 class ControlSession:
-    def __init__(self, stream: _SocketStream | _NamedPipeStream) -> None:
+    def __init__(self, stream: EndpointStream) -> None:
         self._stream = stream
         self._instance_id = bytes(16)
         self._epoch = 0
@@ -259,11 +277,7 @@ def connect(
     ) -> ControlSession:
         if len(capability) != 32 or capability == bytes(32):
             raise ValueError("control capability must be 32 nonzero bytes")
-        stream = (
-            _NamedPipeStream.connect(endpoint, timeout)
-            if os.name == "nt"
-            else _SocketStream.connect(endpoint, timeout)
-        )
+        stream = connect_endpoint(endpoint, timeout)
         session = cls(stream)
         try:
             session._write_outer(