diff --git a/.github/workflows/run-nvx-microvm-tests.yml b/.github/workflows/run-nvx-microvm-tests.yml index eaab406a0..8f9b7acfe 100644 --- a/.github/workflows/run-nvx-microvm-tests.yml +++ b/.github/workflows/run-nvx-microvm-tests.yml @@ -109,22 +109,24 @@ jobs: ${{ inputs.debug-kernel && '--debug-kernel' || '' }} --output-dir "build/test-results/microvm-${{ inputs.backend }}${{ inputs.debug-kernel && '-debug' || '' }}" - - name: Run public managed execution configuration on Linux + - name: Run public managed sandbox acceptance on Linux if: runner.os != 'Windows' && !inputs.debug-kernel shell: bash run: >- python3 scripts/nvx.py test-microvm --backend "${{ inputs.backend }}" --scenario managed-exec-config + --scenario sandbox-lifecycle --output-dir "build/test-results/public-exec-${{ inputs.backend }}" - - name: Run public managed execution configuration on Windows + - name: Run public managed sandbox acceptance on Windows if: runner.os == 'Windows' && !inputs.debug-kernel shell: powershell run: >- python scripts\nvx.py test-microvm --backend "${{ inputs.backend }}" --scenario managed-exec-config + --scenario sandbox-lifecycle --output-dir "build/test-results/public-exec-${{ inputs.backend }}" - name: Run Ubuntu guest smoke tests on Linux @@ -217,12 +219,19 @@ jobs: ) scratch="build/ubuntu-smoke-${{ inputs.backend }}.ext4" cp --reflink=auto build/ubuntu-smoke-scratch.ext4 "${scratch}" + # The probe checks the security profile and that the 32 MiB memory + # and 8-process limits hold. python3 scripts/nvx.py sandbox \ --hypervisor "${{ inputs.backend }}" \ --layer "distro,build/ubuntu-distro.erofs,${uuid}" \ --scratch "${scratch}" \ --entrypoint /sbin/nvx-sandbox-smoke \ + --arg limits \ + --arg 33554432 \ + --arg 8 \ --workload-user 65534:65534 \ + --memory-max 33554432 \ + --pids-max 8 \ --memory-mib 256 - name: Run Ubuntu sandbox live-share smoke test on Linux @@ -687,12 +696,19 @@ jobs: ConvertFrom-Json $scratch = "build\ubuntu-smoke-${{ inputs.backend }}.ext4" Copy-Item build\ubuntu-smoke-scratch.ext4 $scratch + # The probe checks the security profile and that the 32 MiB memory + # and 8-process limits hold. python scripts\nvx.py sandbox ` --hypervisor "${{ inputs.backend }}" ` --layer "distro,build\ubuntu-distro.erofs,$($manifest.uuid)" ` --scratch $scratch ` --entrypoint /sbin/nvx-sandbox-smoke ` + --arg limits ` + --arg 33554432 ` + --arg 8 ` --workload-user 65534:65534 ` + --memory-max 33554432 ` + --pids-max 8 ` --memory-mib 256 - name: Run Ubuntu sandbox live-share smoke test on Windows diff --git a/doc/ci.md b/doc/ci.md index 55e11b479..19dc7a675 100644 --- a/doc/ci.md +++ b/doc/ci.md @@ -221,7 +221,26 @@ descriptor modules, converter implementation, and Dockerfile. Artifact upload retains the Alpine filenames and adds the distinct Ubuntu filenames. Each backend also boots the Ubuntu initramfs and runs `/sbin/nvx-sandbox-smoke` from the Ubuntu EROFS layer as UID/GID 65534 over a -fresh ext4 scratch copy. The same entrypoint then verifies a live virtio-fs +fresh ext4 scratch copy, with a 32 MiB `--memory-max` and an 8-process +`--pids-max`. The probe verifies the +[sandbox security profile](run.md#experimental-single-workload-sandbox): the +fixed identity, empty capability sets and `no_new_privs`, private mount, PID, +and UTS namespaces, the private device view and read-only sysfs, the +container cgroup, the overlay root over read-only image layers with its upper +directory on scratch, and that the memory controller kills an allocation of +twice the limit and the pids controller stops the workload at eight +processes. The live-share runs below repeat the profile checks. Each +backend also runs the public `managed-exec-config` and `sandbox-lifecycle` +scenarios of `test-microvm`, which drive `nvx.py sandbox provision`, `start`, +`exec`, `stop`, and `deprovision` as subprocesses (see +[Run](run.md#managed-lifecycle)): request-scoped working directories and +environments, timeouts, nonzero exits, stdout and stderr, bounded output and +outcome reports, warm-guest state across requests and restarts, identity +refusal, fail-closed lifecycle transitions, a crashed OpenVMM's stale runtime +state, and teardown that leaves no OpenVMM process, control endpoint, +capability, or runtime record while the layer and scratch artifacts survive. +Their bounded evidence is part of the backend's failure logs. The same +entrypoint then verifies a live virtio-fs share inside the container: a read-write `/workspace` share with a denied subdirectory must round-trip guest writes to the host, and a read-only `/opt/hostedtoolcache` share must reject writes and symbolic links. In the diff --git a/doc/contribute.md b/doc/contribute.md index eb1d72597..895714c57 100644 --- a/doc/contribute.md +++ b/doc/contribute.md @@ -22,3 +22,7 @@ guest helpers, SMP behavior, or virtio devices. Add those scenarios to `scripts/nvx_tools/microvm_tests.py` and run them with `scripts/nvx.py test-microvm --backend BACKEND`. Keep guest workloads in `scripts/nvx_tools/microvm_test_scripts` and retain complete failure logs. +The scenarios that drive the public `nvx.py sandbox` commands live in +`scripts/nvx_tools/managed_exec_tests.py` and +`scripts/nvx_tools/sandbox_lifecycle_tests.py`, and `microvm_tests.py` +registers them. diff --git a/doc/design/copilot-adversarial-testing.md b/doc/design/copilot-adversarial-testing.md index 5e2040bf9..9fd730cdf 100644 --- a/doc/design/copilot-adversarial-testing.md +++ b/doc/design/copilot-adversarial-testing.md @@ -85,6 +85,8 @@ The initial workload catalog exercises the same managed control and isolation mechanisms through `test-microvm`; it does not yet drive the public persistent `sandbox provision -> start -> exec -> stop` command sequence. That lifecycle remains required before the workload-adversary mode is considered complete. +The deterministic `sandbox-lifecycle` scenario already drives that sequence, +so it is the regression baseline for such a primitive. Model-generated scripts are intentionally unsupported. A future generator must place its output inside the disposable guest or a separate disposable diff --git a/doc/design/validation.md b/doc/design/validation.md index 5b89b29be..40d6c4282 100644 --- a/doc/design/validation.md +++ b/doc/design/validation.md @@ -39,7 +39,15 @@ localhost-to-guest forwarding under allow, and pre-boot rejection when both directions cannot be enforced. Managed lifecycle coverage authenticates the dedicated control channel, runs multiple workloads in one warm VM across reconnects, preserves guest state, reports execution timeout, and stops the VM -cleanly. Sandbox coverage adds deterministic active block-I/O drain, paired +cleanly. Public sandbox coverage drives `nvx.py sandbox` itself: request-scoped +execution options, identity refusal, fail-closed lifecycle transitions, +scratch-backed state across requests and restarts, bounded output and outcome +reports, a crashed OpenVMM's stale runtime state, and teardown that leaves no +OpenVMM process, control endpoint, capability, or runtime record while the +supplied layer and scratch survive. Its guest probe verifies the sandbox's +identity, capability, `no_new_privs`, namespace, device, sysfs, cgroup, +resource-limit, image-layer, and scratch invariants. Sandbox coverage adds +deterministic active block-I/O drain, paired scratch publication, two private restores, fresh scratch replacement, and pre-entry rejection of missing, corrupt, mismatched, or wrong-geometry media. Denied-filesystem coverage verifies listing suppression, allowed writes, diff --git a/doc/run.md b/doc/run.md index 02f304c11..027979b1c 100644 --- a/doc/run.md +++ b/doc/run.md @@ -411,9 +411,36 @@ python3 scripts/nvx.py sandbox \ --memory-mib 256 ``` -CI uses `/sbin/nvx-sandbox-smoke` as the entrypoint to verify Ubuntu identity, -the fixed non-root account, and a scratch-backed `/tmp` write before clean -guest exit. With `--arg TARGET --arg ro|rw`, it also checks a live share at +CI uses `/sbin/nvx-sandbox-smoke` as the entrypoint to verify the security +profile that this section describes before clean guest exit. Before any other +check, it requires: + +- the fixed `65534:65534` identity as the real, effective, saved, and + file-system IDs, no supplementary groups, and `HOME=/nonexistent`; +- empty inheritable, permitted, effective, bounding, and ambient capability + sets and `NoNewPrivs: 1`; +- mount, PID, and UTS namespaces other than the guest's initial ones, whose + inode numbers Linux 6.18 fixes in its UAPI, with the workload as PID 1 of its + PID namespace and the `--hostname` value as its host name; +- exactly one mount at each of `/`, `/proc`, `/sys`, `/dev`, `/dev/pts`, and + `/dev/shm`, none of which propagates to another namespace: an overlay root + over the EROFS layers whose upper directory is on scratch, `nosuid,nodev,noexec` + procfs, read-only sysfs, and a private `/dev` tmpfs that holds only `fd`, + `full`, `null`, `ptmx`, `pts`, `random`, `shm`, `stderr`, `stdin`, `stdout`, + `tty`, `urandom`, and `zero`, with its own devpts instance; +- the agent-owned `/container` cgroup; and +- Ubuntu image files that the workload cannot modify, and a scratch-backed + `/tmp` write that lands in the overlay. + +A managed workload that mounted into the agent's namespace would stack its +runtime mounts on those of earlier requests, so the single-mount check also +covers repeated managed execs. With `--arg limits --arg MEMORY_MAX --arg +PIDS_MAX`, for a sandbox started with the same `--memory-max` and +`--pids-max`, it also requires the memory controller to kill an allocation of +twice `MEMORY_MAX` with `SIGKILL` while a 1 MiB allocation succeeds, and the +pids controller to stop the workload at `PIDS_MAX` processes, itself included; +keep twice `MEMORY_MAX` within the guest's free memory. With +`--arg TARGET --arg ro|rw`, it also checks a live share at `TARGET` as described below, including symbolic links in an `rw` share; the share needs a host-created, world-writable `nvx-links` directory for them. Repeat the pair to check several shares; with an `rw` and an `ro` share, it @@ -692,7 +719,10 @@ request to retain VM state. Environment files are limited to 1 MiB of UTF-8 JSON. Environments contain at most 256 unique, nonempty names. Each `KEY=VALUE` entry and working-directory path is limited to 4096 UTF-8 bytes; the combined execution request must also -fit the existing 64 KiB control-payload bound. +fit the existing 64 KiB control-payload bound. A managed `exec` forwards at +most 1 MiB of combined standard output and standard error; the guest agent +kills a workload that writes more, and `exec` exits with status 125 and reports +the `output-limit` category. The explicit `test-microvm --scenario managed-exec-config --backend BACKEND` scenario is the authoritative acceptance for these public options. It invokes @@ -718,6 +748,56 @@ synthesize its own variables. The scenario retains bounded subprocess argument and status observations, typed exec outcomes, and OpenVMM logs. Inline environment values are redacted from the retained command observations. +The explicit `test-microvm --scenario sandbox-lifecycle --backend BACKEND` +scenario is the authoritative acceptance for the managed lifecycle itself. It +also runs the public commands as subprocesses against the real kernel, Alpine +control initramfs, Ubuntu EROFS layer, and fresh copies of the same scratch +template, so it has the same artifact requirements and also runs only when +named. It checks that: + +- `provision` rejects a root `--workload-user` before it creates any state, and + the guest refuses an identity that the Ubuntu image lacks before a one-shot + workload starts, and in a managed `start`, which then fails without leaving + a runtime record, capability, control socket, or OpenVMM process; +- every operation fails on a state directory that was never provisioned, a + repeated `provision` leaves the configuration unchanged, and `exec` and + `stop` fail before `start`; +- a repeated `start` and a `deprovision` of a running sandbox fail without + changing its runtime record, capability, or OpenVMM process, and the + sandbox keeps serving requests; +- managed arguments keep leading, trailing, and embedded spaces, tabs, and + newlines, a later request reads the file that an earlier one wrote, and the + file survives `stop` and a new `start`, because the overlay's upper directory + is on scratch; +- `/sbin/nvx-sandbox-smoke` passes its security profile and resource-limit + checks in a sandbox provisioned with `--hostname`, `--memory-max`, and + `--pids-max`, after earlier requests; +- a request that writes more than the 1 MiB output bound exits with status + 125 and an `output-limit` outcome report after the host has received more + than 1 MiB less one 32 KiB read chunk and at most 1 MiB, and a request whose + `--outcome-report` path exists fails before its workload runs and leaves the + file intact; +- `stop` ends OpenVMM and its control socket or pipe, leaves only + `config.json`, `openvmm.log`, and a successful `outcome.json`, and leaves a + cleanly unmounted scratch file system, after which `exec` and `stop` fail; +- after OpenVMM exits without a `stop`, `exec` and `stop` report the stale + runtime state and `start` refuses it; and +- `deprovision` removes only NVX's files: it fails and keeps a foreign file in + the state directory, and once that file is gone, it removes the directory, + which a later `exec` does not recreate. + +Throughout, no OpenVMM process whose arguments name the scenario's fixture +outlives its sandbox, and the EROFS layer keeps its digest and the scratch +image its file. On failure, the scenario stops and deprovisions what it +started, ends every OpenVMM process that names its fixture, and preserves the +fixture for recovery only if that cleanup fails. Its evidence, written to the +output directory with a `sandbox-lifecycle-` prefix, holds bounded command +observations (arguments, statuses, output sizes, and state-directory entries), +the runtime record, the OpenVMM logs, the VM-level and exec outcome reports, the +guest probe's transcript, and the console of the refused one-shot run, each +limited to its last 64 KiB. The command observations hold no workload output, +and no evidence holds the control capability. + Decoder, helper, and direct control-session tests remain useful supplemental coverage for protocol boundaries and guest implementation details. They do not replace or establish support through the public `nvx.py sandbox` interface. diff --git a/doc/usage.md b/doc/usage.md index ff16b8d8a..32f5d330e 100644 --- a/doc/usage.md +++ b/doc/usage.md @@ -275,12 +275,16 @@ python3 scripts/nvx.py test-microvm Runs NVX-owned Linux, SMP, virtio, sandbox, and snapshot correctness scenarios against the public OpenVMM CLI. Repeat `--scenario` to select a subset; without -it, every scenario supported by the selected guest runs, except `smp-lapic`, -which runs only when named: it repeats `smp` and also asserts that every CPU -uses the one-shot counting LAPIC. Alpine remains the +it, every scenario supported by the selected guest runs, except three that run +only when named. `smp-lapic` repeats `smp` and also asserts that every CPU +uses the one-shot counting LAPIC. `managed-exec-config` and +`sandbox-lifecycle` drive the public `sandbox` lifecycle commands (see +[Run](run.md#managed-lifecycle)) and also require the guest-artifact build's +Ubuntu layer and scratch template, which packages omit. Alpine remains the default. Ubuntu and Azure Linux cannot act as sandbox control, so they reject the Alpine-control-only `sandbox-blocks` and `scratch-snapshot` scenarios and -the sandbox-control-dependent `snapshot-tiers` scenario. Ubuntu also rejects +the sandbox-control-dependent `snapshot-tiers`, `managed-exec-config`, and +`sandbox-lifecycle` scenarios. Ubuntu also rejects the Alpine-prompt-specific `console-snapshot` scenario. `--debug-kernel` boots the CI debug kernel, `build/vmlinux-debug`, whose soft-lockup and hung-task detectors the guest's time ABI watcher reports; without `--scenario`, it runs diff --git a/guest/common/nvx-sandbox-smoke b/guest/common/nvx-sandbox-smoke index dea324847..777cd7aa3 100644 --- a/guest/common/nvx-sandbox-smoke +++ b/guest/common/nvx-sandbox-smoke @@ -1,6 +1,279 @@ #!/bin/sh set -eu +# The fixed workload identity, as the four real, effective, saved, and +# file-system IDs that /proc/self/status lists. +workload_ids="65534 65534 65534 65534" +# Linux 6.18 fixes the inode numbers of the initial namespaces in its UAPI +# (enum init_ns_ino in linux/nsfs.h). The agent runs in the initial mount, +# PID, and UTS namespaces, so a workload namespace with another number is +# private. +initial_namespaces="mnt:4026531832 pid:4026531836 uts:4026531838" +# The private /dev that the container runtime creates, in byte order. +device_view="fd full null ptmx pts random shm stderr stdin stdout tty urandom zero" + +fail() { + echo "NVX-UBUNTU-SANDBOX-FAIL: $*" >&2 + exit 1 +} + +# Prints the values of field $2 of the process status file $1, separated by +# single spaces. +status_field() { + while IFS= read -r status_line; do + case "$status_line" in + "$2":*) + set -f + # The values are tab-separated words. + # shellcheck disable=SC2086 + set -- ${status_line#*:} + set +f + printf '%s\n' "$*" + return 0 + ;; + esac + done <"$1" + fail "$1 has no $2 field" +} + +check_identity() { + [ "$(id -u):$(id -g)" = 65534:65534 ] || + fail "the workload runs as $(id -u):$(id -g)" + uids=$(status_field /proc/self/status Uid) + [ "$uids" = "$workload_ids" ] || fail "the workload UIDs are $uids" + gids=$(status_field /proc/self/status Gid) + [ "$gids" = "$workload_ids" ] || fail "the workload GIDs are $gids" + groups=$(status_field /proc/self/status Groups) + [ -z "$groups" ] || fail "the workload has supplementary groups $groups" + [ "${HOME:-}" = /nonexistent ] || fail "the workload home is ${HOME:-unset}" +} + +# The workload holds no capability in any set and cannot gain privileges +# through execve. +check_capabilities() { + for capability_set in CapInh CapPrm CapEff CapBnd CapAmb; do + capabilities=$(status_field /proc/self/status "$capability_set") + [ "$capabilities" = 0000000000000000 ] || + fail "the workload $capability_set set is $capabilities" + done + no_new_privs=$(status_field /proc/self/status NoNewPrivs) + [ "$no_new_privs" = 1 ] || fail "the workload NoNewPrivs is $no_new_privs" +} + +check_namespaces() { + for initial_namespace in $initial_namespaces; do + namespace_type=${initial_namespace%%:*} + initial_inode=${initial_namespace#*:} + namespace_link=$(readlink "/proc/self/ns/$namespace_type") || + fail "cannot read the workload $namespace_type namespace" + namespace_inode=${namespace_link#"$namespace_type:["} + namespace_inode=${namespace_inode%"]"} + case "$namespace_link" in + "$namespace_type:[$namespace_inode]") ;; + *) fail "unexpected $namespace_type namespace link $namespace_link" ;; + esac + case "$namespace_inode" in + '' | *[!0-9]*) + fail "unexpected $namespace_type namespace link $namespace_link" + ;; + esac + [ "$namespace_inode" != "$initial_inode" ] || + fail "the workload shares the initial $namespace_type namespace" + done + # PID 1 of the workload's PID namespace is the workload, not the guest's + # root-owned init. + init_uids=$(status_field /proc/1/status Uid) + [ "$init_uids" = "$workload_ids" ] || + fail "PID 1 of the workload PID namespace runs as $init_uids" + expected_hostname=nvx-sandbox + read -r kernel_command_line >"$image_file") 2>/dev/null; then + fail "the workload can modify image file $image_file" + fi + done + if (: >/usr/nvx-sandbox-image-write) 2>/dev/null; then + fail "the workload can create files in image directory /usr" + fi + printf 'scratch-backed-write\n' >/tmp/nvx-ubuntu-sandbox-smoke + [ "$(cat /tmp/nvx-ubuntu-sandbox-smoke)" = scratch-backed-write ] || + fail "the scratch-backed write did not persist" + [ "$(stat -f -c %T /tmp/nvx-ubuntu-sandbox-smoke)" = overlayfs ] || + fail "the scratch-backed write did not land in the overlay" +} + +# Starts sleepers until a fork fails or $2 have started, then stops and reaps +# them and records how many started in $1. A refused fork ends the shell, +# which still runs its exit trap. +start_sleepers() { + sleepers_file=$1 + sleepers= + sleepers_started=0 + trap 'kill $sleepers 2>/dev/null || :; wait; echo "$sleepers_started" >"$sleepers_file"' EXIT + while [ "$sleepers_started" -lt "$2" ]; do + sleep 600 & + sleepers="$sleepers $!" + sleepers_started=$((sleepers_started + 1)) + done +} + +# The workload's memory.max is $1 bytes, and its pids.max lets it run $2 +# processes, itself included. Keep twice $1 within the guest's free memory, +# so that an unenforced memory.max fails the check instead of the guest. +check_limits() { + memory_max=$1 + pids_max=$2 + for limit in "$memory_max" "$pids_max"; do + case "$limit" in + '' | *[!0-9]*) fail "limits requires numeric MEMORY_MAX and PIDS_MAX" ;; + esac + done + [ "$pids_max" -ge 3 ] || fail "the process limit check requires PIDS_MAX of at least 3" + dd if=/dev/zero of=/dev/null bs=1M count=1 2>/dev/null || + fail "a 1 MiB allocation failed under memory.max" + # The memory controller kills an allocation of twice the limit. The shell + # reports the kill on standard error, so silence it. + memory_status=0 + { dd if=/dev/zero of=/dev/null bs="$((memory_max * 2))" count=1; } \ + 2>/dev/null || memory_status=$? + [ "$memory_status" -eq 137 ] || + fail "memory.max did not kill an allocation of $((memory_max * 2)) bytes: status $memory_status" + # `--pids-max N` admits N workload processes: the CLI sets the container + # cgroup's pids.max to N + 1 for the launcher's `unshare` parent, which + # waits outside the workload's PID namespace. The probe and the subshell + # are two of the N, so the subshell can start $2 - 2 sleepers. The bound + # stops a subshell whose forks the controller does not refuse. + sleepers_record=/tmp/nvx-ubuntu-sandbox-sleepers + rm -f "$sleepers_record" + (start_sleepers "$sleepers_record" "$((pids_max * 4))") 2>/dev/null || : + read -r started <"$sleepers_record" || + fail "the process limit check did not record its sleepers" + rm -f "$sleepers_record" + [ "$started" = "$((pids_max - 2))" ] || + fail "pids.max let the workload start $started sleepers beside the probe and a subshell, expected $((pids_max - 2))" + /bin/true || fail "the workload cannot fork after its processes exit" + echo "NVX-UBUNTU-SANDBOX-LIMITS-OK memory_max=$memory_max pids_max=$pids_max" +} + check_share() { share=$1 share_mode=$2 @@ -222,16 +495,25 @@ check_shares_isolated() { echo "NVX-UBUNTU-SANDBOX-SHARES-ISOLATED-OK rw=$rw_share ro=$ro_share" } -[ "$(id -u)" = 65534 ] -[ "$(id -g)" = 65534 ] -[ "${HOME:-}" = /nonexistent ] -grep -qx 'ID=ubuntu' /etc/os-release -printf 'scratch-backed-write\n' >/tmp/nvx-ubuntu-sandbox-smoke -[ "$(cat /tmp/nvx-ubuntu-sandbox-smoke)" = scratch-backed-write ] +check_identity +check_capabilities +check_namespaces +check_mounts +check_devices +check_cgroup +check_image +echo "NVX-UBUNTU-SANDBOX-PROFILE-OK uid=65534 gid=65534" # Each share is a TARGET MODE pair; a final TARGET alone is read-only. +# `limits MEMORY_MAX PIDS_MAX` checks the workload's resource limits. rw_share= ro_share= while [ "$#" -gt 0 ]; do + if [ "$1" = limits ]; then + [ "$#" -ge 3 ] || fail "limits requires MEMORY_MAX and PIDS_MAX" + check_limits "$2" "$3" + shift 3 + continue + fi share_mode=${2:-ro} check_share "$1" "$share_mode" case "$share_mode" in diff --git a/scripts/nvx_tools/managed_exec_tests.py b/scripts/nvx_tools/managed_exec_tests.py index be1f84fda..3a9ee1bb6 100644 --- a/scripts/nvx_tools/managed_exec_tests.py +++ b/scripts/nvx_tools/managed_exec_tests.py @@ -7,6 +7,7 @@ import subprocess import sys import tempfile +from dataclasses import dataclass from pathlib import Path from typing import Any, cast @@ -26,14 +27,46 @@ } -def _bounded_text(value: bytes) -> str: +@dataclass(frozen=True) +class WorkloadArtifacts: + """The Ubuntu workload layer and scratch template of the public scenarios.""" + + distro: Path + layer_uuid: str + scratch_template: Path + + +def load_workload_artifacts() -> WorkloadArtifacts: + distro = require_file( + artifact_path(UbuntuBuildConstants.DISTRO_NAME), "Ubuntu workload layer" + ) + manifest_path = require_file( + distro.with_name(UbuntuBuildConstants.DISTRO_MANIFEST_NAME), + "Ubuntu layer manifest", + ) + scratch_template = require_file( + artifact_path("ubuntu-smoke-scratch.ext4"), "sandbox scratch template" + ) + try: + manifest: object = json.loads(manifest_path.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as error: + raise ScriptError(f"invalid Ubuntu layer manifest: {manifest_path}") from error + if not isinstance(manifest, dict): + raise ScriptError("Ubuntu layer manifest must contain a UUID string") + layer_uuid = cast(dict[str, object], manifest).get("uuid") + if not isinstance(layer_uuid, str): + raise ScriptError("Ubuntu layer manifest must contain a UUID string") + return WorkloadArtifacts(distro, layer_uuid, scratch_template) + + +def bounded_text(value: bytes) -> str: text = value[:DIAGNOSTIC_LIMIT].decode(errors="replace") if len(value) > DIAGNOSTIC_LIMIT: text += f"... ({len(value) - DIAGNOSTIC_LIMIT} bytes omitted)" return text -def _evidence_argv(command: list[str]) -> list[str]: +def evidence_argv(command: list[str]) -> list[str]: recorded = command.copy() for index, value in enumerate(recorded[:-1]): if value == "--environment": @@ -41,9 +74,7 @@ def _evidence_argv(command: list[str]) -> list[str]: return recorded -def _read_exec_outcome( - path: Path, *, category: str, status_code: int -) -> dict[str, Any]: +def read_exec_outcome(path: Path, *, category: str, status_code: int) -> dict[str, Any]: try: value: object = json.loads(path.read_text(encoding="utf-8")) except (OSError, UnicodeDecodeError, json.JSONDecodeError) as error: @@ -125,34 +156,19 @@ def _default_environment_matches( ) -def _format_errors(errors: list[Exception]) -> str: +def format_errors(errors: list[Exception]) -> str: text = "; ".join(f"{type(error).__name__}: {error}" for error in errors) encoded = text.encode("utf-8", errors="replace") - return _bounded_text(encoded) + return bounded_text(encoded) def run_managed_exec_configuration( backend: str, *, timeout: float, output_dir: Path ) -> None: - distro = require_file( - artifact_path(UbuntuBuildConstants.DISTRO_NAME), "Ubuntu workload layer" - ) - manifest_path = require_file( - distro.with_name(UbuntuBuildConstants.DISTRO_MANIFEST_NAME), - "Ubuntu layer manifest", - ) - scratch_template = require_file( - artifact_path("ubuntu-smoke-scratch.ext4"), "sandbox scratch template" - ) - try: - manifest: object = json.loads(manifest_path.read_text(encoding="utf-8")) - except (OSError, UnicodeDecodeError, json.JSONDecodeError) as error: - raise ScriptError(f"invalid Ubuntu layer manifest: {manifest_path}") from error - if not isinstance(manifest, dict): - raise ScriptError("Ubuntu layer manifest must contain a UUID string") - layer_uuid = cast(dict[str, object], manifest).get("uuid") - if not isinstance(layer_uuid, str): - raise ScriptError("Ubuntu layer manifest must contain a UUID string") + artifacts = load_workload_artifacts() + distro = artifacts.distro + layer_uuid = artifacts.layer_uuid + scratch_template = artifacts.scratch_template output_dir.mkdir(parents=True, exist_ok=True) checks: list[dict[str, object]] = [] root = Path(tempfile.mkdtemp(prefix="nvx-public-exec-")) @@ -187,7 +203,7 @@ def invoke( checks.append( { "operation": operation, - "argv": _evidence_argv([str(value) for value in result.args]), + "argv": evidence_argv([str(value) for value in result.args]), "expected_returncode": expected, "returncode": result.returncode, "stdout_bytes": len(result.stdout), @@ -198,8 +214,8 @@ def invoke( if result.returncode != expected: raise RuntimeError( f"public sandbox {operation} returned {result.returncode}, " - f"expected {expected}; stdout={_bounded_text(result.stdout)!r}; " - f"stderr={_bounded_text(result.stderr)!r}" + f"expected {expected}; stdout={bounded_text(result.stdout)!r}; " + f"stderr={bounded_text(result.stderr)!r}" ) return result @@ -216,8 +232,8 @@ def expect_output( if result.stdout != expected or result.stderr: raise RuntimeError( "public managed workload returned unexpected output: " - f"stdout={_bounded_text(result.stdout)!r}, " - f"stderr={_bounded_text(result.stderr)!r}" + f"stdout={bounded_text(result.stdout)!r}, " + f"stderr={bounded_text(result.stderr)!r}" ) def expect_rejection( @@ -226,8 +242,8 @@ def expect_rejection( if result.stdout or message not in result.stderr: raise RuntimeError( "public managed validation returned unexpected diagnostics: " - f"stdout={_bounded_text(result.stdout)!r}, " - f"stderr={_bounded_text(result.stderr)!r}" + f"stdout={bounded_text(result.stdout)!r}, " + f"stderr={bounded_text(result.stderr)!r}" ) def persist_evidence() -> None: @@ -255,7 +271,7 @@ def persist_evidence() -> None: if errors: raise RuntimeError( "public managed evidence persistence failed: " - f"{_format_errors(errors)}" + f"{format_errors(errors)}" ) from errors[0] invoke( @@ -309,7 +325,7 @@ def persist_evidence() -> None: if identity.stderr: raise RuntimeError( "public workload identity probe returned diagnostics: " - f"{_bounded_text(identity.stderr)!r}" + f"{bounded_text(identity.stderr)!r}" ) workload_name, workload_home = _read_workload_identity(identity.stdout) expected_defaults = { @@ -411,7 +427,7 @@ def persist_evidence() -> None: ) if result.stdout != b"public stdout" or result.stderr != b"public stderr": raise RuntimeError("public managed stdout/stderr were not preserved") - _read_exec_outcome(exit_outcome, category="exit", status_code=7) + read_exec_outcome(exit_outcome, category="exit", status_code=7) for limit in (0, 3_600_001, 86_400_000, 0xFFFFFFFF): expect_output( @@ -431,7 +447,7 @@ def persist_evidence() -> None: raise RuntimeError( "public timed-out workload returned unexpected output" ) - _read_exec_outcome(timeout_outcome, category="timeout", status_code=124) + read_exec_outcome(timeout_outcome, category="timeout", status_code=124) expect_output(workload("/bin/pwd"), b"/\n") except Exception as error: acceptance_error = error @@ -460,7 +476,7 @@ def persist_evidence() -> None: except Exception as error: cleanup_errors.append(error) else: - preserved_path = _bounded_text( + preserved_path = bounded_text( str(root).encode("utf-8", errors="replace") ) preservation_reported = True @@ -473,18 +489,18 @@ def persist_evidence() -> None: if cleanup_errors: raise RuntimeError( f"{acceptance_error}; cleanup failed: " - f"{_format_errors(cleanup_errors)}" + f"{format_errors(cleanup_errors)}" ) from acceptance_error raise acceptance_error.with_traceback(acceptance_error.__traceback__) if cleanup_errors: raise RuntimeError( - f"public managed cleanup failed: {_format_errors(cleanup_errors)}" + f"public managed cleanup failed: {format_errors(cleanup_errors)}" ) from cleanup_errors[0] if not scratch_template.is_file() or not distro.is_file(): raise RuntimeError("managed cleanup removed a supplied workload artifact") except Exception as error: if root.exists() and not preservation_reported: - preserved_path = _bounded_text(str(root).encode("utf-8", errors="replace")) + preserved_path = bounded_text(str(root).encode("utf-8", errors="replace")) raise RuntimeError( f"{error}; managed fixture preserved for recovery: {preserved_path}" ) from error diff --git a/scripts/nvx_tools/microvm_tests.py b/scripts/nvx_tools/microvm_tests.py index 2bf1bc361..3b3153d29 100644 --- a/scripts/nvx_tools/microvm_tests.py +++ b/scripts/nvx_tools/microvm_tests.py @@ -60,6 +60,7 @@ from .guests import GUEST_NAMES, GuestDescriptor, guest_descriptor from .managed_exec_tests import run_managed_exec_configuration from .openvmm_process import OpenvmmProcess, TcpConsole +from .sandbox_lifecycle_tests import run_sandbox_lifecycle from .time_abi import ( ABI_VERSION, CHECK_CPU_BUDGET_US, @@ -101,6 +102,7 @@ "restore-memory", "restore-processors", "sandbox-blocks", + "sandbox-lifecycle", "scratch-snapshot", "smp", "smp-snapshot", @@ -116,10 +118,15 @@ ( "managed-exec-config", "sandbox-blocks", + "sandbox-lifecycle", "scratch-snapshot", "snapshot-tiers", ) ) +# The public `nvx.py sandbox` scenarios need the Ubuntu scratch template that +# the guest-artifact build produces and packages omit, so they run only when +# named. +PUBLIC_SANDBOX_SCENARIOS = frozenset(("managed-exec-config", "sandbox-lifecycle")) # The spec runs the same-host restore cases on the CI debug kernel, whose # soft-lockup and hung-task detectors the guest's time ABI watcher reports. DEBUG_KERNEL_SCENARIOS = ( @@ -281,7 +288,8 @@ def configure_parser(parser: argparse.ArgumentParser) -> None: choices=(*MICROVM_TEST_SCENARIOS, *MICROVM_EXPLICIT_SCENARIOS), help=( "scenario to run; repeat to select multiple (default: all except " - "smp-lapic, which runs only when named)" + "smp-lapic, managed-exec-config, and sandbox-lifecycle, which run " + "only when named)" ), ) parser.add_argument( @@ -5393,7 +5401,7 @@ def run(args: argparse.Namespace) -> int: scenario for scenario in defaults if scenario not in unsupported_scenarios - and scenario != "managed-exec-config" + and scenario not in PUBLIC_SANDBOX_SCENARIOS ) else: scenarios = tuple(dict.fromkeys(args.scenario)) @@ -5550,6 +5558,9 @@ def run(args: argparse.Namespace) -> int: run_managed_exec_configuration( args.backend, timeout=args.timeout, output_dir=output_dir ) + if "sandbox-lifecycle" in scenarios: + print(f"Running public managed sandbox lifecycle on OpenVMM/{args.backend}") + run_sandbox_lifecycle(args.backend, timeout=args.timeout, output_dir=output_dir) if "managed-lifecycle" in scenarios: print(f"Running managed microVM lifecycle on OpenVMM/{args.backend}") run_managed_lifecycle( diff --git a/scripts/nvx_tools/sandbox_lifecycle.py b/scripts/nvx_tools/sandbox_lifecycle.py index f781f7399..99bd34fd5 100644 --- a/scripts/nvx_tools/sandbox_lifecycle.py +++ b/scripts/nvx_tools/sandbox_lifecycle.py @@ -443,6 +443,11 @@ def _runtime_process(runtime: dict[str, Any]) -> tuple[int, int | None]: return pid, start_time +def runtime_process_running(runtime: dict[str, Any]) -> bool: + """Returns whether the OpenVMM process of a runtime record still runs.""" + return _process_running(*_runtime_process(runtime)) + + def _load_running(state_dir: Path) -> tuple[dict[str, Any], bytes]: runtime_path = state_dir / RUNTIME_NAME if not runtime_path.is_file(): diff --git a/scripts/nvx_tools/sandbox_lifecycle_tests.py b/scripts/nvx_tools/sandbox_lifecycle_tests.py new file mode 100644 index 000000000..b190605a9 --- /dev/null +++ b/scripts/nvx_tools/sandbox_lifecycle_tests.py @@ -0,0 +1,756 @@ +"""Public CLI acceptance for the managed sandbox lifecycle and security profile. + +The scenario runs `nvx.py sandbox` as subprocesses against the real kernel, the +Alpine control initramfs, the Ubuntu EROFS layer, and ext4 scratch copies. It +checks identity admission, fail-closed lifecycle transitions, warm-guest state +across exec requests and restarts, the guest security profile and resource +limits, bounded output and outcome reports, and that stop, a crash, and +deprovision leave no OpenVMM process, control endpoint, capability, or runtime +record behind while the supplied layer and scratch artifacts stay intact. +""" + +from __future__ import annotations + +import contextlib +import hashlib +import json +import os +import shutil +import signal +import struct +import subprocess +import sys +import tempfile +import time +from pathlib import Path +from typing import Any, cast + +from . import sandbox_lifecycle +from .build_constants import BuildConstants +from .common import sha256_file +from .managed_exec_tests import ( + WorkloadArtifacts, + bounded_text, + evidence_argv, + format_errors, + load_workload_artifacts, + read_exec_outcome, +) + +PROBE = "/sbin/nvx-sandbox-smoke" +HOSTNAME = "nvx-lifecycle" +MEMORY_MIB = 256 +MEMORY_MAX = 32 * 1024 * 1024 +PIDS_MAX = 8 +# An identity that the Ubuntu layer's user database lacks. +UNAVAILABLE_IDENTITY = "4242:4242" +# The guest agent forwards at most this much output for one exec request, and +# reads it in chunks of at most OUTPUT_CHUNK_BYTES. It drops the chunk that +# would exceed the bound, so more than OUTPUT_LIMIT_BYTES - OUTPUT_CHUNK_BYTES +# bytes reach the host. +OUTPUT_LIMIT_BYTES = 1024 * 1024 +OUTPUT_CHUNK_BYTES = 32 * 1024 +STATE_PATH = "/tmp/nvx lifecycle state" +STATE_ARGUMENTS = (" leading and trailing ", "tab\tseparated", "line one\nline two") +STATE_CONTENT = "".join(f"{argument}\n" for argument in STATE_ARGUMENTS).encode() +REFUSED_MARKER = "/tmp/nvx-refused-request" +FOREIGN_NAME = "operator-note.txt" +FOREIGN_CONTENT = "operator data\n" +EVIDENCE_PREFIX = "sandbox-lifecycle-" +EVIDENCE_LIMIT = 64 * 1024 +# ext4 sets this incompatible feature while its journal needs recovery. +EXT4_INCOMPAT_RECOVER = 0x4 +MANAGED_SUCCESS = {"operation": "managed", "category": "success", "status_code": 0} + +CONFIG = sandbox_lifecycle.CONFIG_NAME +RUNTIME = sandbox_lifecycle.RUNTIME_NAME +CAPABILITY = sandbox_lifecycle.CAPABILITY_NAME +LOG = sandbox_lifecycle.LOG_NAME +SOCKET = sandbox_lifecycle.CONTROL_SOCKET_NAME +OUTCOME = sandbox_lifecycle.OUTCOME_NAME +# Windows hosts serve the control endpoint as a named pipe outside the state. +RUNNING_ENTRIES = frozenset( + {CONFIG, RUNTIME, CAPABILITY, LOG} + if os.name == "nt" + else {CONFIG, RUNTIME, CAPABILITY, LOG, SOCKET} +) +STOPPED_ENTRIES = frozenset({CONFIG, LOG, OUTCOME}) + + +def openvmm_processes(marker: str) -> tuple[int, ...]: + """Returns the IDs of the OpenVMM processes whose arguments contain marker.""" + processes: list[int] = [] + if sys.platform == "win32": + script = ( + "Get-CimInstance Win32_Process -Filter \"Name = 'openvmm.exe'\" | " + "ForEach-Object { [string]$_.ProcessId + [char]9 + " + "[string]$_.CommandLine }" + ) + result = subprocess.run( + ["powershell.exe", "-NoProfile", "-NonInteractive", "-Command", script], + capture_output=True, + timeout=120, + check=False, + ) + if result.returncode != 0: + raise RuntimeError( + f"cannot list OpenVMM processes: {bounded_text(result.stderr)}" + ) + for line in result.stdout.decode(errors="replace").splitlines(): + pid, _, command_line = line.partition("\t") + if marker in command_line: + processes.append(int(pid)) + return tuple(sorted(processes)) + for entry in Path("/proc").iterdir(): + if not entry.name.isdigit(): + continue + try: + arguments = (entry / "cmdline").read_bytes().split(b"\0") + except OSError: + continue + if Path(os.fsdecode(arguments[0])).name == "openvmm" and any( + marker.encode() in argument for argument in arguments + ): + processes.append(int(entry.name)) + return tuple(sorted(processes)) + + +def control_endpoint_present(runtime: dict[str, Any]) -> bool: + endpoint = str(runtime["control_endpoint"]) + if sys.platform == "win32": + return endpoint.rsplit("/", 1)[-1] in os.listdir("//./pipe/") + return os.path.lexists(endpoint) + + +def terminate_process(pid: int) -> None: + if sys.platform == "win32": + os.kill(pid, signal.SIGTERM) + else: + os.kill(pid, signal.SIGKILL) + + +def kill_openvmm(runtime: dict[str, Any], timeout: float) -> None: + """Ends the OpenVMM process of a runtime record, as a crash would.""" + if sandbox_lifecycle.runtime_process_running(runtime): + try: + terminate_process(int(runtime["pid"])) + except OSError: + # The process may have exited since the check. + if sandbox_lifecycle.runtime_process_running(runtime): + raise + deadline = time.monotonic() + timeout + while sandbox_lifecycle.runtime_process_running(runtime): + if time.monotonic() >= deadline: + raise RuntimeError(f"OpenVMM process {runtime['pid']} survived its kill") + time.sleep(0.05) + + +def scratch_journal_clean(path: Path) -> bool: + with path.open("rb") as image: + image.seek(1024 + 0x60) + incompatible = image.read(4) + if len(incompatible) != 4: + raise RuntimeError(f"scratch superblock is truncated: {path}") + return not struct.unpack(" list[str] | None: + if not state.is_dir(): + return None + return sorted(path.name for path in state.iterdir()) + + +def bounded_evidence(data: bytes, omitted: int = 0) -> bytes: + """Keeps the end of data, where a failure shows, within EVIDENCE_LIMIT. + + `omitted` counts bytes that preceded data in its source. + """ + omitted += max(0, len(data) - EVIDENCE_LIMIT) + data = data[-EVIDENCE_LIMIT:] + if omitted: + return f"... ({omitted} earlier bytes omitted)\n".encode() + data + return data + + +def read_runtime(state: Path) -> dict[str, Any]: + path = state / RUNTIME + try: + value: object = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as error: + raise RuntimeError(f"sandbox runtime record is unreadable: {path}") from error + if not isinstance(value, dict): + raise RuntimeError(f"sandbox runtime record is not an object: {path}") + runtime = cast(dict[str, Any], value) + if set(runtime) != {"format", "pid", "start_time", "control_endpoint"}: + raise RuntimeError(f"sandbox runtime record has unexpected fields: {path}") + return runtime + + +class LifecycleAcceptance: + def __init__( + self, + backend: str, + *, + timeout: float, + output_dir: Path, + artifacts: WorkloadArtifacts, + root: Path, + ) -> None: + self.backend = backend + self.timeout = timeout + self.output_dir = output_dir + self.artifacts = artifacts + self.root = root + self.state = root / "state" + self.identity_state = root / "identity-state" + self.scratch = root / "scratch.ext4" + self.checks: list[dict[str, object]] = [] + + def launch_options(self, scratch: Path) -> list[str]: + return [ + "--hypervisor", + self.backend, + "--layer", + f"distro,{self.artifacts.distro},{self.artifacts.layer_uuid}", + "--scratch", + str(scratch), + "--memory-mib", + str(MEMORY_MIB), + ] + + def run_cli( + self, operation: str, *arguments: str, state: Path | None + ) -> subprocess.CompletedProcess[bytes]: + command = [ + sys.executable, + str(BuildConstants.REPO_ROOT / "scripts" / "nvx.py"), + "sandbox", + operation, + ] + if state is not None: + command.extend(("--state-dir", str(state))) + command.extend(("--timeout", str(self.timeout), *arguments)) + return subprocess.run( + command, + cwd=BuildConstants.REPO_ROOT, + capture_output=True, + timeout=2 * self.timeout + 30, + ) + + def invoke( + self, + operation: str, + *arguments: str, + state: Path | None, + expected: int | None = 0, + diagnostic: bytes | None = None, + ) -> subprocess.CompletedProcess[bytes]: + """Runs one public operation and records only its bounded observations. + + It requires status `expected` unless that is None, and `diagnostic` in + standard error when given. + """ + result = self.run_cli(operation, *arguments, state=state) + self.checks.append( + { + "operation": operation, + "argv": evidence_argv([str(value) for value in result.args]), + "expected_returncode": expected, + "returncode": result.returncode, + "stdout_bytes": len(result.stdout), + "stderr_bytes": len(result.stderr), + "state_entries": None if state is None else state_entries(state), + } + ) + if (expected is not None and result.returncode != expected) or ( + diagnostic is not None and diagnostic not in result.stderr + ): + wanted = "" if diagnostic is None else f" with {diagnostic.decode()!r}" + raise RuntimeError( + f"public sandbox {operation} returned {result.returncode}, expected " + f"{expected}{wanted}; stdout={bounded_text(result.stdout)!r}; " + f"stderr={bounded_text(result.stderr)!r}" + ) + return result + + def exec( + self, + entrypoint: str, + *arguments: str, + expected: int | None = 0, + diagnostic: bytes | None = None, + ) -> subprocess.CompletedProcess[bytes]: + return self.invoke( + "exec", + "--entrypoint", + entrypoint, + *arguments, + state=self.state, + expected=expected, + diagnostic=diagnostic, + ) + + def require_entries( + self, state: Path, expected: frozenset[str], description: str + ) -> None: + entries = state_entries(state) + if entries is None or frozenset(entries) != expected: + raise RuntimeError( + f"{description}: the state directory holds {entries}, expected " + f"{sorted(expected)}" + ) + + def require_output( + self, result: subprocess.CompletedProcess[bytes], stdout: bytes, what: str + ) -> None: + if result.returncode != 0 or result.stdout != stdout or result.stderr: + raise RuntimeError( + f"{what} returned status {result.returncode}, " + f"stdout={bounded_text(result.stdout)!r}, " + f"stderr={bounded_text(result.stderr)!r}" + ) + + def require_processes(self, expected: tuple[int, ...], description: str) -> None: + processes = openvmm_processes(self.root.name) + if processes != expected: + raise RuntimeError( + f"{description}: OpenVMM processes {list(processes)} run for the " + f"fixture, expected {list(expected)}" + ) + + def save_evidence(self, name: str, data: bytes, omitted: int = 0) -> None: + (self.output_dir / f"{EVIDENCE_PREFIX}{name}").write_bytes( + bounded_evidence(data, omitted) + ) + + def copy_evidence(self, source: Path, name: str) -> None: + if not source.is_file(): + return + with source.open("rb") as stream: + size = stream.seek(0, os.SEEK_END) + stream.seek(max(0, size - EVIDENCE_LIMIT)) + data = stream.read(EVIDENCE_LIMIT) + self.save_evidence(name, data, size - len(data)) + + def write_checks(self) -> None: + (self.output_dir / f"{EVIDENCE_PREFIX}checks.json").write_text( + json.dumps(self.checks, indent=2) + "\n", encoding="utf-8" + ) + + def run(self) -> None: + layer_digest = sha256_file(self.artifacts.distro) + self.check_identity_admission() + self.check_unprovisioned() + self.provision() + runtime = self.start(description="first start") + self.check_running_transitions(runtime) + self.check_warm_state() + self.check_security_profile() + self.check_bounded_results() + self.check_deprovision_while_running(runtime) + self.stop(runtime, layer_digest) + restarted = self.start(description="restart") + result = self.exec("/bin/cat", f"--arg={STATE_PATH}", expected=None) + self.require_output(result, STATE_CONTENT, "the restarted sandbox's state") + self.check_crash(restarted) + self.check_deprovision() + if not self.scratch.is_file(): + raise RuntimeError("deprovision removed the supplied scratch image") + if sha256_file(self.artifacts.distro) != layer_digest: + raise RuntimeError("the sandbox lifecycle modified the EROFS layer") + self.require_processes((), "after deprovision") + + def check_identity_admission(self) -> None: + # The CLI rejects a root identity before it creates any state. + self.invoke( + "provision", + *self.launch_options(self.scratch), + "--workload-user", + "0:0", + state=self.state, + expected=2, + diagnostic=b"--workload-user UID must be between 1 and 4294967295", + ) + if self.state.exists(): + raise RuntimeError("a rejected root identity created sandbox state") + + # The guest refuses an identity that the image lacks before it starts + # the workload. + run_scratch = self.root / "run-scratch.ext4" + shutil.copyfile(self.artifacts.scratch_template, run_scratch) + result = self.invoke( + "run", + *self.launch_options(run_scratch), + "--workload-user", + UNAVAILABLE_IDENTITY, + "--entrypoint", + PROBE, + state=None, + expected=125, + ) + console = result.stdout + result.stderr + self.save_evidence("identity-run.log", console) + if ( + b"configured workload UID is unavailable" not in console + or b"NVX-UBUNTU-SANDBOX-PROFILE-OK" in console + ): + raise RuntimeError("an unavailable one-shot identity was not refused") + + # A managed start with that identity fails closed. + identity_scratch = self.root / "identity-scratch.ext4" + shutil.copyfile(self.artifacts.scratch_template, identity_scratch) + self.invoke( + "provision", + *self.launch_options(identity_scratch), + "--workload-user", + UNAVAILABLE_IDENTITY, + state=self.identity_state, + ) + self.invoke( + "start", state=self.identity_state, expected=1, diagnostic=b"error:" + ) + self.copy_evidence(self.identity_state / LOG, "identity-openvmm.log") + entries = set(state_entries(self.identity_state) or ()) + if entries - {OUTCOME} != {CONFIG, LOG}: + raise RuntimeError( + f"a failed start left lifecycle state behind: {sorted(entries)}" + ) + self.require_processes((), "after a refused managed identity") + self.invoke("deprovision", state=self.identity_state) + if self.identity_state.exists(): + raise RuntimeError("deprovision kept the refused sandbox's state") + + def check_unprovisioned(self) -> None: + for operation in ("start", "exec", "stop", "deprovision"): + self.invoke( + operation, + state=self.state, + expected=1, + diagnostic=b"sandbox state path is not a plain directory", + ) + if self.state.exists(): + raise RuntimeError(f"{operation} created unprovisioned sandbox state") + + def provision(self) -> None: + shutil.copyfile(self.artifacts.scratch_template, self.scratch) + options = ( + *self.launch_options(self.scratch), + "--hostname", + HOSTNAME, + "--memory-max", + str(MEMORY_MAX), + "--pids-max", + str(PIDS_MAX), + ) + self.invoke("provision", *options, state=self.state) + self.require_entries(self.state, frozenset({CONFIG}), "after provision") + config = (self.state / CONFIG).read_bytes() + self.invoke( + "provision", + *options, + state=self.state, + expected=1, + diagnostic=b"sandbox is already provisioned", + ) + if (self.state / CONFIG).read_bytes() != config: + raise RuntimeError("a repeated provision changed the configuration") + for operation in ("exec", "stop"): + self.invoke( + operation, + state=self.state, + expected=1, + diagnostic=b"sandbox is not running", + ) + self.require_entries( + self.state, frozenset({CONFIG}), f"after {operation} before start" + ) + + def start(self, *, description: str) -> dict[str, Any]: + self.invoke("start", state=self.state) + self.require_entries(self.state, RUNNING_ENTRIES, f"after the {description}") + runtime = read_runtime(self.state) + self.save_evidence( + "runtime.json", (json.dumps(runtime, indent=2) + "\n").encode() + ) + if not sandbox_lifecycle.runtime_process_running(runtime): + raise RuntimeError(f"OpenVMM is not running after the {description}") + if not control_endpoint_present(runtime): + raise RuntimeError( + f"the control endpoint is absent after the {description}" + ) + self.require_processes((int(runtime["pid"]),), f"after the {description}") + return runtime + + def check_running_transitions(self, runtime: dict[str, Any]) -> None: + def capability_digest() -> bytes: + return hashlib.sha256((self.state / CAPABILITY).read_bytes()).digest() + + record = (self.state / RUNTIME).read_bytes() + capability = capability_digest() + self.invoke( + "start", + state=self.state, + expected=1, + diagnostic=b"sandbox is already running or has stale runtime state", + ) + if (self.state / RUNTIME).read_bytes() != record: + raise RuntimeError("a repeated start changed the runtime record") + if capability_digest() != capability: + raise RuntimeError("a repeated start changed the control capability") + self.require_entries(self.state, RUNNING_ENTRIES, "after a repeated start") + self.require_processes((int(runtime["pid"]),), "after a repeated start") + + def check_warm_state(self) -> None: + # Managed arguments keep their whitespace, and a later request sees the + # file that an earlier one wrote. + result = self.exec( + "/bin/sh", + "--arg=-c", + '--arg=printf \'%s\\n\' "$@" >"$0"', + f"--arg={STATE_PATH}", + *(f"--arg={argument}" for argument in STATE_ARGUMENTS), + expected=None, + ) + self.require_output(result, b"", "the state-writing request") + result = self.exec("/bin/cat", f"--arg={STATE_PATH}", expected=None) + self.require_output(result, STATE_CONTENT, "the state-reading request") + + def check_security_profile(self) -> None: + # The earlier requests already entered their own namespaces, so the + # probe's check that each container mount appears once also shows + # that they did not mount into the agent's namespace. + result = self.exec( + PROBE, "--arg=limits", f"--arg={MEMORY_MAX}", f"--arg={PIDS_MAX}" + ) + self.save_evidence("probe.log", result.stdout + result.stderr) + for marker in ( + b"NVX-UBUNTU-SANDBOX-PROFILE-OK uid=65534 gid=65534", + f"NVX-UBUNTU-SANDBOX-LIMITS-OK memory_max={MEMORY_MAX} " + f"pids_max={PIDS_MAX}".encode(), + b"NVX-UBUNTU-SANDBOX-OK uid=65534 gid=65534", + ): + if marker not in result.stdout: + raise RuntimeError( + f"the sandbox security probe did not report {marker.decode()}" + ) + + def check_bounded_results(self) -> None: + # The agent stops forwarding output at its bound, and the outcome + # report carries only the operation, category, and status. + report = self.root / "output-limit-outcome.json" + result = self.exec( + "/usr/bin/head", + "--arg=-c", + f"--arg={2 * OUTPUT_LIMIT_BYTES}", + "--arg=/dev/zero", + "--outcome-report", + str(report), + expected=125, + ) + if ( + not OUTPUT_LIMIT_BYTES - OUTPUT_CHUNK_BYTES + < len(result.stdout) + <= OUTPUT_LIMIT_BYTES + or result.stdout.strip(b"\0") + or result.stderr + ): + raise RuntimeError( + "the output limit forwarded " + f"{len(result.stdout)} stdout and {len(result.stderr)} stderr bytes" + ) + read_exec_outcome(report, category="output-limit", status_code=125) + self.copy_evidence(report, "output-limit-outcome.json") + + # An existing report path fails the request before the workload runs. + existing = self.root / "existing-outcome.json" + existing.write_text(FOREIGN_CONTENT, encoding="utf-8") + self.exec( + "/bin/touch", + f"--arg={REFUSED_MARKER}", + "--outcome-report", + str(existing), + expected=1, + diagnostic=b"outcome report already exists", + ) + if existing.read_text(encoding="utf-8") != FOREIGN_CONTENT: + raise RuntimeError("a refused request overwrote an existing report") + result = self.exec( + "/bin/sh", "--arg=-c", f"--arg=test ! -e {REFUSED_MARKER}", expected=None + ) + self.require_output(result, b"", "the refused request's marker check") + + def check_deprovision_while_running(self, runtime: dict[str, Any]) -> None: + record = (self.state / RUNTIME).read_bytes() + self.invoke( + "deprovision", + state=self.state, + expected=1, + diagnostic=b"sandbox must be stopped before deprovision", + ) + self.require_entries(self.state, RUNNING_ENTRIES, "after deprovision refused") + if (self.state / RUNTIME).read_bytes() != record: + raise RuntimeError("a refused deprovision changed the runtime record") + self.require_processes((int(runtime["pid"]),), "after deprovision refused") + result = self.exec("/bin/cat", f"--arg={STATE_PATH}", expected=None) + self.require_output( + result, STATE_CONTENT, "the state after deprovision refused" + ) + + def stop(self, runtime: dict[str, Any], layer_digest: str) -> None: + self.invoke("stop", state=self.state) + self.require_entries(self.state, STOPPED_ENTRIES, "after stop") + if sandbox_lifecycle.runtime_process_running(runtime): + raise RuntimeError("OpenVMM still runs after stop") + if control_endpoint_present(runtime): + raise RuntimeError("stop left the control endpoint behind") + self.require_processes((), "after stop") + self.copy_evidence(self.state / OUTCOME, "outcome.json") + report: object = json.loads((self.state / OUTCOME).read_text(encoding="utf-8")) + outcome = ( + cast(dict[str, object], report).get("outcome") + if isinstance(report, dict) + else None + ) + if outcome != MANAGED_SUCCESS: + raise RuntimeError(f"the managed outcome is {outcome!r}") + # A clean journal shows that stop unmounted the overlay and scratch + # before the VM powered off. + if not scratch_journal_clean(self.scratch): + raise RuntimeError("stop left the scratch file system mounted") + if sha256_file(self.artifacts.distro) != layer_digest: + raise RuntimeError("the running sandbox modified the EROFS layer") + for operation in ("exec", "stop"): + self.invoke( + operation, + state=self.state, + expected=1, + diagnostic=b"sandbox is not running", + ) + self.require_entries( + self.state, STOPPED_ENTRIES, f"after {operation} after stop" + ) + + def check_crash(self, runtime: dict[str, Any]) -> None: + # OpenVMM exits without a stop, so its runtime record goes stale. + kill_openvmm(runtime, self.timeout) + self.require_processes((), "after OpenVMM was killed") + if sys.platform == "win32" and control_endpoint_present(runtime): + raise RuntimeError("the control pipe outlived OpenVMM") + stale = b"sandbox runtime state is stale because the OpenVMM process" + for operation, diagnostic in ( + ("exec", stale), + ("stop", stale), + ("start", b"sandbox is already running or has stale runtime state"), + ): + self.invoke(operation, state=self.state, expected=1, diagnostic=diagnostic) + self.require_entries( + self.state, RUNNING_ENTRIES, f"after {operation} with stale state" + ) + self.require_processes((), "after the stale-state requests") + + def check_deprovision(self) -> None: + self.copy_evidence(self.state / LOG, "openvmm.log") + foreign = self.state / FOREIGN_NAME + foreign.write_text(FOREIGN_CONTENT, encoding="utf-8") + self.invoke( + "deprovision", + state=self.state, + expected=1, + diagnostic=( + b"sandbox state directory contains files not owned by NVX: " + + FOREIGN_NAME.encode() + ), + ) + self.require_entries( + self.state, frozenset({FOREIGN_NAME}), "after deprovision kept foreign data" + ) + if foreign.read_text(encoding="utf-8") != FOREIGN_CONTENT: + raise RuntimeError("deprovision changed a file that NVX does not own") + foreign.unlink() + self.invoke("deprovision", state=self.state) + if self.state.exists(): + raise RuntimeError("deprovision kept the sandbox state directory") + self.invoke( + "exec", + "--entrypoint", + "/bin/true", + state=self.state, + expected=1, + diagnostic=b"sandbox state path is not a plain directory", + ) + if self.state.exists(): + raise RuntimeError("exec recreated a deprovisioned sandbox's state") + + def clean_up(self) -> list[Exception]: + """Stops and deprovisions what a failed acceptance left behind.""" + errors: list[Exception] = [] + states = [ + (state, label) + for state, label in ((self.state, ""), (self.identity_state, "identity-")) + if state.is_dir() + ] + for state, _ in states: + try: + if (state / RUNTIME).is_file(): + runtime = read_runtime(state) + self.run_cli("stop", state=state) + kill_openvmm(runtime, self.timeout) + except Exception as error: + errors.append(error) + # Every OpenVMM process whose arguments name the fixture is the + # acceptance's own, even one that no runtime record names. + try: + for pid in openvmm_processes(self.root.name): + with contextlib.suppress(OSError): + terminate_process(pid) + deadline = time.monotonic() + self.timeout + while strays := openvmm_processes(self.root.name): + if time.monotonic() >= deadline: + raise RuntimeError(f"OpenVMM processes {list(strays)} survived") + time.sleep(0.05) + except Exception as error: + errors.append(error) + for state, label in states: + try: + self.copy_evidence(state / LOG, f"{label}openvmm.log") + (state / FOREIGN_NAME).unlink(missing_ok=True) + result = self.run_cli("deprovision", state=state) + if result.returncode != 0: + raise RuntimeError( + f"cannot deprovision {state}: " + f"{bounded_text(result.stderr).strip()}" + ) + except Exception as error: + errors.append(error) + return errors + + +def run_sandbox_lifecycle(backend: str, *, timeout: float, output_dir: Path) -> None: + artifacts = load_workload_artifacts() + output_dir.mkdir(parents=True, exist_ok=True) + root = Path(tempfile.mkdtemp(prefix="nvx-public-lifecycle-")) + acceptance = LifecycleAcceptance( + backend, + timeout=timeout, + output_dir=output_dir, + artifacts=artifacts, + root=root, + ) + try: + acceptance.run() + except Exception as error: + errors = acceptance.clean_up() + try: + acceptance.write_checks() + except Exception as evidence_error: + errors.append(evidence_error) + if acceptance.state.exists() or acceptance.identity_state.exists(): + preserved = bounded_text(str(root).encode("utf-8", errors="replace")) + errors.append(RuntimeError(f"fixture preserved for recovery: {preserved}")) + else: + shutil.rmtree(root, ignore_errors=True) + if errors: + raise RuntimeError(f"{error}; cleanup: {format_errors(errors)}") from error + raise + acceptance.write_checks() + shutil.rmtree(root) diff --git a/scripts/test_microvm_tests.py b/scripts/test_microvm_tests.py index ccb8d2b84..d2eeab099 100644 --- a/scripts/test_microvm_tests.py +++ b/scripts/test_microvm_tests.py @@ -14,6 +14,7 @@ import subprocess import sys import tempfile +import time import unittest from collections.abc import Callable from contextlib import ExitStack, redirect_stdout @@ -30,6 +31,8 @@ managed_exec_tests, microvm_tests, openvmm_process, + sandbox_lifecycle, + sandbox_lifecycle_tests, time_abi, ) from nvx_tools.build_constants import ( # noqa: E402 @@ -775,6 +778,627 @@ def test_failed_stop_attempts_guarded_deprovision(self): self.assertFalse(fixture_root.exists()) +_LIFECYCLE_STATE_SCRIPT = 'printf \'%s\\n\' "$@" >"$0"' +_CliResult = tuple[int, bytes, bytes] + + +class _FakeSandboxCli: + """Emulates `nvx.py sandbox` and the OpenVMM processes that it starts. + + Each fault names one regression of the public lifecycle that the + acceptance must detect. + """ + + def __init__(self, *faults: str) -> None: + self.faults = set(faults) + self.running: dict[int, bool] = {} + self.endpoints: dict[int, str] = {} + self.guest_files: dict[Path, dict[str, bytes]] = {} + self.operations: list[str] = [] + self.capabilities: list[bytes] = [] + self.probed = False + self.next_pid = 4000 + + def __call__( + self, command: list[str], **_: object + ) -> subprocess.CompletedProcess[bytes]: + operation = command[3] + self.operations.append(operation) + values: dict[str, str] = {} + arguments: list[str] = [] + index = 4 + while index < len(command): + if command[index].startswith("--arg="): + arguments.append(command[index].removeprefix("--arg=")) + index += 1 + else: + values[command[index]] = command[index + 1] + index += 2 + state = Path(values["--state-dir"]) if "--state-dir" in values else None + handlers: dict[ + str, + Callable[[Path | None, dict[str, str], list[str]], _CliResult], + ] = { + "run": self._run, + "provision": self._provision, + "start": self._start, + "exec": self._exec, + "stop": self._stop, + "deprovision": self._deprovision, + } + returncode, stdout, stderr = handlers[operation](state, values, arguments) + return subprocess.CompletedProcess(command, returncode, stdout, stderr) + + @staticmethod + def _error(message: str) -> _CliResult: + return 1, b"", f"error: {message}\n".encode() + + def _missing(self, state: Path) -> _CliResult | None: + if not state.is_dir(): + return self._error(f"sandbox state path is not a plain directory: {state}") + return None + + def _running(self, state: Path) -> tuple[dict[str, object], _CliResult | None]: + refusal = self._missing(state) + if refusal is not None: + return {}, refusal + if not (state / "runtime.json").is_file(): + return {}, self._error("sandbox is not running") + runtime = json.loads((state / "runtime.json").read_text(encoding="utf-8")) + if not self.running.get(runtime["pid"], False): + return {}, self._error( + "sandbox runtime state is stale because the OpenVMM process is " + "not running" + ) + return runtime, None + + def _run( + self, state: Path | None, values: dict[str, str], _: list[str] + ) -> _CliResult: + assert state is None + assert values["--workload-user"] == sandbox_lifecycle_tests.UNAVAILABLE_IDENTITY + if "run-starts-workload" in self.faults: + return 0, b"NVX-UBUNTU-SANDBOX-PROFILE-OK uid=65534 gid=65534\n", b"" + return ( + 125, + b">> openvmm\nNVX-SANDBOX-ERROR: configured workload UID is unavailable\n", + b"", + ) + + def _provision( + self, state: Path | None, values: dict[str, str], _: list[str] + ) -> _CliResult: + assert state is not None + if values.get("--workload-user") == "0:0": + return ( + 2, + b"", + b"nvx.py sandbox: error: argument --workload-user: " + b"--workload-user UID must be between 1 and 4294967295\n", + ) + if (state / "config.json").exists(): + return self._error("sandbox is already provisioned") + state.mkdir(parents=True, exist_ok=True) + (state / "config.json").write_text( + json.dumps({"user": values.get("--workload-user", "65534:65534")}), + encoding="utf-8", + ) + return 0, b"", b"" + + def _start( + self, state: Path | None, _: dict[str, str], __: list[str] + ) -> _CliResult: + assert state is not None + refusal = self._missing(state) + if refusal is not None: + return refusal + if (state / "runtime.json").exists() and "duplicate-start" not in self.faults: + return self._error("sandbox is already running or has stale runtime state") + (state / "outcome.json").unlink(missing_ok=True) + with (state / "openvmm.log").open("a", encoding="utf-8", newline="\n") as log: + if "noisy-log" in self.faults: + log.write("noise\n" * 40_000) + log.write("OpenVMM started\n") + config = json.loads((state / "config.json").read_text(encoding="utf-8")) + if config["user"] != "65534:65534": + # The guest refuses the identity and powers off. + (state / "outcome.json").write_text("{}", encoding="utf-8") + if "failed-start-keeps-runtime" in self.faults: + (state / "runtime.json").write_text("{}", encoding="utf-8") + return self._error("managed control endpoint closed") + pid = self.next_pid + self.next_pid += 1 + self.running[pid] = True + endpoint = ( + f"//./pipe/openvmm-microvm-{pid}" + if os.name == "nt" + else str(state / "control.sock") + ) + self.endpoints[pid] = endpoint + if os.name != "nt": + (state / "control.sock").touch() + capability = os.urandom(32) + self.capabilities.append(capability) + (state / "control.capability").write_bytes(capability) + (state / "runtime.json").write_text( + json.dumps( + { + "format": 1, + "pid": pid, + "start_time": pid * 7, + "control_endpoint": endpoint, + } + ), + encoding="utf-8", + ) + if "restart-loses-state" in self.faults: + self.guest_files.pop(state, None) + return 0, b"", b"" + + def _exec( + self, state: Path | None, values: dict[str, str], arguments: list[str] + ) -> _CliResult: + assert state is not None + _, refusal = self._running(state) + if refusal is not None: + return refusal + report = ( + Path(values["--outcome-report"]) if "--outcome-report" in values else None + ) + if ( + report is not None + and report.exists() + and "overwrite-report" not in self.faults + ): + return self._error(f"outcome report already exists: {report}") + files = self.guest_files.setdefault(state, {}) + entrypoint = values["--entrypoint"] + returncode, category, stdout, stderr = 0, "exit", b"", b"" + if entrypoint == "/bin/sh" and arguments[:2] == ["-c", _LIFECYCLE_STATE_SCRIPT]: + if "exec-loses-state" not in self.faults: + files[arguments[2]] = "".join( + f"{argument}\n" for argument in arguments[3:] + ).encode() + elif entrypoint == "/bin/sh" and arguments[1].startswith("test ! -e "): + returncode = int(arguments[1].removeprefix("test ! -e ") in files) + elif entrypoint == "/bin/cat": + if arguments[0] in files: + stdout = files[arguments[0]] + else: + returncode, stderr = 1, b"cat: No such file or directory\n" + elif entrypoint == "/bin/touch": + files[arguments[0]] = b"" + elif entrypoint == sandbox_lifecycle_tests.PROBE: + self.probed = True + if "probe-fails" in self.faults: + returncode = 1 + stderr = b"NVX-UBUNTU-SANDBOX-FAIL: the workload shares a namespace\n" + else: + stdout = ( + b"NVX-UBUNTU-SANDBOX-PROFILE-OK uid=65534 gid=65534\n" + + f"NVX-UBUNTU-SANDBOX-LIMITS-OK memory_max={arguments[1]} " + f"pids_max={arguments[2]}\n".encode() + + b"NVX-UBUNTU-SANDBOX-OK uid=65534 gid=65534\n" + ) + elif entrypoint == "/usr/bin/head": + size = sandbox_lifecycle_tests.OUTPUT_LIMIT_BYTES + if "unbounded-output" in self.faults: + size = int(arguments[1]) + elif "short-output" in self.faults: + size -= sandbox_lifecycle_tests.OUTPUT_CHUNK_BYTES + returncode, category, stdout = 125, "output-limit", bytes(size) + elif entrypoint != "/bin/true": + raise AssertionError(f"unexpected workload {entrypoint}") + if report is not None: + report.write_text( + json.dumps( + { + "schema_version": 1, + "operation_id": "a" * 32, + "outcome": { + "operation": "exec", + "category": category, + "status_code": returncode, + }, + } + ), + encoding="utf-8", + ) + return returncode, stdout, stderr + + def _stop(self, state: Path | None, _: dict[str, str], __: list[str]) -> _CliResult: + assert state is not None + runtime, refusal = self._running(state) + if refusal is not None: + return refusal + pid = cast(int, runtime["pid"]) + if "stop-leaks-process" not in self.faults: + self.terminate(pid) + (state / "runtime.json").unlink() + if "stop-leaks-capability" not in self.faults: + (state / "control.capability").unlink() + (state / "control.sock").unlink(missing_ok=True) + (state / "outcome.json").write_text( + json.dumps( + { + "schema_version": 1, + "outcome": sandbox_lifecycle_tests.MANAGED_SUCCESS, + "network_policy": {}, + "teardown": {}, + } + ), + encoding="utf-8", + ) + return 0, b"", b"" + + def _deprovision( + self, state: Path | None, _: dict[str, str], __: list[str] + ) -> _CliResult: + assert state is not None + refusal = self._missing(state) + if refusal is not None: + return refusal + runtime_path = state / "runtime.json" + if runtime_path.is_file(): + runtime = json.loads(runtime_path.read_text(encoding="utf-8")) + if self.running.get(runtime.get("pid"), False): + return self._error("sandbox must be stopped before deprovision") + if "cleanup-deprovision-fails" in self.faults and self.probed: + return self._error("injected deprovision failure") + for name in ( + "runtime.json", + "control.capability", + "control.sock", + "outcome.json", + "openvmm.log", + "config.json", + ): + (state / name).unlink(missing_ok=True) + unknown = sorted(path.name for path in state.iterdir()) + if unknown and "deprovision-removes-foreign" not in self.faults: + return self._error( + "sandbox state directory contains files not owned by NVX: " + + ", ".join(unknown) + ) + shutil.rmtree(state) + self.guest_files.pop(state, None) + return 0, b"", b"" + + def processes(self, _: str) -> tuple[int, ...]: + return tuple(sorted(pid for pid, running in self.running.items() if running)) + + def process_running(self, runtime: dict[str, object]) -> bool: + return self.running.get(cast(int, runtime["pid"]), False) + + def endpoint_present(self, runtime: dict[str, object]) -> bool: + return runtime["control_endpoint"] in self.endpoints.values() + + def terminate(self, pid: int) -> None: + self.running[pid] = False + self.endpoints.pop(pid, None) + + +class PublicSandboxLifecycleTests(unittest.TestCase): + EXPECTED_OPERATIONS = [ + # Identity admission. + "provision", + "run", + "provision", + "start", + "deprovision", + # Every operation fails before provision. + "start", + "exec", + "stop", + "deprovision", + # Provision, and the operations that need a running sandbox. + "provision", + "provision", + "exec", + "stop", + # A running sandbox. + "start", + "start", + "exec", + "exec", + "exec", + "exec", + "exec", + "exec", + "deprovision", + "exec", + # Stop, and the operations that need a running sandbox. + "stop", + "exec", + "stop", + # Restart, crash, and stale runtime state. + "start", + "exec", + "exec", + "stop", + "start", + # Deprovision. + "deprovision", + "deprovision", + "exec", + ] + + def _run(self, root: Path, fake: _FakeSandboxCli) -> Path: + distro = root / "ubuntu-distro.erofs" + distro.write_bytes(b"distro") + template = root / "ubuntu-smoke-scratch.ext4" + template.write_bytes(bytes(4096)) + artifacts = managed_exec_tests.WorkloadArtifacts( + distro, "12345678-1234-1234-1234-123456789abc", template + ) + output_dir = root / "results" + mkdtemp = tempfile.mkdtemp + + def fixture(prefix: str) -> str: + return mkdtemp(prefix=prefix, dir=root) + + with ( + patch.object( + sandbox_lifecycle_tests, + "load_workload_artifacts", + return_value=artifacts, + ), + patch.object( + sandbox_lifecycle_tests.tempfile, "mkdtemp", side_effect=fixture + ), + patch.object(sandbox_lifecycle_tests.subprocess, "run", side_effect=fake), + patch.object( + sandbox_lifecycle_tests, + "openvmm_processes", + side_effect=fake.processes, + ), + patch.object( + sandbox_lifecycle_tests, + "control_endpoint_present", + side_effect=fake.endpoint_present, + ), + patch.object( + sandbox_lifecycle_tests, + "terminate_process", + side_effect=fake.terminate, + ), + patch.object( + sandbox_lifecycle, + "runtime_process_running", + side_effect=fake.process_running, + ), + ): + sandbox_lifecycle_tests.run_sandbox_lifecycle( + "whp", timeout=2, output_dir=output_dir + ) + return output_dir + + def test_acceptance_drives_the_complete_public_lifecycle(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + fake = _FakeSandboxCli() + output_dir = self._run(root, fake) + checks = json.loads( + (output_dir / "sandbox-lifecycle-checks.json").read_text( + encoding="utf-8" + ) + ) + evidence = {path.name: path.read_bytes() for path in output_dir.iterdir()} + fixtures = list(root.glob("nvx-public-lifecycle-*")) + + self.assertEqual(fake.operations, self.EXPECTED_OPERATIONS) + self.assertEqual(fixtures, []) + self.assertFalse(any(fake.running.values())) + self.assertEqual( + set(evidence), + { + "sandbox-lifecycle-checks.json", + "sandbox-lifecycle-identity-run.log", + "sandbox-lifecycle-identity-openvmm.log", + "sandbox-lifecycle-runtime.json", + "sandbox-lifecycle-probe.log", + "sandbox-lifecycle-output-limit-outcome.json", + "sandbox-lifecycle-outcome.json", + "sandbox-lifecycle-openvmm.log", + }, + ) + self.assertEqual( + [record["operation"] for record in checks], self.EXPECTED_OPERATIONS + ) + for record in checks: + self.assertEqual( + set(record), + { + "operation", + "argv", + "expected_returncode", + "returncode", + "stdout_bytes", + "stderr_bytes", + "state_entries", + }, + ) + # Managed arguments keep their whitespace. + self.assertTrue( + any( + [ + f"--arg={argument}" + for argument in sandbox_lifecycle_tests.STATE_ARGUMENTS + ] + == record["argv"][-3:] + for record in checks + ) + ) + # The bounded reports hold neither workload output nor the capability. + self.assertNotIn( + b"NVX-UBUNTU-SANDBOX", evidence["sandbox-lifecycle-checks.json"] + ) + for capability in fake.capabilities: + for content in evidence.values(): + self.assertNotIn(capability, content) + self.assertNotIn(capability.hex().encode(), content) + self.assertIn( + b"NVX-UBUNTU-SANDBOX-LIMITS-OK", evidence["sandbox-lifecycle-probe.log"] + ) + + def test_acceptance_detects_lifecycle_regressions(self): + regressions = { + "run-starts-workload": "run returned 0, expected 125", + "failed-start-keeps-runtime": "a failed start left lifecycle state behind", + "duplicate-start": "start returned 0, expected 1", + "exec-loses-state": "the state-reading request returned status 1", + "probe-fails": "exec returned 1, expected 0", + "unbounded-output": "the output limit forwarded 2097152 stdout", + "short-output": "the output limit forwarded 1015808 stdout", + "overwrite-report": "exec returned 0, expected 1", + "stop-leaks-capability": "after stop: the state directory holds", + "stop-leaks-process": "OpenVMM still runs after stop", + "restart-loses-state": "the restarted sandbox's state returned status 1", + "deprovision-removes-foreign": "deprovision returned 0, expected 1", + } + for fault, message in regressions.items(): + with ( + self.subTest(fault=fault), + tempfile.TemporaryDirectory() as temporary, + ): + root = Path(temporary) + fake = _FakeSandboxCli(fault) + with self.assertRaisesRegex(RuntimeError, re.escape(message)): + self._run(root, fake) + # Cleanup ends every OpenVMM process and removes the fixture. + self.assertFalse(any(fake.running.values())) + self.assertEqual(list(root.glob("nvx-public-lifecycle-*")), []) + self.assertTrue( + (root / "results" / "sandbox-lifecycle-checks.json").is_file() + ) + + def test_evidence_keeps_the_end_of_long_logs(self): + with tempfile.TemporaryDirectory() as temporary: + output_dir = self._run(Path(temporary), _FakeSandboxCli("noisy-log")) + log = (output_dir / "sandbox-lifecycle-openvmm.log").read_bytes() + + limit = sandbox_lifecycle_tests.EVIDENCE_LIMIT + header, separator, tail = log.partition(b"\n") + self.assertRegex(header, rb"^\.\.\. \(\d+ earlier bytes omitted\)$") + self.assertEqual(separator, b"\n") + self.assertEqual(len(tail), limit) + self.assertTrue(tail.endswith(b"noise\nOpenVMM started\n")) + + def test_bounded_evidence_keeps_the_end_of_its_data(self): + bounded = sandbox_lifecycle_tests.bounded_evidence + limit = sandbox_lifecycle_tests.EVIDENCE_LIMIT + self.assertEqual(bounded(b"short"), b"short") + self.assertEqual( + bounded(b"a" * 10 + b"b" * limit), + b"... (10 earlier bytes omitted)\n" + b"b" * limit, + ) + self.assertEqual(bounded(b"tail", 7), b"... (7 earlier bytes omitted)\ntail") + + def test_failed_cleanup_preserves_the_fixture(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + fake = _FakeSandboxCli("probe-fails", "cleanup-deprovision-fails") + with self.assertRaisesRegex( + RuntimeError, + "(?s)exec returned 1, expected 0.*fixture preserved for recovery", + ) as raised: + self._run(root, fake) + fixtures = list(root.glob("nvx-public-lifecycle-*")) + + self.assertEqual(len(fixtures), 1) + self.assertIn(str(fixtures[0]), str(raised.exception)) + self.assertTrue((fixtures[0] / "state" / "config.json").is_file()) + self.assertFalse(any(fake.running.values())) + self.assertTrue( + (root / "results" / "sandbox-lifecycle-openvmm.log").is_file() + ) + + def test_scratch_journal_check_reads_the_ext4_recovery_flag(self): + with tempfile.TemporaryDirectory() as temporary: + image = Path(temporary) / "scratch.ext4" + superblock = bytearray(2048) + image.write_bytes(superblock) + self.assertTrue(sandbox_lifecycle_tests.scratch_journal_clean(image)) + superblock[1024 + 0x60] = sandbox_lifecycle_tests.EXT4_INCOMPAT_RECOVER + image.write_bytes(superblock) + self.assertFalse(sandbox_lifecycle_tests.scratch_journal_clean(image)) + image.write_bytes(bytes(1030)) + with self.assertRaisesRegex(RuntimeError, "truncated"): + sandbox_lifecycle_tests.scratch_journal_clean(image) + + def test_kill_ends_the_recorded_process(self): + if os.name != "nt" and sys.platform != "linux": + self.skipTest("process identity requires a Linux or Windows host") + child = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(60)"]) + try: + runtime: dict[str, object] = { + "format": 1, + "pid": child.pid, + "start_time": sandbox_lifecycle._process_start_time(child.pid), + "control_endpoint": "unused", + } + self.assertTrue(sandbox_lifecycle.runtime_process_running(runtime)) + sandbox_lifecycle_tests.kill_openvmm(runtime, 30) + self.assertFalse(sandbox_lifecycle.runtime_process_running(runtime)) + self.assertNotEqual(child.wait(timeout=30), 0) + finally: + if child.poll() is None: + child.kill() + child.wait(timeout=30) + + def test_control_endpoint_presence_follows_the_endpoint(self): + if sys.platform == "win32": + from multiprocessing.connection import Listener + + name = f"openvmm-microvm-test-{os.getpid()}" + runtime: dict[str, object] = {"control_endpoint": f"//./pipe/{name}"} + listener = Listener(rf"\\.\pipe\{name}", family="AF_PIPE") + try: + self.assertTrue( + sandbox_lifecycle_tests.control_endpoint_present(runtime) + ) + finally: + listener.close() + self.assertFalse(sandbox_lifecycle_tests.control_endpoint_present(runtime)) + return + with tempfile.TemporaryDirectory() as temporary: + endpoint = Path(temporary) / "control.sock" + runtime = {"control_endpoint": str(endpoint)} + self.assertFalse(sandbox_lifecycle_tests.control_endpoint_present(runtime)) + endpoint.touch() + self.assertTrue(sandbox_lifecycle_tests.control_endpoint_present(runtime)) + + @unittest.skipUnless(sys.platform == "linux", "scans the Linux process table") + def test_process_scan_finds_openvmm_processes_naming_the_fixture(self): + shell = shutil.which("sh") + assert shell is not None + with tempfile.TemporaryDirectory() as temporary: + openvmm = Path(temporary) / "openvmm" + shutil.copyfile(shell, openvmm) + openvmm.chmod(0o755) + # The shell blocks in its read builtin, so it starts no other + # process and keeps its arguments. + child = subprocess.Popen( + [str(openvmm), "-c", "read -r line", "nvx-public-lifecycle-scan"], + stdin=subprocess.PIPE, + ) + try: + deadline = time.monotonic() + 30 + while child.pid not in sandbox_lifecycle_tests.openvmm_processes( + "nvx-public-lifecycle-scan" + ): + self.assertLess(time.monotonic(), deadline) + time.sleep(0.05) + self.assertEqual( + sandbox_lifecycle_tests.openvmm_processes("nvx-other-fixture"), () + ) + finally: + child.kill() + child.wait(timeout=30) + if child.stdin is not None: + child.stdin.close() + + class GuestIdentityScriptTests(unittest.TestCase): def test_guest_identity_checks_fail_before_success_markers(self): descriptor = microvm_tests.guest_descriptor("ubuntu") @@ -4994,6 +5618,67 @@ def require(path: Path, _description: str) -> Path: "whp", timeout=args.timeout, output_dir=output_dir ) + def test_runner_dispatches_public_sandbox_lifecycle(self): + with tempfile.TemporaryDirectory() as temporary: + output_dir = Path(temporary) + args = nvx.parse_args( + [ + "test-microvm", + "--backend", + "mshv", + "--scenario", + "sandbox-lifecycle", + "--output-dir", + str(output_dir), + ] + ) + + def require(path: Path, _description: str) -> Path: + return path + + with ( + patch.object(microvm_tests, "validate_openvmm_test_backend"), + patch.object(microvm_tests, "require_file", side_effect=require), + patch.object(microvm_tests, "run_sandbox_lifecycle") as run, + patch.object( + microvm_tests, "run_managed_exec_configuration" + ) as managed_exec, + ): + self.assertEqual(microvm_tests.run(args), 0) + run.assert_called_once_with( + "mshv", timeout=args.timeout, output_dir=output_dir + ) + managed_exec.assert_not_called() + + def test_runner_runs_public_sandbox_scenarios_only_when_named(self): + def require(path: Path, _description: str) -> Path: + return path + + with tempfile.TemporaryDirectory() as temporary: + args = nvx.parse_args( + ["test-microvm", "--backend", "kvm", "--output-dir", temporary] + ) + with ( + patch.object(microvm_tests, "validate_openvmm_test_backend"), + patch.object( + microvm_tests, + "MICROVM_TEST_SCENARIOS", + ("managed-exec-config", "sandbox-lifecycle", "guest-boot"), + ), + patch.object(microvm_tests, "require_file", side_effect=require), + patch.object(microvm_tests, "run_guest_boot") as guest_boot, + patch.object(microvm_tests, "run_sandbox_lifecycle") as lifecycle, + patch.object( + microvm_tests, "run_managed_exec_configuration" + ) as managed_exec, + ): + self.assertEqual(microvm_tests.run(args), 0) + + guest_boot.assert_called_once() + lifecycle.assert_not_called() + managed_exec.assert_not_called() + self.assertIn("sandbox-lifecycle", microvm_tests.SANDBOX_CONTROL_SCENARIOS) + def test_managed_container_launch_prepares_identity_and_static_helper(self): root = Path(__file__).resolve().parent.parent bootstrap = (root / "guest" / "common" / "nvx-init-agent").read_text() diff --git a/scripts/test_nvx_tools.py b/scripts/test_nvx_tools.py index a374b7693..665fc2049 100644 --- a/scripts/test_nvx_tools.py +++ b/scripts/test_nvx_tools.py @@ -4974,6 +4974,74 @@ def test_windows_setup_selects_largest_data_volume_for_benchmark_scratch(self): ], ) + def test_cli_validation_runs_every_script_test_module(self): + # Each scripts/test_*.py module must run in both CLI validation jobs; + # test_control_session.py once ran in neither (#229). + action = ( + BuildConstants.REPO_ROOT + / ".github" + / "actions" + / "validate-nvx" + / "action.yml" + ).read_text(encoding="utf-8") + modules = { + path.name + for path in (BuildConstants.REPO_ROOT / "scripts").glob("test_*.py") + } + self.assertIn("test_control_session.py", modules) + for step in ("Validate NVX CLI on Linux", "Validate NVX CLI on Windows"): + with self.subTest(step=step): + script = _composite_action_script(action, step) + invoked = re.findall(r"(?:^|\s)scripts[/\\](test_\w+\.py)", script) + self.assertEqual(sorted(invoked), sorted(modules)) + + def test_ci_runs_the_public_sandbox_acceptance_on_every_backend(self): + workflows = BuildConstants.REPO_ROOT / ".github" / "workflows" + workflow = (workflows / "ci.yml").read_text(encoding="utf-8") + microvm_workflow = (workflows / "run-nvx-microvm-tests.yml").read_text( + encoding="utf-8" + ) + for backend in ("kvm", "mshv", "whp"): + with self.subTest(backend=backend): + job = _workflow_job(workflow, f"nvx-microvm-tests-{backend}") + self.assertIn( + "uses: ./.github/workflows/run-nvx-microvm-tests.yml", job + ) + self.assertIn(f"backend: {backend}", job) + self.assertNotIn("debug-kernel: true", job) + for system in ("Linux", "Windows"): + with self.subTest(system=system): + acceptance = _workflow_step( + microvm_workflow, + "test", + f"Run public managed sandbox acceptance on {system}", + ) + self.assertIn("!inputs.debug-kernel", acceptance) + self.assertIn("--scenario managed-exec-config", acceptance) + self.assertIn("--scenario sandbox-lifecycle", acceptance) + self.assertIn( + '--output-dir "build/test-results/public-exec-${{ inputs.backend }}"', + acceptance, + ) + smoke = _workflow_step( + microvm_workflow, + "test", + f"Run Ubuntu sandbox layer smoke test on {system}", + ) + for option in ( + "--entrypoint /sbin/nvx-sandbox-smoke", + "--arg limits", + "--arg 33554432", + "--arg 8", + "--memory-max 33554432", + "--pids-max 8", + ): + self.assertIn(option, smoke) + upload = _workflow_step( + microvm_workflow, "test", "Upload NVX microVM failure logs" + ) + self.assertIn("build/test-results/public-exec-${{ inputs.backend }}", upload) + @unittest.skipUnless(os.name == "nt", "requires Windows PowerShell") def test_windows_cli_validation_stops_at_each_failed_command(self): action = ( @@ -9097,6 +9165,340 @@ def test_eperm_share_requires_reads_to_fail_with_eperm_and_writes_to_fail(self): self.assertNotIn("NVX-UBUNTU-SANDBOX-SHARE-OK", result.stdout) +_SANDBOX_MOUNTINFO = ( + "20 1 0:20 / / rw,relatime - overlay overlay rw," + "lowerdir=/run/nvx/layers/distro,upperdir=/run/nvx/scratch/upper," + "workdir=/run/nvx/scratch/work,metacopy=on,xino=on\n" + "21 20 0:21 / /.nvx-agent ro,nosuid,nodev,relatime - tmpfs tmpfs ro,mode=755\n" + "22 20 0:22 / /proc rw,nosuid,nodev,noexec,relatime - proc proc rw\n" + "23 20 0:23 / /sys ro,nosuid,nodev,noexec,relatime - sysfs sysfs ro\n" + "24 20 0:24 / /dev rw,nosuid,relatime - tmpfs tmpfs rw,mode=755\n" + "25 24 0:25 / /dev/pts rw,relatime - devpts devpts rw,mode=620,ptmxmode=666\n" + "26 24 0:26 / /dev/shm rw,nosuid,nodev,relatime - tmpfs tmpfs rw\n" +) +_SANDBOX_STATUS = ( + "Name:\tsh\nUid:\t65534\t65534\t65534\t65534\n" + "Gid:\t65534\t65534\t65534\t65534\nGroups:\t\nNoNewPrivs:\t1\n" + + "".join( + f"{name}:\t0000000000000000\n" + for name in ("CapInh", "CapPrm", "CapEff", "CapBnd", "CapAmb") + ) +) + + +@unittest.skipIf(os.name == "nt", "requires a POSIX shell and process tools") +class SandboxSmokeProfileTests(unittest.TestCase): + SMOKE = Path(__file__).parents[1] / "guest" / "common" / "nvx-sandbox-smoke" + + def setUp(self): + shell = _posix_shell() + if shell is None: + self.skipTest("POSIX shell is unavailable") + self.shell = shell + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) + self.source = self.SMOKE.read_text(encoding="utf-8") + + def _fixture(self, name: str, content: str) -> str: + path = self.root / name + path.write_text(content, encoding="utf-8") + return path.as_posix() + + def _run( + self, + names: tuple[str, ...], + call: str, + replacements: dict[str, str] | None = None, + ) -> subprocess.CompletedProcess[str]: + # The globals before fail() configure the checks. + script = self.source[: self.source.index("\nfail() {\n") + 1] + "".join( + _shell_function(self.source, name) for name in ("fail", *names) + ) + for old, new in (replacements or {}).items(): + script = script.replace(old, new) + return subprocess.run( + [self.shell, "-s"], + input=f"{script}{call}\n", + text=True, + capture_output=True, + check=False, + ) + + def test_status_fields_join_their_values(self): + status = self._fixture("status", _SANDBOX_STATUS) + result = self._run( + ("status_field",), + f"status_field {status} Uid\nstatus_field {status} Groups\n" + f"status_field {status} Missing", + ) + + self.assertEqual(result.returncode, 1, result.stderr) + self.assertEqual(result.stdout, "65534 65534 65534 65534\n\n") + self.assertIn(f"{status} has no Missing field", result.stderr) + + def test_identity_and_capability_checks_read_the_process_status(self): + identity = ( + 'id() { case "$1" in -u | -g) echo 65534 ;; esac; }\nHOME=/nonexistent\n' + ) + mutations = { + "valid": ("", ""), + "saved-uid": ( + "Uid:\t65534\t65534\t65534\t65534", + "Uid:\t65534\t65534\t0\t65534", + ), + "groups": ("Groups:\t", "Groups:\t27 "), + "bounding": ( + "CapBnd:\t0000000000000000", + "CapBnd:\t000001ffffffffff", + ), + "no-new-privs": ("NoNewPrivs:\t1", "NoNewPrivs:\t0"), + } + for mutation, (old, new) in mutations.items(): + with self.subTest(mutation=mutation): + status = self._fixture("status", _SANDBOX_STATUS.replace(old, new)) + result = self._run( + ("status_field", "check_identity", "check_capabilities"), + f"{identity}check_identity\ncheck_capabilities\necho checked", + {"/proc/self/status": status}, + ) + if mutation == "valid": + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout, "checked\n") + else: + self.assertEqual(result.returncode, 1, result.stdout) + self.assertIn("NVX-UBUNTU-SANDBOX-FAIL", result.stderr) + + def test_namespace_check_rejects_the_initial_namespaces(self): + def run( + links: dict[str, str], + *, + init_status: str = _SANDBOX_STATUS, + command_line: str = "console=hvc0 nvx_hostname=nvx-lifecycle", + hostname: str = "nvx-lifecycle", + ) -> subprocess.CompletedProcess[str]: + readlink = ( + 'readlink() { case "$1" in ' + + " ".join( + f"*/{name}) echo '{link}' ;;" for name, link in links.items() + ) + + " esac; }\n" + ) + return self._run( + ("status_field", "check_namespaces"), + f"{readlink}check_namespaces\necho checked", + { + "/proc/1/status": self._fixture("init-status", init_status), + "/proc/cmdline": self._fixture("cmdline", command_line + "\n"), + "/proc/sys/kernel/hostname": self._fixture( + "hostname", hostname + "\n" + ), + }, + ) + + private = { + "mnt": "mnt:[4026532200]", + "pid": "pid:[4026532201]", + "uts": "uts:[4026532202]", + } + result = run(private) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout, "checked\n") + result = run(private, command_line="console=hvc0", hostname="nvx-sandbox") + self.assertEqual(result.returncode, 0, result.stderr) + + for name, initial in ( + ("mnt", "mnt:[4026531832]"), + ("pid", "pid:[4026531836]"), + ("uts", "uts:[4026531838]"), + ): + with self.subTest(namespace=name): + result = run({**private, name: initial}) + self.assertEqual(result.returncode, 1, result.stdout) + self.assertIn(f"shares the initial {name} namespace", result.stderr) + for link in ("pid:[]", "pid:[42a]", "mnt:[42]", "pid:42"): + with self.subTest(link=link): + result = run({**private, "pid": link}) + self.assertEqual(result.returncode, 1, result.stdout) + self.assertIn("unexpected pid namespace link", result.stderr) + result = run( + private, + init_status=_SANDBOX_STATUS.replace( + "Uid:\t65534\t65534\t65534\t65534", "Uid:\t0\t0\t0\t0" + ), + ) + self.assertEqual(result.returncode, 1, result.stdout) + self.assertIn("PID 1 of the workload PID namespace runs as 0", result.stderr) + result = run(private, hostname="nvx-sandbox") + self.assertEqual(result.returncode, 1, result.stdout) + self.assertIn("expected nvx-lifecycle", result.stderr) + + def test_mount_check_requires_private_runtime_mounts_once(self): + functions = ("mount_records", "require_mount", "check_mounts") + mountinfo = self._fixture("mountinfo", _SANDBOX_MOUNTINFO) + result = self._run( + functions, + "check_mounts\necho checked", + {"/proc/self/mountinfo": mountinfo}, + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout, "checked\n") + + proc = "22 20 0:22 / /proc rw,nosuid,nodev,noexec,relatime - proc proc rw\n" + mutations = { + "stacked": ( + proc, + proc + proc.replace("22 20 0:22", "27 22 0:27"), + "several mounts are stacked at /proc", + ), + "propagation": ( + proc, + proc.replace("relatime - proc", "relatime shared:5 - proc"), + "the mount at /proc propagates: shared:5", + ), + "writable-sysfs": ( + "/sys ro,nosuid", + "/sys rw,nosuid", + "/sys is mounted rw,nosuid,nodev,noexec,relatime, without ro", + ), + "upper": ( + "upperdir=/run/nvx/scratch/upper", + "upperdir=/run/nvx/upper", + "upper layer is not on scratch", + ), + "lower": ( + "lowerdir=/run/nvx/layers/distro", + "lowerdir=/srv/distro", + "does not use the image layers", + ), + "missing-shm": ( + "26 24 0:26 / /dev/shm rw,nosuid,nodev,relatime - tmpfs tmpfs rw\n", + "", + "nothing is mounted at /dev/shm", + ), + "host-devices": ( + "- tmpfs tmpfs rw,mode=755\n", + "- devtmpfs devtmpfs rw,mode=755\n", + "/dev is devtmpfs, expected tmpfs", + ), + } + for mutation, (old, new, message) in mutations.items(): + with self.subTest(mutation=mutation): + self.assertIn(old, _SANDBOX_MOUNTINFO) + mountinfo = self._fixture( + "mountinfo", _SANDBOX_MOUNTINFO.replace(old, new, 1) + ) + result = self._run( + functions, "check_mounts", {"/proc/self/mountinfo": mountinfo} + ) + self.assertEqual(result.returncode, 1, result.stdout) + self.assertIn(message, result.stderr) + + def test_cgroup_check_requires_the_container_cgroup(self): + for content, returncode in ( + ("0::/container\n", 0), + ("0::/agent\n", 1), + ("12:pids:/container\n", 1), + ): + with self.subTest(content=content): + cgroup = self._fixture("cgroup", content) + result = self._run( + ("check_cgroup",), "check_cgroup", {"/proc/self/cgroup": cgroup} + ) + self.assertEqual(result.returncode, returncode, result.stderr) + + def test_limits_check_validates_its_arguments(self): + for arguments, message in ( + ("abc 8", "limits requires numeric MEMORY_MAX and PIDS_MAX"), + ("33554432 ''", "limits requires numeric MEMORY_MAX and PIDS_MAX"), + ("33554432 2", "requires PIDS_MAX of at least 3"), + ): + with self.subTest(arguments=arguments): + result = self._run( + ("start_sleepers", "check_limits"), f"check_limits {arguments}" + ) + self.assertEqual(result.returncode, 1, result.stdout) + self.assertIn(message, result.stderr) + + @unittest.skipUnless(sys.platform == "linux", "inspects the Linux process table") + def test_sleeper_subshell_records_and_reaps_its_sleepers(self): + record = self.root / "sleepers" + result = self._run( + ("start_sleepers",), + f'(start_sleepers "{record.as_posix()}" 3)\necho done', + # A distinct duration identifies this test's sleepers. + {"sleep 600": "sleep 611.25"}, + ) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(record.read_text(encoding="utf-8"), "3\n") + for process in Path("/proc").iterdir(): + if process.name.isdigit(): + try: + command = (process / "cmdline").read_bytes() + except OSError: + continue + self.assertNotEqual(command, b"sleep\x00611.25\x00") + + def test_main_body_runs_the_profile_then_limits_and_shares(self): + body = self.source[self.source.index("\ncheck_identity\n") + 1 :] + stubs = "".join( + f'{name}() {{ echo "{name} $*"; }}\n' + for name in ( + "check_identity", + "check_capabilities", + "check_namespaces", + "check_mounts", + "check_devices", + "check_cgroup", + "check_image", + "check_limits", + "check_share", + "check_shares_isolated", + ) + ) + script = ( + self.source[: self.source.index("\nfail() {\n") + 1] + + _shell_function(self.source, "fail") + + stubs + + body + ) + + def run(*arguments: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [self.shell, "-s", "--", *arguments], + input=script, + text=True, + capture_output=True, + check=False, + ) + + result = run("limits", "33554432", "8", "/workspace", "rw", "/tools") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual( + result.stdout.splitlines(), + [ + "check_identity ", + "check_capabilities ", + "check_namespaces ", + "check_mounts ", + "check_devices ", + "check_cgroup ", + "check_image ", + "NVX-UBUNTU-SANDBOX-PROFILE-OK uid=65534 gid=65534", + "check_limits 33554432 8", + "check_share /workspace rw", + "check_share /tools ro", + "check_shares_isolated /workspace /tools", + "NVX-UBUNTU-SANDBOX-OK uid=65534 gid=65534", + ], + ) + result = run("limits", "33554432") + self.assertEqual(result.returncode, 1, result.stdout) + self.assertIn("limits requires MEMORY_MAX and PIDS_MAX", result.stderr) + self.assertNotIn("NVX-UBUNTU-SANDBOX-OK", result.stdout) + + class ManagedAgentCgroupTests(unittest.TestCase): SOURCE = Path(__file__).parents[1] / "guest" / "common" / "nvx-managed-agent.c" @@ -12122,6 +12524,28 @@ def test_process_identity_requires_the_recorded_start_time(self): # Records from earlier NVX versions identify OpenVMM by process ID alone. self.assertTrue(sandbox_lifecycle._process_running(pid, None)) + def test_runtime_records_report_whether_their_process_runs(self): + if os.name != "nt" and sys.platform != "linux": + self.skipTest("process identity requires a Linux or Windows host") + pid = os.getpid() + start_time = sandbox_lifecycle._process_start_time(pid) + assert isinstance(start_time, int) + + self.assertTrue( + sandbox_lifecycle.runtime_process_running( + {"pid": pid, "start_time": start_time} + ) + ) + self.assertFalse( + sandbox_lifecycle.runtime_process_running( + {"pid": pid, "start_time": start_time + 1} + ) + ) + with self.assertRaisesRegex(common.ScriptError, "invalid process ID"): + sandbox_lifecycle.runtime_process_running({"pid": "openvmm"}) + with self.assertRaisesRegex(common.ScriptError, "invalid process start"): + sandbox_lifecycle.runtime_process_running({"pid": pid, "start_time": -1}) + def test_process_identity_counts_an_exited_process_as_stopped(self): if os.name != "nt" and sys.platform != "linux": self.skipTest("process identity requires a Linux or Windows host")