From c1df9b7c98e2c4606005628f52a9a7ea50d7d472 Mon Sep 17 00:00:00 2001 From: Akhil Gupta Date: Sun, 11 Oct 2026 16:08:53 -0700 Subject: [PATCH 1/5] fix: restore initramfs provenance on release install --- scripts/nvx497-release.py | 1787 +++ scripts/nvx497-test.py | 21503 ++++++++++++++++++++++++++++++++++++ 2 files changed, 23290 insertions(+) create mode 100644 scripts/nvx497-release.py create mode 100644 scripts/nvx497-test.py diff --git a/scripts/nvx497-release.py b/scripts/nvx497-release.py new file mode 100644 index 000000000..26095eebe --- /dev/null +++ b/scripts/nvx497-release.py @@ -0,0 +1,1787 @@ +"""Source collection, verification, and release packaging workflows.""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +import shutil +import stat +import subprocess +import sys +import tarfile +import tempfile +import urllib.error +import urllib.parse +import urllib.request +import uuid +import zipfile +from collections.abc import Callable, Sequence +from dataclasses import dataclass +from pathlib import Path, PurePosixPath +from typing import cast + +from .archive import create_reproducible_release_archive, create_reproducible_tar_gz +from .azurelinux import input_sha256 as azurelinux_input_sha256 +from .azurelinux import package_lock_sha256 as azurelinux_package_lock_sha256 +from .build import ( + assert_required_kernel_config, + build_docker_linux_source, + initramfs_provenance_inputs, + kernel_provenance_inputs, +) +from .build_config import DockerBuildConfig +from .build_constants import ( + AlpineBuildConstants, + AzureLinuxBuildConstants, + BuildConstants, + InitramfsBuildConstants, + KernelBuildConstants, + OpenVMMBuildConstants, + ReleaseBuildConstants, + UbuntuBuildConstants, +) +from .collect_alpine_sources import collect_alpine_sources +from .collect_ubuntu_sources import collect_ubuntu_sources +from .common import ( + ScriptError, + VerifiedChecksumInventory, + artifact_path, + credential_safe_opener, + download, + git_output, + openvmm_binary_path, + openvmm_git_state, + require_file, + sha256_file, + verify_sha256_sums, + write_sha256_sums, +) +from .development_release import development_release_tag +from .ubuntu import converter_input_sha256, customization_files, package_lock_sha256 + +GITHUB_API_VERSION = "2022-11-28" +# HEAD often has no release, as with CI's [skip ci] baseline commits. Each +# commit searched costs one GitHub API request. +_RELEASE_SEARCH_COMMITS = 20 + + +@dataclass(frozen=True) +class _ReleaseAsset: + tag: str + name: str + url: str + size: int + + +class _GitHubReleaseQueryError(ScriptError): + def __init__(self, status: int, message: str) -> None: + self.status = status + super().__init__(message) + + +class _ReleaseRestoreError(ScriptError): + """Raised when release publication and restoration both fail.""" + + +def create_release_archive(source: Path, destination: Path) -> None: + if source.is_symlink(): + raise ScriptError(f"release package directory must not be a symlink: {source}") + source = source.resolve() + destination = destination.resolve() + if destination == source or source in destination.parents: + raise ScriptError("release archive destination must be outside its source") + accepted_inventory = verify_sha256_sums(source) + destination.parent.mkdir(parents=True, exist_ok=True) + staging = destination.with_name(f".staging-{uuid.uuid4().hex}-{destination.name}") + snapshot_parent = Path( + tempfile.mkdtemp( + prefix=f".snapshot-{uuid.uuid4().hex}-", + dir=destination.parent, + ) + ) + snapshot_parent.chmod(0o700) + snapshot = snapshot_parent / source.name + try: + _capture_release_snapshot(source, snapshot, accepted_inventory) + create_reproducible_release_archive(snapshot, staging) + _verify_release_archive(staging, source.name, accepted_inventory) + staging.replace(destination) + finally: + staging.unlink(missing_ok=True) + shutil.rmtree(snapshot_parent, ignore_errors=True) + print(f">> archived {source} as {destination}") + + +def _github_headers(token: str | None, accept: str) -> dict[str, str]: + headers = { + "Accept": accept, + "X-GitHub-Api-Version": GITHUB_API_VERSION, + "User-Agent": "nvx-release-downloader", + } + if token is not None: + headers["Authorization"] = f"Bearer {token}" + return headers + + +def _github_error_message(error: urllib.error.HTTPError) -> str: + try: + payload = error.read().decode("utf-8", "replace").strip() + except (AttributeError, OSError, ValueError): + return "" + if not payload: + return "" + try: + document: object = json.loads(payload) + except json.JSONDecodeError: + return " ".join(payload.split())[:200] + if isinstance(document, dict): + message = cast(dict[str, object], document).get("message") + if isinstance(message, str): + return message + return "" + + +def _github_error_hint(error: urllib.error.HTTPError, token: str | None) -> str: + if token is None and error.code in (401, 403, 404): + return "set GH_TOKEN to a token that can read the repository" + if error.code == 401: + return "the configured GitHub token was rejected; refresh or replace it" + if error.code == 403: + if error.headers.get("x-ratelimit-remaining") == "0": + return "the GitHub API rate limit is exhausted; retry later" + return ( + "the configured GitHub token lacks access; grant it read access to " + "the repository contents and authorize it for organization single sign-on" + ) + if error.code == 404: + return ( + "verify --repository and that the configured GitHub token can read " + "that repository" + ) + return "" + + +def _github_json( + url: str, + token: str | None, + *, + missing_ok: bool = False, +) -> object | None: + request = urllib.request.Request( + url, + headers=_github_headers(token, "application/vnd.github+json"), + ) + try: + with credential_safe_opener().open(request) as response: + return json.load(response) + except urllib.error.HTTPError as error: + if missing_ok and error.code == 404: + return None + message = _github_error_message(error) + hint = _github_error_hint(error, token) + detail = f": {message.rstrip('.')}" if message else "" + advice = f"; {hint}" if hint else "" + raise _GitHubReleaseQueryError( + error.code, + f"GitHub release query failed with HTTP {error.code}{detail}{advice}", + ) from error + except (OSError, urllib.error.URLError) as error: + raise ScriptError(f"GitHub release query failed: {error}") from error + + +def _first_parent_commits(limit: int) -> list[str]: + try: + output = git_output( + "rev-list", "--first-parent", f"--max-count={limit}", "HEAD" + ) + except subprocess.CalledProcessError as error: + detail = str(error.stderr or "").strip() or str(error) + raise ScriptError( + f"download needs a Git checkout to find its release: {detail}" + ) from error + except (OSError, subprocess.SubprocessError) as error: + raise ScriptError( + f"download needs a Git checkout to find its release: {error}" + ) from error + return output.split() + + +def _version_at(commit: str) -> str | None: + try: + return git_output("show", f"{commit}:VERSION") or None + except subprocess.CalledProcessError: + return None + except (OSError, subprocess.SubprocessError) as error: + raise ScriptError( + f"download failed to read VERSION at {commit}: {error}" + ) from error + + +def _checkout_release_asset( + repository: str, + platform: str, + token: str | None, +) -> _ReleaseAsset: + repository_parts = repository.split("/") + if len(repository_parts) != 2 or not all(repository_parts): + raise ScriptError("GitHub repository must be OWNER/REPOSITORY") + encoded_repository = "/".join( + urllib.parse.quote(part, safe="") for part in repository_parts + ) + repository_url = f"https://api.github.com/repos/{encoded_repository}" + + extension = ".zip" if platform.startswith("windows-") else ".tar.gz" + asset_pattern = re.compile(rf"^nvx-.+-{re.escape(platform)}{re.escape(extension)}$") + commits = _first_parent_commits(_RELEASE_SEARCH_COMMITS) + for distance, commit in enumerate(commits): + version = _version_at(commit) + if version is None: + continue + tag = development_release_tag(version, commit) + release = _github_json( + f"{repository_url}/releases/tags/{urllib.parse.quote(tag, safe='')}", + token, + missing_ok=True, + ) + if release is None: + continue + if not isinstance(release, dict): + raise ScriptError("GitHub release query returned an invalid response") + release_document = cast(dict[str, object], release) + if release_document.get("draft") is True: + continue + assets = release_document.get("assets") + if not isinstance(assets, list): + continue + for asset_value in cast(list[object], assets): + if not isinstance(asset_value, dict): + continue + asset = cast(dict[str, object], asset_value) + name = asset.get("name") + asset_url = asset.get("url") + size = asset.get("size") + if ( + isinstance(name, str) + and asset_pattern.fullmatch(name) is not None + and isinstance(asset_url, str) + and isinstance(size, int) + and not isinstance(size, bool) + ): + if distance: + plural = "s" if distance > 1 else "" + print( + f">> no {platform} release for HEAD; using {tag}, " + f"{distance} first-parent commit{plural} earlier" + ) + return _ReleaseAsset(tag, name, asset_url, size) + # GitHub also answers 404 for a repository that the caller cannot read, so + # query it to report that case with its usual hint. + _github_json(repository_url, token) + raise ScriptError( + f"no {repository} release with an NVX package for {platform} was built " + f"from HEAD or its first-parent ancestors ({len(commits)} commits " + "searched); check out a released commit or build the artifacts" + ) + + +def _checkout_release_asset_with_fallback( + repository: str, + platform: str, + token: str | None, +) -> tuple[_ReleaseAsset, str | None]: + if token is None: + return _checkout_release_asset(repository, platform, None), None + + try: + return _checkout_release_asset(repository, platform, token), token + except _GitHubReleaseQueryError as authenticated_error: + if authenticated_error.status not in (401, 403): + raise + print( + f">> {authenticated_error}; retrying without credentials", + file=sys.stderr, + ) + try: + return _checkout_release_asset(repository, platform, None), None + except _GitHubReleaseQueryError as public_error: + raise authenticated_error from public_error + + +def _same_file_identity(first: os.stat_result, second: os.stat_result) -> bool: + return first.st_dev == second.st_dev and first.st_ino == second.st_ino + + +def _require_safe_source_parents(root: Path, relative: PurePosixPath) -> None: + root_metadata = root.lstat() + if stat.S_ISLNK(root_metadata.st_mode) or not stat.S_ISDIR(root_metadata.st_mode): + raise ScriptError(f"release snapshot source is not a directory: {root}") + current = root + for part in relative.parts[:-1]: + current /= part + metadata = current.lstat() + if stat.S_ISLNK(metadata.st_mode) or not stat.S_ISDIR(metadata.st_mode): + raise ScriptError( + f"release snapshot source has an unsafe directory: {relative}" + ) + + +def _copy_pinned_regular_file( + source_root: Path, + snapshot_root: Path, + relative_name: str, + expected_sha256: str, +) -> None: + relative = PurePosixPath(relative_name) + _require_safe_source_parents(source_root, relative) + source = source_root.joinpath(*relative.parts) + before = source.lstat() + if stat.S_ISLNK(before.st_mode) or not stat.S_ISREG(before.st_mode): + raise ScriptError( + f"release snapshot source is not a regular file: {relative_name}" + ) + + flags = os.O_RDONLY | getattr(os, "O_BINARY", 0) + flags |= getattr(os, "O_NOFOLLOW", 0) + descriptor = os.open(source, flags) + try: + opened = os.fstat(descriptor) + after_open = source.lstat() + if ( + not stat.S_ISREG(opened.st_mode) + or not stat.S_ISREG(after_open.st_mode) + or not _same_file_identity(before, opened) + or not _same_file_identity(opened, after_open) + ): + raise ScriptError( + f"release snapshot source changed while opening: {relative_name}" + ) + + destination = snapshot_root.joinpath(*relative.parts) + destination.parent.mkdir(parents=True, exist_ok=True) + digest = hashlib.sha256() + with os.fdopen(descriptor, "rb", closefd=False) as source_file: + with destination.open("xb") as destination_file: + while chunk := source_file.read(1024 * 1024): + destination_file.write(chunk) + digest.update(chunk) + after_read = os.fstat(descriptor) + if not _same_file_identity(opened, after_read): + raise ScriptError( + f"release snapshot source changed while reading: {relative_name}" + ) + actual_sha256 = digest.hexdigest() + if actual_sha256 != expected_sha256: + raise ScriptError( + f"release snapshot source changed for {relative_name}: " + f"{actual_sha256}, expected {expected_sha256}" + ) + destination.chmod( + 0o755 if relative.parts[0] == "bin" or before.st_mode & 0o111 else 0o644 + ) + finally: + os.close(descriptor) + + +def _capture_release_snapshot( + source: Path, + snapshot: Path, + accepted: VerifiedChecksumInventory, +) -> None: + snapshot.mkdir(mode=0o700) + expected_files = dict(accepted.files) + expected_files["SHA256SUMS"] = accepted.checksum_sha256 + for relative_name, expected_sha256 in sorted(expected_files.items()): + _copy_pinned_regular_file( + source, + snapshot, + relative_name, + expected_sha256, + ) + captured = verify_sha256_sums(snapshot) + if captured != accepted: + raise ScriptError( + "captured release snapshot does not match the accepted source inventory" + ) + + +def _canonical_archive_member(name: str, *, is_directory: bool) -> str: + normalized_name = name[:-1] if is_directory and name.endswith("/") else name + path = PurePosixPath(normalized_name) + if ( + not normalized_name + or "\\" in normalized_name + or path.is_absolute() + or "." in path.parts + or ".." in path.parts + or any(":" in part for part in path.parts) + or path.as_posix() != normalized_name + or (not is_directory and name.endswith("/")) + ): + raise ScriptError(f"unsafe path in release archive: {name}") + return normalized_name + + +def _validate_archive_layout( + entries: Sequence[tuple[str, bool]], + *, + expected_root: str | None = None, +) -> tuple[str, dict[str, bool]]: + layout: dict[str, bool] = {} + for name, is_directory in entries: + canonical = _canonical_archive_member(name, is_directory=is_directory) + if canonical in layout: + raise ScriptError(f"duplicate path in release archive: {canonical}") + layout[canonical] = is_directory + roots = {PurePosixPath(name).parts[0] for name in layout} + if len(roots) != 1: + raise ScriptError("release archive must contain exactly one package root") + root = roots.pop() + if expected_root is not None and root != expected_root: + raise ScriptError(f"release archive root is {root}, expected {expected_root}") + if root in layout and layout[root] is not True: + raise ScriptError("release archive package root must be a directory") + if expected_root is not None: + if layout.get(root) is not True: + raise ScriptError("release archive omits its package root directory") + for name in layout: + parent = PurePosixPath(name).parent + while parent != PurePosixPath("."): + parent_name = parent.as_posix() + if layout.get(parent_name) is not True: + raise ScriptError( + f"release archive omits directory entry {parent_name}" + ) + parent = parent.parent + return root, layout + + +def _release_archive_layout( + archive_path: Path, + *, + expected_root: str | None = None, +) -> tuple[str, dict[str, bool]]: + try: + if archive_path.name.endswith(".tar.gz"): + with tarfile.open(archive_path, "r:gz") as archive: + entries: list[tuple[str, bool]] = [] + for member in archive.getmembers(): + if not (member.isfile() or member.isdir()): + raise ScriptError( + f"unsupported entry in release archive: {member.name}" + ) + entries.append((member.name, member.isdir())) + return _validate_archive_layout(entries, expected_root=expected_root) + if archive_path.suffix == ".zip": + with zipfile.ZipFile(archive_path) as archive: + entries = [] + for member in archive.infolist(): + is_directory = member.is_dir() + file_type = (member.external_attr >> 16) & 0o170000 + allowed_types = ( + (0, stat.S_IFDIR) if is_directory else (0, stat.S_IFREG) + ) + if file_type not in allowed_types: + raise ScriptError( + f"unsupported entry in release archive: {member.filename}" + ) + entries.append((member.filename, is_directory)) + return _validate_archive_layout(entries, expected_root=expected_root) + except (tarfile.TarError, zipfile.BadZipFile) as error: + raise ScriptError( + f"invalid release archive {archive_path.name}: {error}" + ) from error + raise ScriptError(f"unsupported release archive: {archive_path.name}") + + +def _extract_release_archive(archive_path: Path, destination: Path) -> None: + _release_archive_layout(archive_path) + try: + if archive_path.name.endswith(".tar.gz"): + with tarfile.open(archive_path, "r:gz") as archive: + for member in archive.getmembers(): + canonical = _canonical_archive_member( + member.name, + is_directory=member.isdir(), + ) + target = destination.joinpath(*PurePosixPath(canonical).parts) + if member.isdir(): + target.mkdir(parents=True, exist_ok=True) + target.chmod(member.mode & 0o777) + continue + target.parent.mkdir(parents=True, exist_ok=True) + source = archive.extractfile(member) + if source is None: + raise ScriptError( + f"could not read release archive entry: {member.name}" + ) + with source, target.open("xb") as output: + shutil.copyfileobj(source, output, length=1024 * 1024) + target.chmod(member.mode & 0o777) + return + if archive_path.suffix == ".zip": + with zipfile.ZipFile(archive_path) as archive: + for member in archive.infolist(): + is_directory = member.is_dir() + canonical = _canonical_archive_member( + member.filename, + is_directory=is_directory, + ) + target = destination.joinpath(*PurePosixPath(canonical).parts) + mode = (member.external_attr >> 16) & 0o777 + if is_directory: + target.mkdir(parents=True, exist_ok=True) + if mode: + target.chmod(mode) + continue + target.parent.mkdir(parents=True, exist_ok=True) + with archive.open(member) as source, target.open("xb") as output: + shutil.copyfileobj(source, output, length=1024 * 1024) + if mode: + target.chmod(mode) + return + except (tarfile.TarError, zipfile.BadZipFile) as error: + raise ScriptError( + f"invalid release archive {archive_path.name}: {error}" + ) from error + raise ScriptError(f"unsupported release archive: {archive_path.name}") + + +def _filesystem_archive_layout(destination: Path) -> dict[str, bool]: + layout: dict[str, bool] = {} + for path in destination.rglob("*"): + metadata = path.lstat() + relative = path.relative_to(destination).as_posix() + if stat.S_ISLNK(metadata.st_mode): + raise ScriptError(f"release archive extracted a symlink: {relative}") + if stat.S_ISDIR(metadata.st_mode): + layout[relative] = True + elif stat.S_ISREG(metadata.st_mode): + layout[relative] = False + else: + raise ScriptError(f"release archive extracted a special file: {relative}") + return layout + + +def _verify_release_archive( + archive_path: Path, + expected_root: str, + accepted: VerifiedChecksumInventory, +) -> None: + root, archived_layout = _release_archive_layout( + archive_path, + expected_root=expected_root, + ) + extraction = Path( + tempfile.mkdtemp( + prefix=f".{archive_path.name}.verify-", + dir=archive_path.parent, + ) + ) + try: + _extract_release_archive(archive_path, extraction) + extracted_layout = _filesystem_archive_layout(extraction) + if extracted_layout != archived_layout: + raise ScriptError( + "release archive extraction did not preserve its exact member layout" + ) + archived = verify_sha256_sums(extraction / root) + if archived != accepted: + raise ScriptError( + "release archive does not match the accepted source inventory" + ) + finally: + shutil.rmtree(extraction, ignore_errors=True) + + +def _replace_runtime_file(source: Path, destination: Path) -> None: + destination.parent.mkdir(parents=True, exist_ok=True) + temporary = destination.with_name(f"{destination.name}.part") + temporary.unlink(missing_ok=True) + try: + shutil.copy2(source, temporary) + temporary.replace(destination) + finally: + temporary.unlink(missing_ok=True) + + +def _packaged_guest_artifact_names(package_root: Path) -> list[str]: + source_manifest = package_root / "SOURCE-MANIFEST.json" + if not source_manifest.exists(): + return list(ReleaseBuildConstants.GUEST_ARTIFACT_NAMES) + manifest = _read_json_object(source_manifest, "packaged source manifest") + names: list[str] = [] + if isinstance(manifest.get("linux"), dict): + names.extend( + ( + KernelBuildConstants.BINARY_NAME, + KernelBuildConstants.CONFIG_NAME, + ) + ) + if isinstance(manifest.get("alpine"), dict): + names.extend( + ( + AlpineBuildConstants.INITRAMFS_NAME, + AlpineBuildConstants.PACKAGE_MANIFEST_NAME, + ) + ) + if isinstance(manifest.get("ubuntu"), dict): + names.extend( + ( + UbuntuBuildConstants.INITRAMFS_NAME, + UbuntuBuildConstants.PACKAGE_MANIFEST_NAME, + UbuntuBuildConstants.DISTRO_NAME, + UbuntuBuildConstants.DISTRO_MANIFEST_NAME, + ) + ) + if isinstance(manifest.get("azurelinux"), dict): + names.extend(ReleaseBuildConstants.AZURELINUX_ARTIFACT_NAMES) + if not names: + raise ScriptError("packaged source manifest does not declare guest artifacts") + return names + + +def _install_release_archive(archive_path: Path) -> None: + with tempfile.TemporaryDirectory(prefix="nvx-release-") as temporary: + extraction_root = Path(temporary) + _extract_release_archive(archive_path, extraction_root) + checksum_files = list(extraction_root.glob("*/SHA256SUMS")) + if len(checksum_files) != 1: + raise ScriptError( + "release archive must contain one package root with SHA256SUMS" + ) + package_root = checksum_files[0].parent + verify_sha256_sums(package_root) + + binary_destination = openvmm_binary_path() + binary_source = require_file( + package_root / "bin" / binary_destination.name, + "packaged OpenVMM binary", + ) + guest_sources = { + name: require_file( + package_root / "guest" / name, + f"packaged guest artifact {name}", + ) + for name in _packaged_guest_artifact_names(package_root) + } + provenance_sources = { + name: require_file( + package_root / "provenance" / name, + f"packaged provenance artifact {name}", + ) + for name in ( + OpenVMMBuildConstants.PROVENANCE_NAME, + KernelBuildConstants.PROVENANCE_NAME, + InitramfsBuildConstants.PROVENANCE_NAME, + ) + } + _replace_runtime_file(binary_source, binary_destination) + for name, source in guest_sources.items(): + _replace_runtime_file(source, artifact_path(name)) + for name in ReleaseBuildConstants.GUEST_ARTIFACT_NAMES: + if name not in guest_sources: + artifact_path(name).unlink(missing_ok=True) + for name, source in provenance_sources.items(): + _replace_runtime_file(source, artifact_path(name)) + + +def download_checkout_release(repository: str, platform: str) -> None: + token = os.environ.get("GH_TOKEN") or os.environ.get("GITHUB_TOKEN") + asset, download_token = _checkout_release_asset_with_fallback( + repository, + platform, + token, + ) + print(f">> downloading {asset.name} from {asset.tag}") + with tempfile.TemporaryDirectory(prefix="nvx-download-") as temporary: + archive_path = Path(temporary) / asset.name + download( + asset.url, + archive_path, + headers=_github_headers(download_token, "application/octet-stream"), + opener=credential_safe_opener(), + ) + actual_size = archive_path.stat().st_size + if actual_size != asset.size: + raise ScriptError( + f"downloaded {asset.name} is {actual_size} bytes, expected {asset.size}" + ) + _install_release_archive(archive_path) + print(f">> installed {asset.tag} for {platform}") + + +def _copy_release_file(source: Path, destination: Path) -> None: + require_file(source, source.name) + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source, destination) + + +def _validate_alpine_sources(package_manifests: list[Path]) -> frozenset[str]: + source_root = BuildConstants.SOURCE_DIR / AlpineBuildConstants.GUEST_NAME + source_manifest_path = require_file( + source_root / "manifest.json", + "collected Alpine source manifest", + ) + source_manifest = _read_json_object( + source_manifest_path, + "collected Alpine source manifest", + ) + if source_manifest.get("format") != AlpineBuildConstants.SOURCE_MANIFEST_FORMAT: + raise ScriptError( + "collected Alpine sources use an unsupported manifest format; " + "run collect-sources again" + ) + recorded_executables = source_manifest.get("executables") + if not isinstance(recorded_executables, list) or not all( + isinstance(path, str) for path in cast(list[object], recorded_executables) + ): + raise ScriptError( + "collected Alpine source manifest has an invalid executables list" + ) + executables = frozenset(cast(list[str], recorded_executables)) + source_packages = cast(list[dict[str, str]], source_manifest["packages"]) + collected = { + (package["package"], package["version"], package["commit"]): package + for package in source_packages + } + missing: list[str] = [] + for manifest_path in package_manifests: + package_manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + for package in package_manifest["packages"]: + key = ( + package["name"], + package["version"], + package["aports_commit"], + ) + if key not in collected: + missing.append(f"{key[0]}-{key[1]} ({key[2]})") + if missing: + raise ScriptError( + "collected Alpine sources do not cover packaged APKs: " + + ", ".join(sorted(missing)) + ) + for package in collected.values(): + for field in ("recipe", "source_directory"): + path = source_root / package[field] + if not path.exists(): + raise ScriptError( + f"collected Alpine {field} is missing for " + f"{package['package']}: {path}" + ) + checksummed = dict(verify_sha256_sums(source_root).files) + for path in sorted(executables): + if path not in checksummed or not path.startswith("recipes/"): + raise ScriptError( + f"collected Alpine executable is not a checksummed recipe file: {path}" + ) + return executables + + +def _validate_ubuntu_sources(package_manifests: list[Path]) -> None: + source_root = BuildConstants.SOURCE_DIR / UbuntuBuildConstants.GUEST_NAME + source_manifest_path = require_file( + source_root / "manifest.json", + "collected Ubuntu source manifest", + ) + source_manifest = json.loads(source_manifest_path.read_text(encoding="utf-8")) + collected = { + (package["source_name"], package["source_version"]): package + for package in source_manifest["packages"] + } + missing: list[str] = [] + for manifest_path in package_manifests: + package_manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + for package in package_manifest["packages"]: + key = (package["source_name"], package["source_version"]) + if key not in collected: + missing.append(f"{key[0]}={key[1]}") + if missing: + raise ScriptError( + "collected Ubuntu sources do not cover packaged binaries: " + + ", ".join(sorted(set(missing))) + ) + for package in collected.values(): + dsc = source_root / package["dsc"] + require_file(dsc, f"Ubuntu source metadata for {package['source_name']}") + for source_file in package["files"]: + require_file( + source_root / source_file["path"], + f"Ubuntu source member {source_file['name']}", + ) + verify_sha256_sums(source_root) + + +def _create_source_archive( + output: Path, + inputs: list[tuple[Path, str]], + *, + mode: Callable[[tarfile.TarInfo], int] | None = None, +) -> None: + print(f">> creating source archive {output}") + create_reproducible_tar_gz(output, inputs, mode=mode) + + +def _project_source_archive( + output: Path, + version: str, + package_manifests: list[Path], +) -> None: + root = f"nvx-project-source-{version}" + inputs = [ + (BuildConstants.REPO_ROOT / relative, f"{root}/{relative}") + for relative in ReleaseBuildConstants.PROJECT_SOURCE_PATHS + ] + inputs.extend( + ( + manifest, + f"{root}/build/{manifest.name}", + ) + for manifest in package_manifests + ) + inputs.append( + ( + artifact_path(KernelBuildConstants.CONFIG_NAME), + f"{root}/{BuildConstants.BUILD_DIRECTORY_NAME}/{KernelBuildConstants.CONFIG_NAME}", + ) + ) + _create_source_archive(output, inputs) + + +def _alpine_source_archive( + output: Path, + version: str, + package_manifests: list[Path], + executables: frozenset[str], +) -> None: + root = f"nvx-alpine-source-{version}" + inputs = [ + (BuildConstants.SOURCE_DIR / AlpineBuildConstants.GUEST_NAME, f"{root}/sources") + ] + inputs.extend( + ( + manifest, + f"{root}/manifests/{manifest.name}", + ) + for manifest in package_manifests + ) + executable_members = {f"{root}/sources/{path}" for path in executables} + + def mode(member: tarfile.TarInfo) -> int: + # Apply the recorded modes, which hosts such as Windows cannot store. + if member.isdir() or member.name in executable_members: + return 0o755 + return 0o644 + + _create_source_archive(output, inputs, mode=mode) + + +def _ubuntu_source_archive( + output: Path, + version: str, + package_manifests: list[Path], +) -> None: + root = f"nvx-ubuntu-source-{version}" + inputs = [ + (BuildConstants.SOURCE_DIR / UbuntuBuildConstants.GUEST_NAME, f"{root}/sources") + ] + inputs.extend( + ( + manifest, + f"{root}/manifests/{manifest.name}", + ) + for manifest in package_manifests + ) + _create_source_archive(output, inputs) + + +def _validate_linux_source_archive(path: Path) -> None: + expected_members = { + KernelBuildConstants.CONFIG_NAME: artifact_path( + KernelBuildConstants.CONFIG_NAME + ).read_bytes(), + "SOURCE-MANIFEST.json": ( + BuildConstants.REPO_ROOT / "SOURCE-MANIFEST.json" + ).read_bytes(), + } + manifest = json.loads(expected_members["SOURCE-MANIFEST.json"]) + expected_members.update( + { + patch: (BuildConstants.REPO_ROOT / patch).read_bytes() + for patch in manifest["linux"]["patches"] + } + ) + with tarfile.open(path, "r:gz") as archive: + members = archive.getmembers() + names = [member.name for member in members] + if not any( + name.endswith( + f"{KernelBuildConstants.SOURCE_NAME}/drivers/tty/hvc/hvc_xe9.c" + ) + for name in names + ): + raise ScriptError(f"{path} does not contain the patched xe9 HVC driver") + for suffix, expected in expected_members.items(): + matches = [member for member in members if member.name.endswith(suffix)] + if len(matches) != 1: + raise ScriptError( + f"{path} contains {len(matches)} members ending in {suffix}" + ) + extracted = archive.extractfile(matches[0]) + if extracted is None or extracted.read() != expected: + raise ScriptError(f"{path} has stale contents for {suffix}") + + +def _validate_azurelinux_manifest(manifest_path: Path) -> None: + manifest = _read_json_object(manifest_path, "Azure Linux initramfs manifest") + initramfs = artifact_path(AzureLinuxBuildConstants.INITRAMFS_NAME) + if ( + manifest.get("format") != AzureLinuxBuildConstants.PACKAGE_MANIFEST_VERSION + or manifest.get("guest") != AzureLinuxBuildConstants.GUEST_NAME + or manifest.get("artifact") != initramfs.name + or manifest.get("artifact_sha256") != sha256_file(initramfs) + or manifest.get("package_manifest_format") + != AzureLinuxBuildConstants.PACKAGE_MANIFEST_FORMAT + or manifest.get("image") != AzureLinuxBuildConstants.IMAGE + ): + raise ScriptError("Azure Linux initramfs manifest is invalid") + if manifest.get("input_sha256") != azurelinux_input_sha256(): + raise ScriptError( + "Azure Linux initramfs manifest does not match the current build inputs" + ) + + +def _guest_release_inputs( + *, + include_azurelinux: bool, +) -> tuple[list[str], list[Path], list[Path], list[Path]]: + guest_names = [ + name + for name in ReleaseBuildConstants.GUEST_ARTIFACT_NAMES + if include_azurelinux + or name not in ReleaseBuildConstants.AZURELINUX_ARTIFACT_NAMES + ] + for name in guest_names: + require_file(artifact_path(name), f"required guest artifact {name}") + alpine_manifests = [artifact_path(AlpineBuildConstants.PACKAGE_MANIFEST_NAME)] + ubuntu_manifests = [ + artifact_path(UbuntuBuildConstants.PACKAGE_MANIFEST_NAME), + artifact_path(UbuntuBuildConstants.DISTRO_MANIFEST_NAME), + ] + azurelinux_manifests: list[Path] = [] + if include_azurelinux: + azurelinux_manifests.append( + artifact_path(AzureLinuxBuildConstants.PACKAGE_MANIFEST_NAME) + ) + _validate_azurelinux_manifest(azurelinux_manifests[0]) + expected_input_sha256 = converter_input_sha256(customization_files()) + for artifact_name, manifest in zip( + (UbuntuBuildConstants.INITRAMFS_NAME, UbuntuBuildConstants.DISTRO_NAME), + ubuntu_manifests, + strict=True, + ): + artifact = artifact_path(artifact_name) + document = _read_json_object(manifest, f"{artifact_name} manifest") + if ( + document.get("format") != UbuntuBuildConstants.PACKAGE_MANIFEST_VERSION + or document.get("guest") != UbuntuBuildConstants.GUEST_NAME + or document.get("artifact") != artifact.name + or document.get("artifact_sha256") != sha256_file(artifact) + or document.get("input_sha256") != expected_input_sha256 + ): + raise ScriptError( + f"Ubuntu artifact manifest does not match {artifact_name}" + ) + return guest_names, alpine_manifests, ubuntu_manifests, azurelinux_manifests + + +def _read_json_object(path: Path, description: str) -> dict[str, object]: + try: + value: object = json.loads(path.read_text(encoding="utf-8")) + except (json.JSONDecodeError, OSError) as error: + raise ScriptError(f"invalid {description}: {error}") from error + if not isinstance(value, dict): + raise ScriptError(f"invalid {description}: expected a JSON object") + return cast(dict[str, object], value) + + +def _validate_openvmm_provenance( + binary: Path, + provenance_path: Path, +) -> dict[str, object]: + provenance = _read_json_object(provenance_path, "OpenVMM build provenance") + revision, source_clean = openvmm_git_state(OpenVMMBuildConstants.DIRECTORY) + if ( + provenance.get("format") != OpenVMMBuildConstants.PROVENANCE_FORMAT + or provenance.get("source_revision") != revision + or provenance.get("source_clean") is not True + or not source_clean + or provenance.get("executable_sha256") != sha256_file(binary) + ): + raise ScriptError( + "OpenVMM build provenance does not match the current clean pinned " + "source and executable" + ) + return provenance + + +def _validate_kernel_provenance( + kernel: Path, + kernel_config: Path, + provenance_path: Path, +) -> dict[str, object]: + provenance = _read_json_object(provenance_path, "kernel build provenance") + expected_inputs = kernel_provenance_inputs() + if ( + provenance.get("format") != KernelBuildConstants.PROVENANCE_FORMAT + or provenance.get("source") != expected_inputs["source"] + or provenance.get("input_config") != expected_inputs["input_config"] + or provenance.get("kernel_sha256") != sha256_file(kernel) + or provenance.get("config_sha256") != sha256_file(kernel_config) + ): + raise ScriptError( + "kernel build provenance does not match the current source, config, " + "and vmlinux" + ) + assert_required_kernel_config(kernel_config) + return provenance + + +def _validate_initramfs_provenance( + initramfs: Path, + package_manifest: Path, + provenance_path: Path, +) -> dict[str, object]: + provenance = _read_json_object( + provenance_path, + "initramfs build provenance", + ) + if ( + provenance.get("format") != InitramfsBuildConstants.PROVENANCE_FORMAT + or provenance.get("inputs") != initramfs_provenance_inputs() + or provenance.get("initramfs_sha256") != sha256_file(initramfs) + or provenance.get("package_manifest_sha256") != sha256_file(package_manifest) + ): + raise ScriptError( + "initramfs build provenance does not match the current source, " + "package manifest, and initramfs" + ) + return provenance + + +def _require_runtime_provenance_paths() -> tuple[Path, Path, Path]: + return ( + require_file( + artifact_path(OpenVMMBuildConstants.PROVENANCE_NAME), + "OpenVMM build provenance", + ), + require_file( + artifact_path(KernelBuildConstants.PROVENANCE_NAME), + "kernel build provenance", + ), + require_file( + artifact_path(InitramfsBuildConstants.PROVENANCE_NAME), + "initramfs build provenance", + ), + ) + + +def validate_runtime_artifact_provenance() -> None: + binary = require_file(openvmm_binary_path(), "OpenVMM release binary") + kernel = require_file( + artifact_path(KernelBuildConstants.BINARY_NAME), + "required guest artifact vmlinux", + ) + initramfs = require_file( + artifact_path(AlpineBuildConstants.INITRAMFS_NAME), + "required guest artifact initramfs.cpio.gz", + ) + package_manifest = require_file( + artifact_path(AlpineBuildConstants.PACKAGE_MANIFEST_NAME), + "initramfs package manifest", + ) + kernel_config = require_file( + artifact_path(KernelBuildConstants.CONFIG_NAME), + "required guest artifact vmlinux.config", + ) + ( + openvmm_provenance_path, + kernel_provenance_path, + initramfs_provenance_path, + ) = _require_runtime_provenance_paths() + _validate_openvmm_provenance(binary, openvmm_provenance_path) + _validate_kernel_provenance(kernel, kernel_config, kernel_provenance_path) + _validate_initramfs_provenance( + initramfs, + package_manifest, + initramfs_provenance_path, + ) + + +def _packaged_source_manifest( + root_manifest: dict[str, object], + release_version: str, + release_root: Path, + openvmm_provenance: dict[str, object], + binary_name: str, + *, + include_azurelinux: bool, +) -> bytes: + distribution = root_manifest.get("distribution") + openvmm = root_manifest.get("openvmm") + linux = root_manifest.get("linux") + alpine = root_manifest.get("alpine") + ubuntu = root_manifest.get("ubuntu") + azurelinux = root_manifest.get("azurelinux") + if not all( + isinstance(section, dict) + for section in (distribution, openvmm, linux, alpine, ubuntu, azurelinux) + ): + raise ScriptError("SOURCE-MANIFEST.json is missing a required object") + distribution_section = cast(dict[str, object], distribution) + openvmm_section = cast(dict[str, object], openvmm) + linux_section = cast(dict[str, object], linux) + alpine_section = cast(dict[str, object], alpine) + ubuntu_section = cast(dict[str, object], ubuntu) + distribution_section["version"] = release_version + openvmm_section["source_revision"] = openvmm_provenance["source_revision"] + openvmm_section["executable_sha256"] = sha256_file( + release_root / "bin" / binary_name + ) + linux_section["kernel_sha256"] = sha256_file( + release_root / "guest" / KernelBuildConstants.BINARY_NAME + ) + linux_section["config_sha256"] = sha256_file( + release_root / "guest" / KernelBuildConstants.CONFIG_NAME + ) + alpine_section["initramfs_sha256"] = sha256_file( + release_root / "guest" / AlpineBuildConstants.INITRAMFS_NAME + ) + alpine_section["initramfs_package_manifest_sha256"] = sha256_file( + release_root / "guest" / AlpineBuildConstants.PACKAGE_MANIFEST_NAME + ) + ubuntu_section["initramfs_sha256"] = sha256_file( + release_root / "guest" / UbuntuBuildConstants.INITRAMFS_NAME + ) + ubuntu_section["initramfs_package_manifest_sha256"] = sha256_file( + release_root / "guest" / UbuntuBuildConstants.PACKAGE_MANIFEST_NAME + ) + ubuntu_section["distro_layer_sha256"] = sha256_file( + release_root / "guest" / UbuntuBuildConstants.DISTRO_NAME + ) + ubuntu_section["distro_layer_manifest_sha256"] = sha256_file( + release_root / "guest" / UbuntuBuildConstants.DISTRO_MANIFEST_NAME + ) + if include_azurelinux: + azurelinux_section = cast(dict[str, object], azurelinux) + azurelinux_section["initramfs_sha256"] = sha256_file( + release_root / "guest" / AzureLinuxBuildConstants.INITRAMFS_NAME + ) + azurelinux_section["initramfs_package_manifest_sha256"] = sha256_file( + release_root / "guest" / AzureLinuxBuildConstants.PACKAGE_MANIFEST_NAME + ) + else: + del root_manifest["azurelinux"] + return (json.dumps(root_manifest, indent=2, sort_keys=True) + "\n").encode("utf-8") + + +def _validate_root_manifest_contract(manifest: dict[str, object]) -> None: + distribution = manifest.get("distribution") + openvmm = manifest.get("openvmm") + if not isinstance(distribution, dict): + raise ScriptError("SOURCE-MANIFEST.json distribution must identify nvx") + distribution_section = cast(dict[str, object], distribution) + if distribution_section.get("name") != "nvx": + raise ScriptError("SOURCE-MANIFEST.json distribution must identify nvx") + expected_openvmm = { + "microvm_abi_version": OpenVMMBuildConstants.MICROVM_ABI_VERSION, + "control_session_protocol_version": OpenVMMBuildConstants.CONTROL_SESSION_PROTOCOL_VERSION, + "control_contract_revision": OpenVMMBuildConstants.CONTROL_CONTRACT_REVISION, + } + if not isinstance(openvmm, dict): + raise ScriptError( + "SOURCE-MANIFEST.json OpenVMM contract does not match the build contract" + ) + openvmm_section = cast(dict[str, object], openvmm) + for field, expected in expected_openvmm.items(): + if openvmm_section.get(field) != expected: + raise ScriptError( + "SOURCE-MANIFEST.json OpenVMM contract does not match the " + "build contract" + ) + if "source_revision" in openvmm_section: + raise ScriptError( + "root SOURCE-MANIFEST.json must not duplicate the OpenVMM gitlink revision" + ) + if "guest_agent" in manifest: + raise ScriptError( + "root SOURCE-MANIFEST.json must not contain product guest-agent metadata" + ) + + +def _validate_source_manifest_metadata( + manifest: dict[str, object], + kernel_inputs: dict[str, object], +) -> list[str]: + _validate_root_manifest_contract(manifest) + if manifest.get("format") != ReleaseBuildConstants.SOURCE_MANIFEST_FORMAT: + raise ScriptError("SOURCE-MANIFEST.json format must be 1") + linux_value = manifest.get("linux") + alpine_value = manifest.get("alpine") + ubuntu_value = manifest.get("ubuntu") + azurelinux_value = manifest.get("azurelinux") + source_value = kernel_inputs.get("source") + config_value = kernel_inputs.get("input_config") + if not all( + isinstance(section, dict) + for section in ( + linux_value, + alpine_value, + ubuntu_value, + azurelinux_value, + source_value, + config_value, + ) + ): + raise ScriptError("SOURCE-MANIFEST.json is missing source metadata") + linux = cast(dict[str, object], linux_value) + alpine = cast(dict[str, object], alpine_value) + ubuntu = cast(dict[str, object], ubuntu_value) + azurelinux = cast(dict[str, object], azurelinux_value) + source = cast(dict[str, object], source_value) + input_config = cast(dict[str, object], config_value) + source_patches = source.get("patches") + if not isinstance(source_patches, list): + raise ScriptError("kernel provenance source patches must be a list") + patch_paths: list[str] = [] + for patch_value in cast(list[object], source_patches): + if not isinstance(patch_value, dict): + raise ScriptError("kernel provenance source patch must be an object") + patch = cast(dict[str, object], patch_value) + path = patch.get("path") + if not isinstance(path, str): + raise ScriptError("kernel provenance source patch path must be a string") + patch_paths.append(path) + expected_linux: dict[str, object] = { + "version": KernelBuildConstants.VERSION, + "upstream_url": KernelBuildConstants.URL, + "upstream_archive_sha256": KernelBuildConstants.SHA256, + "source_cache": ( + Path(BuildConstants.CACHE_DIRECTORY_NAME) + / KernelBuildConstants.SOURCE_DIRECTORY_NAME + / KernelBuildConstants.SOURCE_NAME + ).as_posix(), + "source_archive": ( + BuildConstants.SOURCE_RELATIVE_DIRECTORY + / KernelBuildConstants.SOURCE_DIRECTORY_NAME + / KernelBuildConstants.SOURCE_ARCHIVE_NAME + ).as_posix(), + "generated_final_config": ( + Path(BuildConstants.BUILD_DIRECTORY_NAME) / KernelBuildConstants.CONFIG_NAME + ).as_posix(), + "input_config": input_config.get("path"), + "patches": patch_paths, + } + for field, expected in expected_linux.items(): + if linux.get(field) != expected: + raise ScriptError( + f"SOURCE-MANIFEST.json Linux {field} does not match the build pin" + ) + expected_alpine: dict[str, object] = { + "version": AlpineBuildConstants.VERSION, + "branch": AlpineBuildConstants.BRANCH, + "architecture": AlpineBuildConstants.ARCHITECTURE, + "minirootfs_url": AlpineBuildConstants.MINIROOTFS_URL, + "minirootfs_sha256": AlpineBuildConstants.MINIROOTFS_SHA256, + "guest_sources": [ + path.as_posix() for path in AlpineBuildConstants.GUEST_SOURCE_DIRECTORIES + ], + "package_manifests": [ + ( + Path(BuildConstants.BUILD_DIRECTORY_NAME) + / AlpineBuildConstants.PACKAGE_MANIFEST_NAME + ).as_posix() + ], + "source_output": ( + BuildConstants.SOURCE_RELATIVE_DIRECTORY / AlpineBuildConstants.GUEST_NAME + ).as_posix(), + } + for field, expected in expected_alpine.items(): + if alpine.get(field) != expected: + raise ScriptError( + f"SOURCE-MANIFEST.json Alpine {field} does not match the build pin" + ) + expected_ubuntu: dict[str, object] = { + "distribution": UbuntuBuildConstants.DISTRIBUTION, + "version": UbuntuBuildConstants.VERSION, + "codename": UbuntuBuildConstants.CODENAME, + "architecture": UbuntuBuildConstants.ARCHITECTURE, + "base_url": UbuntuBuildConstants.BASE_URL, + "base_sha256": UbuntuBuildConstants.BASE_SHA256, + "archive_keyring_url": UbuntuBuildConstants.ARCHIVE_KEYRING_URL, + "archive_keyring_sha256": UbuntuBuildConstants.ARCHIVE_KEYRING_SHA256, + "package_lock": UbuntuBuildConstants.PACKAGE_LOCK_RELATIVE_PATH.as_posix(), + "package_lock_sha256": package_lock_sha256(), + "guest_sources": [ + *( + path.as_posix() + for path in UbuntuBuildConstants.GUEST_SOURCE_DIRECTORIES + ), + UbuntuBuildConstants.PACKAGE_LOCK_RELATIVE_PATH.as_posix(), + ], + "package_manifests": [ + ( + Path(BuildConstants.BUILD_DIRECTORY_NAME) + / UbuntuBuildConstants.PACKAGE_MANIFEST_NAME + ).as_posix(), + ( + Path(BuildConstants.BUILD_DIRECTORY_NAME) + / UbuntuBuildConstants.DISTRO_MANIFEST_NAME + ).as_posix(), + ], + "source_output": ( + BuildConstants.SOURCE_RELATIVE_DIRECTORY / UbuntuBuildConstants.GUEST_NAME + ).as_posix(), + "erofs_converter_format": UbuntuBuildConstants.EROFS_FORMAT, + } + for field, expected in expected_ubuntu.items(): + if ubuntu.get(field) != expected: + raise ScriptError( + f"SOURCE-MANIFEST.json Ubuntu {field} does not match the build pin" + ) + expected_azurelinux: dict[str, object] = { + "distribution": AzureLinuxBuildConstants.DISTRIBUTION, + "version": AzureLinuxBuildConstants.VERSION, + "architecture": AzureLinuxBuildConstants.ARCHITECTURE, + "image": AzureLinuxBuildConstants.IMAGE, + "package_lock": AzureLinuxBuildConstants.PACKAGE_LOCK_RELATIVE_PATH.as_posix(), + "package_lock_sha256": azurelinux_package_lock_sha256(), + "guest_sources": [ + *( + path.as_posix() + for path in AzureLinuxBuildConstants.GUEST_SOURCE_DIRECTORIES + ), + AzureLinuxBuildConstants.PACKAGE_LOCK_RELATIVE_PATH.as_posix(), + ], + "package_manifests": [ + ( + Path(BuildConstants.BUILD_DIRECTORY_NAME) + / AzureLinuxBuildConstants.PACKAGE_MANIFEST_NAME + ).as_posix() + ], + } + for field, expected in expected_azurelinux.items(): + if azurelinux.get(field) != expected: + raise ScriptError( + f"SOURCE-MANIFEST.json Azure Linux {field} does not match the build pin" + ) + return patch_paths + + +def _validate_release_replacement(destination: Path, force: bool) -> None: + if not destination.exists(): + return + if not force: + raise ScriptError( + f"release directory already exists: {destination}; " + "pass --force to replace it" + ) + release_root = ( + BuildConstants.REPO_ROOT / ReleaseBuildConstants.DIRECTORY_NAME + ).resolve() + if destination == release_root or release_root not in destination.parents: + raise ScriptError("--force may only replace a version directory below dist/") + + +def _publish_release_directory( + staging: Path, + destination: Path, + *, + force: bool, +) -> None: + backup = destination.with_name(f".{destination.name}.backup-{uuid.uuid4().hex}") + moved_prior = False + try: + _validate_release_replacement(destination, force) + if destination.exists(): + destination.replace(backup) + moved_prior = True + staging.replace(destination) + except ScriptError: + shutil.rmtree(staging) + raise + except OSError as publish_error: + if moved_prior: + try: + backup.replace(destination) + except OSError as restore_error: + raise _ReleaseRestoreError( + "failed to publish the staged release and restore the prior " + f"release; prior release remains at {backup} and staged " + f"release remains at {staging}: {restore_error}" + ) from publish_error + shutil.rmtree(staging, ignore_errors=True) + raise ScriptError(f"failed to publish staged release: {publish_error}") from ( + publish_error + ) + if moved_prior: + shutil.rmtree(backup) + + +def collect_release_sources(config: DockerBuildConfig) -> None: + _guest_names, alpine_manifests, ubuntu_manifests, _azurelinux_manifests = ( + _guest_release_inputs(include_azurelinux=False) + ) + collect_alpine_sources( + alpine_manifests, + BuildConstants.SOURCE_DIR / AlpineBuildConstants.GUEST_NAME, + BuildConstants.REPO_ROOT + / BuildConstants.CACHE_DIRECTORY_NAME + / AlpineBuildConstants.APORTS_CACHE_DIRECTORY_NAME, + ) + collect_ubuntu_sources( + ubuntu_manifests, + BuildConstants.SOURCE_DIR / UbuntuBuildConstants.GUEST_NAME, + BuildConstants.REPO_ROOT + / BuildConstants.CACHE_DIRECTORY_NAME + / UbuntuBuildConstants.SOURCE_CACHE_DIRECTORY_NAME, + ) + build_docker_linux_source(config) + print( + "!! Azure Linux corresponding source is not collected: source-inclusive " + "packages omit the Azure Linux guest", + file=sys.stderr, + ) + print(f">> collected release sources under {BuildConstants.SOURCE_DIR}") + + +def package_release( + *, + version: str | None, + destination: Path | None, + include_source: bool, + force: bool, +) -> None: + guest_names, alpine_manifests, ubuntu_manifests, azurelinux_manifests = ( + _guest_release_inputs(include_azurelinux=not include_source) + ) + linux_source_archive = ( + BuildConstants.SOURCE_DIR + / KernelBuildConstants.SOURCE_DIRECTORY_NAME + / KernelBuildConstants.SOURCE_ARCHIVE_NAME + ) + alpine_executables: frozenset[str] = frozenset() + if include_source: + alpine_executables = _validate_alpine_sources(alpine_manifests) + _validate_ubuntu_sources(ubuntu_manifests) + require_file(linux_source_archive, "Linux corresponding-source archive") + _validate_linux_source_archive(linux_source_archive) + print( + "!! source-inclusive package omits the Azure Linux guest: " + "collect-sources does not collect its corresponding source", + file=sys.stderr, + ) + else: + print( + "!! binary-only package: publish matching Linux, Alpine, Ubuntu, and " + "Azure Linux corresponding source separately", + file=sys.stderr, + ) + package_manifests = [ + *alpine_manifests, + *ubuntu_manifests, + *azurelinux_manifests, + ] + + release_version = ( + version + or (BuildConstants.REPO_ROOT / "VERSION").read_text(encoding="ascii").strip() + ) + release_destination = ( + destination + or BuildConstants.REPO_ROOT + / ReleaseBuildConstants.DIRECTORY_NAME + / release_version + ).resolve() + _validate_release_replacement(release_destination, force) + binary = require_file(openvmm_binary_path(), "OpenVMM release binary") + kernel = require_file( + artifact_path(KernelBuildConstants.BINARY_NAME), + "required guest artifact vmlinux", + ) + kernel_config = require_file( + artifact_path(KernelBuildConstants.CONFIG_NAME), + "required guest artifact vmlinux.config", + ) + initramfs = require_file( + artifact_path(AlpineBuildConstants.INITRAMFS_NAME), + "required guest artifact initramfs.cpio.gz", + ) + package_manifest = require_file( + artifact_path(AlpineBuildConstants.PACKAGE_MANIFEST_NAME), + "initramfs package manifest", + ) + ( + openvmm_provenance_path, + kernel_provenance_path, + initramfs_provenance_path, + ) = _require_runtime_provenance_paths() + openvmm_provenance = _validate_openvmm_provenance( + binary, + openvmm_provenance_path, + ) + kernel_provenance = _validate_kernel_provenance( + kernel, + kernel_config, + kernel_provenance_path, + ) + initramfs_provenance = _validate_initramfs_provenance( + initramfs, + package_manifest, + initramfs_provenance_path, + ) + root_manifest_path = require_file( + BuildConstants.REPO_ROOT / "SOURCE-MANIFEST.json", + "source manifest", + ) + root_manifest = _read_json_object(root_manifest_path, "source manifest") + _validate_source_manifest_metadata(root_manifest, kernel_provenance) + for name in ("LICENSE", "README.md", "THIRD_PARTY_NOTICES.md"): + require_file(BuildConstants.REPO_ROOT / name, name) + require_file(OpenVMMBuildConstants.DIRECTORY / "LICENSE", "OpenVMM license") + require_file( + BuildConstants.REPO_ROOT / "kernel" / "COPYING-LINUX", "Linux copyright notice" + ) + + release_destination.parent.mkdir(parents=True, exist_ok=True) + staging = Path( + tempfile.mkdtemp( + prefix=f".{release_destination.name}.staging-", + dir=release_destination.parent, + ) + ) + preserve_staging = False + try: + _copy_release_file(binary, staging / "bin" / binary.name) + for name in guest_names: + _copy_release_file(artifact_path(name), staging / "guest" / name) + _copy_release_file( + openvmm_provenance_path, + staging / "provenance" / OpenVMMBuildConstants.PROVENANCE_NAME, + ) + _copy_release_file( + kernel_provenance_path, + staging / "provenance" / KernelBuildConstants.PROVENANCE_NAME, + ) + _copy_release_file( + initramfs_provenance_path, + staging / "provenance" / InitramfsBuildConstants.PROVENANCE_NAME, + ) + for name in ("LICENSE", "README.md", "THIRD_PARTY_NOTICES.md"): + _copy_release_file(BuildConstants.REPO_ROOT / name, staging / name) + _copy_release_file( + OpenVMMBuildConstants.DIRECTORY / "LICENSE", + staging / "licenses" / "LICENSE-OPENVMM", + ) + _copy_release_file( + BuildConstants.REPO_ROOT / "kernel" / "COPYING-LINUX", + staging / "licenses" / "COPYING-LINUX", + ) + if include_source: + source_destination = staging / "source" + _copy_release_file( + linux_source_archive, + source_destination / linux_source_archive.name, + ) + _project_source_archive( + source_destination / f"nvx-project-source-{release_version}.tar.gz", + release_version, + package_manifests, + ) + _alpine_source_archive( + source_destination / f"nvx-alpine-source-{release_version}.tar.gz", + release_version, + alpine_manifests, + alpine_executables, + ) + _ubuntu_source_archive( + source_destination / f"nvx-ubuntu-source-{release_version}.tar.gz", + release_version, + ubuntu_manifests, + ) + packaged_binary = staging / "bin" / binary.name + packaged_kernel = staging / "guest" / KernelBuildConstants.BINARY_NAME + packaged_config = staging / "guest" / KernelBuildConstants.CONFIG_NAME + packaged_initramfs = staging / "guest" / AlpineBuildConstants.INITRAMFS_NAME + packaged_package_manifest = ( + staging / "guest" / AlpineBuildConstants.PACKAGE_MANIFEST_NAME + ) + packaged_openvmm_provenance = ( + staging / "provenance" / OpenVMMBuildConstants.PROVENANCE_NAME + ) + packaged_kernel_provenance = ( + staging / "provenance" / KernelBuildConstants.PROVENANCE_NAME + ) + packaged_initramfs_provenance = ( + staging / "provenance" / InitramfsBuildConstants.PROVENANCE_NAME + ) + if sha256_file(packaged_binary) != openvmm_provenance["executable_sha256"]: + raise ScriptError( + "packaged OpenVMM executable does not match its build provenance" + ) + if sha256_file(packaged_kernel) != kernel_provenance["kernel_sha256"]: + raise ScriptError("packaged kernel does not match its build provenance") + if sha256_file(packaged_config) != kernel_provenance["config_sha256"]: + raise ScriptError( + "packaged kernel config does not match its build provenance" + ) + if ( + sha256_file(packaged_initramfs) != initramfs_provenance["initramfs_sha256"] + or sha256_file(packaged_package_manifest) + != initramfs_provenance["package_manifest_sha256"] + ): + raise ScriptError( + "packaged initramfs artifacts do not match their build provenance" + ) + if ( + _read_json_object( + packaged_openvmm_provenance, + "packaged OpenVMM build provenance", + ) + != openvmm_provenance + ): + raise ScriptError("packaged OpenVMM provenance changed while staging") + if ( + _read_json_object( + packaged_kernel_provenance, + "packaged kernel build provenance", + ) + != kernel_provenance + ): + raise ScriptError("packaged kernel provenance changed while staging") + if ( + _read_json_object( + packaged_initramfs_provenance, + "packaged initramfs build provenance", + ) + != initramfs_provenance + ): + raise ScriptError("packaged initramfs provenance changed while staging") + (staging / "SOURCE-MANIFEST.json").write_bytes( + _packaged_source_manifest( + root_manifest, + release_version, + staging, + openvmm_provenance, + binary.name, + include_azurelinux=not include_source, + ) + ) + write_sha256_sums(staging) + verify_sha256_sums(staging) + _publish_release_directory(staging, release_destination, force=force) + except _ReleaseRestoreError: + preserve_staging = True + raise + finally: + if staging.exists() and not preserve_staging: + shutil.rmtree(staging) + print(f">> packaged {release_destination}") + + +def verify_source_tree() -> None: + manifest = _read_json_object( + BuildConstants.REPO_ROOT / "SOURCE-MANIFEST.json", + "source manifest", + ) + patch_paths = _validate_source_manifest_metadata( + manifest, + kernel_provenance_inputs(), + ) + required = { + BuildConstants.REPO_ROOT + / "kernel" + / "config-microvm": "kernel build configuration", + BuildConstants.REPO_ROOT + / "kernel" + / "patches" + / "0001-microvm-xe9-earlycon.patch": "xe9 early console patch", + BuildConstants.REPO_ROOT + / "kernel" + / "patches" + / "0002-microvm-hvc-xe9.patch": "xe9 HVC patch", + BuildConstants.REPO_ROOT / "kernel" / "COPYING-LINUX": "Linux copyright notice", + BuildConstants.REPO_ROOT / "guest" / "common" / "init": "common guest init", + BuildConstants.REPO_ROOT + / "guest" + / "alpine" + / "nvx-container-enter": "Alpine sandbox container helper", + UbuntuBuildConstants.PACKAGE_LOCK: "Ubuntu supplemental package lock", + OpenVMMBuildConstants.DIRECTORY / "Cargo.toml": "initialized OpenVMM submodule", + } + required.update( + { + BuildConstants.REPO_ROOT / patch: f"kernel patch {patch}" + for patch in patch_paths + } + ) + for path, description in required.items(): + require_file(path, description) + forbidden = ( + BuildConstants.REPO_ROOT / "third_party" / "linux", + BuildConstants.REPO_ROOT / "third_party" / "alpine-sources", + BuildConstants.REPO_ROOT / "third_party" / "ubuntu-sources", + ) + present = [str(path) for path in forbidden if path.exists()] + if present: + raise ScriptError( + "generated third-party sources must not be checked out here: " + + ", ".join(present) + ) + config_path = BuildConstants.REPO_ROOT / "kernel" / "config-microvm" + config = config_path.read_text(encoding="utf-8") + for setting in ( + *KernelBuildConstants.REQUIRED_DIRECT_BOOT_CONFIG, + "CONFIG_HVC_XE9=y", + "CONFIG_VIRTIO_FS=y", + "CONFIG_FUSE_FS=y", + *KernelBuildConstants.REQUIRED_SANDBOX_CONFIG, + *KernelBuildConstants.REQUIRED_NETWORK_CONFIG, + *KernelBuildConstants.REQUIRED_TIME_ABI_CONFIG, + *KernelBuildConstants.REQUIRED_HARDENING_CONFIG, + ): + if setting not in config.splitlines(): + raise ScriptError(f"{config_path} is missing {setting}") + hvc_patch_text = ( + BuildConstants.REPO_ROOT / "kernel" / "patches" / "0002-microvm-hvc-xe9.patch" + ).read_text(encoding="utf-8") + for marker in ("config HVC_XE9", "hvc_xe9.o", "hvc_xe9.c"): + if marker not in hvc_patch_text: + raise ScriptError(f"xe9 HVC patch is missing {marker}") + generated_config = artifact_path(KernelBuildConstants.CONFIG_NAME) + if generated_config.is_file(): + generated = generated_config.read_text(encoding="utf-8").splitlines() + for setting in ( + *KernelBuildConstants.REQUIRED_DIRECT_BOOT_CONFIG, + "CONFIG_HVC_XE9=y", + *KernelBuildConstants.REQUIRED_SANDBOX_CONFIG, + *KernelBuildConstants.REQUIRED_NETWORK_CONFIG, + ): + if setting not in generated: + raise ScriptError(f"{generated_config} is missing {setting}") + head = subprocess.run( + ["git", "-C", OpenVMMBuildConstants.DIRECTORY, "rev-parse", "HEAD"], + check=True, + capture_output=True, + text=True, + ).stdout.strip() + expected = subprocess.run( + ["git", "-C", BuildConstants.REPO_ROOT, "rev-parse", ":openvmm"], + check=True, + capture_output=True, + text=True, + ).stdout.strip() + if head != expected: + raise ScriptError(f"OpenVMM submodule is at {head}, expected {expected}") + print(">> source tree and submodule metadata are consistent") diff --git a/scripts/nvx497-test.py b/scripts/nvx497-test.py new file mode 100644 index 000000000..4a82421e2 --- /dev/null +++ b/scripts/nvx497-test.py @@ -0,0 +1,21503 @@ +#!/usr/bin/env python3 +# pyright: reportPrivateUsage=false + +import argparse +import ast +import contextlib +import errno +import hashlib +import http.client +import http.server +import io +import json +import lzma +import os +import queue +import re +import select +import shlex +import shutil +import signal +import struct +import subprocess +import sys +import tarfile +import tempfile +import threading +import time +import unittest +import urllib.error +import urllib.request +import uuid +import zipfile +from collections.abc import Callable +from pathlib import Path, PurePosixPath +from typing import Any, cast +from unittest.mock import MagicMock, call, patch + +sys.path.insert(0, str(Path(__file__).parent)) +import nvx # noqa: E402 +from nvx_tools import ( # noqa: E402 + aci_edge_sandboxes_tests, + archive, + azurelinux, + benchmark, + build, + build_config, + ci, + collect_alpine_sources, + collect_ubuntu_sources, + common, + control_session, + guests, + release, + sandbox, + sandbox_lifecycle, + ubuntu, +) +from nvx_tools.build_constants import ( # noqa: E402 + AlpineBuildConstants, + AzureLinuxBuildConstants, + BuildConstants, + DockerBuildConstants, + InitramfsBuildConstants, + KernelBuildConstants, + OpenVMMBuildConstants, + ReleaseBuildConstants, + UbuntuBuildConstants, + ZstdBuildConstants, +) +from nvx_tools.time_abi import ( # noqa: E402 + CPU_GENERATIONS, + OPENVMM_CPU_PROFILES, + CpuGeneration, + CpuModel, + HostCpu, + cpu_profile_request_url, +) + + +def _workflow_job(workflow: str, job_name: str) -> str: + lines = workflow.splitlines() + start = lines.index(f" {job_name}:") + end = next( + ( + index + for index, line in enumerate(lines[start + 1 :], start + 1) + if len(line) > 2 and line[:2] == " " and not line[2].isspace() + ), + len(lines), + ) + return "\n".join(lines[start:end]) + + +def _composite_action_step(action: str, step_name: str) -> str: + lines = action.splitlines() + start = lines.index(f" - name: {step_name}") + end = next( + ( + index + for index, line in enumerate(lines[start + 1 :], start + 1) + if line.startswith(" - ") + ), + len(lines), + ) + return "\n".join(lines[start:end]) + + +def _composite_action_steps(action: str) -> list[str]: + lines = action.splitlines() + starts = [index for index, line in enumerate(lines) if line.startswith(" - ")] + return [ + "\n".join(lines[start:end]) + for start, end in zip(starts, starts[1:] + [len(lines)], strict=True) + ] + + +def _composite_action_script(action: str, step_name: str) -> str: + lines = _composite_action_step(action, step_name).splitlines() + start = lines.index(" run: |") + 1 + end = next( + ( + index + for index, line in enumerate(lines[start:], start) + if line and not line.startswith(" ") + ), + len(lines), + ) + return "\n".join(line[8:] for line in lines[start:end]) + + +def _workflow_step(workflow: str, job_name: str, step_name: str) -> str: + lines = _workflow_job(workflow, job_name).splitlines() + start = lines.index(f" - name: {step_name}") + end = next( + ( + index + for index, line in enumerate(lines[start + 1 :], start + 1) + if line.startswith(" - ") + ), + len(lines), + ) + return "\n".join(lines[start:end]) + + +def _workflow_steps(workflow: str, job_name: str) -> list[str]: + lines = _workflow_job(workflow, job_name).splitlines() + starts = [index for index, line in enumerate(lines) if line.startswith(" - ")] + return [ + "\n".join(lines[start:end]) + for start, end in zip(starts, starts[1:] + [len(lines)], strict=True) + ] + + +def _yaml_field(configuration: str, field: str) -> list[str]: + """Return the first value of a field, with one entry per literal block line.""" + lines = configuration.splitlines() + for index, line in enumerate(lines): + stripped = line.lstrip() + if not stripped.startswith(f"{field}: "): + continue + value = stripped[len(field) + 2 :].rstrip() + if value != "|": + return [value] + indent = len(line) - len(stripped) + values: list[str] = [] + for block_line in lines[index + 1 :]: + block_value = block_line.strip() + if block_value and len(block_line) - len(block_line.lstrip()) <= indent: + break + if block_value: + values.append(block_value) + return values + raise AssertionError(f"configuration has no {field} field") + + +def _yaml_scalar(configuration: str, field: str) -> str: + lines = configuration.splitlines() + for index, line in enumerate(lines): + stripped = line.lstrip() + if stripped.startswith("- "): + stripped = stripped[2:] + if not stripped.startswith(f"{field}: "): + continue + value = stripped[len(field) + 2 :].rstrip() + indent = len(line) - len(line.lstrip()) + if line.lstrip().startswith("- "): + indent += 2 + values: list[str] = [] if value in {">", ">-", "|", "|-"} else [value] + for block_line in lines[index + 1 :]: + block_value = block_line.strip() + if block_value and len(block_line) - len(block_line.lstrip()) <= indent: + break + if block_value: + values.append(block_value) + return " ".join(values) + raise AssertionError(f"configuration has no {field} field") + + +def _outer_parentheses_enclose(expression: str) -> bool: + expression = expression.strip() + if not expression.startswith("("): + return False + depth = 0 + quote: str | None = None + for index, character in enumerate(expression): + if quote is not None: + if character == quote and (index == 0 or expression[index - 1] != "\\"): + quote = None + continue + if character in {"'", '"'}: + quote = character + elif character == "(": + depth += 1 + elif character == ")": + depth -= 1 + if depth == 0 and index != len(expression) - 1: + return False + if depth < 0: + return False + return depth == 0 and quote is None + + +def _write_release_fixture( + root: Path, +) -> tuple[dict[str, Path], dict[str, object], str]: + build_dir = root / "build" + source_dir = build_dir / "sources" + openvmm_dir = root / "openvmm" + binary_name = "openvmm.exe" if os.name == "nt" else "openvmm" + binary = openvmm_dir / "target" / "release" / binary_name + revision = "0bc357bbcf3a654b63dfb51f1103c5751bf3d31f" + guest_names = ReleaseBuildConstants.GUEST_ARTIFACT_NAMES + source_files = ( + set(build._initramfs_source_files()) + | set(ubuntu.customization_files()) + | set(azurelinux.input_files()) + ) + for source in source_files: + destination = root / source.relative_to(BuildConstants.REPO_ROOT) + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(source, destination) + for path in ( + binary, + *(build_dir / name for name in guest_names), + root / "LICENSE", + root / "README.md", + root / "THIRD_PARTY_NOTICES.md", + openvmm_dir / "LICENSE", + root / "kernel" / "COPYING-LINUX", + ): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(path.name.encode("ascii")) + for artifact_name, manifest_name in ( + ( + "initramfs-ubuntu.cpio.gz", + "initramfs-ubuntu.cpio.gz.packages.json", + ), + ("ubuntu-distro.erofs", "ubuntu-distro.erofs.manifest.json"), + ): + artifact = build_dir / artifact_name + input_sha256 = ubuntu.converter_input_sha256(ubuntu.customization_files()) + (build_dir / manifest_name).write_text( + json.dumps( + { + "format": 1, + "guest": "ubuntu", + "artifact": artifact.name, + "artifact_sha256": common.sha256_file(artifact), + "input_sha256": input_sha256, + "packages": [], + } + ), + encoding="utf-8", + ) + (build_dir / AzureLinuxBuildConstants.PACKAGE_MANIFEST_NAME).write_text( + json.dumps( + { + "format": AzureLinuxBuildConstants.PACKAGE_MANIFEST_VERSION, + "guest": AzureLinuxBuildConstants.GUEST_NAME, + "artifact": AzureLinuxBuildConstants.INITRAMFS_NAME, + "artifact_sha256": common.sha256_file( + build_dir / AzureLinuxBuildConstants.INITRAMFS_NAME + ), + "package_manifest_format": ( + AzureLinuxBuildConstants.PACKAGE_MANIFEST_FORMAT + ), + "image": AzureLinuxBuildConstants.IMAGE, + "input_sha256": azurelinux.input_sha256(), + } + ), + encoding="utf-8", + ) + generated_config = build_dir / "vmlinux.config" + generated_config.write_text( + "\n".join( + ( + *KernelBuildConstants.REQUIRED_DIRECT_BOOT_CONFIG, + *KernelBuildConstants.REQUIRED_VIRTIO_CONSOLE_CONFIG, + *KernelBuildConstants.REQUIRED_SHARED_STATUS_CONFIG, + *KernelBuildConstants.REQUIRED_SANDBOX_CONFIG, + *KernelBuildConstants.REQUIRED_NETWORK_CONFIG, + *KernelBuildConstants.REQUIRED_TIME_ABI_CONFIG, + *KernelBuildConstants.REQUIRED_HARDENING_CONFIG, + ) + ) + + "\n", + encoding="utf-8", + ) + manifest = { + "format": 1, + "distribution": {"name": "nvx", "version": "0.1.0"}, + "openvmm": { + "microvm_abi_version": 2, + "control_session_protocol_version": 1, + "control_contract_revision": "nvx-microvm-v2-control-v2", + }, + "linux": { + "version": "6.18.38", + "upstream_url": ( + "https://cdn.kernel.org/pub/linux/kernel/v6.x/linux-6.18.38.tar.xz" + ), + "upstream_archive_sha256": KernelBuildConstants.SHA256, + "source_cache": ".cache/linux/linux-6.18.38", + "source_archive": ("build/sources/linux/nvx-linux-source-6.18.38.tar.gz"), + "generated_final_config": "build/vmlinux.config", + "input_config": "kernel/config-microvm", + "patches": ["kernel/patches/example.patch"], + }, + "alpine": { + "version": "3.24.1", + "branch": "v3.24", + "architecture": "x86_64", + "minirootfs_url": ( + "https://dl-cdn.alpinelinux.org/alpine/v3.24/releases/" + "x86_64/alpine-minirootfs-3.24.1-x86_64.tar.gz" + ), + "minirootfs_sha256": AlpineBuildConstants.MINIROOTFS_SHA256, + "guest_sources": ["guest/common", "guest/alpine"], + "package_manifests": ["build/initramfs.cpio.gz.packages.json"], + "source_output": "build/sources/alpine", + }, + "ubuntu": { + "distribution": "Ubuntu Base", + "version": UbuntuBuildConstants.VERSION, + "codename": UbuntuBuildConstants.CODENAME, + "architecture": UbuntuBuildConstants.ARCHITECTURE, + "base_url": UbuntuBuildConstants.BASE_URL, + "base_sha256": UbuntuBuildConstants.BASE_SHA256, + "archive_keyring_url": (UbuntuBuildConstants.ARCHIVE_KEYRING_URL), + "archive_keyring_sha256": (UbuntuBuildConstants.ARCHIVE_KEYRING_SHA256), + "package_lock": "ubuntu/packages.lock.json", + "package_lock_sha256": ubuntu.package_lock_sha256(), + "guest_sources": [ + "guest/common", + "guest/ubuntu", + "ubuntu/packages.lock.json", + ], + "package_manifests": [ + "build/initramfs-ubuntu.cpio.gz.packages.json", + "build/ubuntu-distro.erofs.manifest.json", + ], + "source_output": "build/sources/ubuntu", + "erofs_converter_format": UbuntuBuildConstants.EROFS_FORMAT, + }, + "azurelinux": { + "distribution": "Azure Linux", + "version": AzureLinuxBuildConstants.VERSION, + "architecture": AzureLinuxBuildConstants.ARCHITECTURE, + "image": AzureLinuxBuildConstants.IMAGE, + "package_lock": "azurelinux/packages.lock.json", + "package_lock_sha256": azurelinux.package_lock_sha256(), + "guest_sources": [ + *( + path.as_posix() + for path in AzureLinuxBuildConstants.GUEST_SOURCE_DIRECTORIES + ), + "azurelinux/packages.lock.json", + ], + "package_manifests": [ + ( + Path(BuildConstants.BUILD_DIRECTORY_NAME) + / AzureLinuxBuildConstants.PACKAGE_MANIFEST_NAME + ).as_posix() + ], + }, + } + (root / "SOURCE-MANIFEST.json").write_text( + json.dumps(manifest), + encoding="utf-8", + ) + kernel_inputs: dict[str, object] = { + "source": { + "version": "6.18.38", + "patches": [ + { + "path": "kernel/patches/example.patch", + "sha256": "2" * 64, + } + ], + }, + "input_config": { + "path": "kernel/config-microvm", + "sha256": "1" * 64, + }, + } + (build_dir / OpenVMMBuildConstants.PROVENANCE_NAME).write_text( + json.dumps( + { + "format": 1, + "source_revision": revision, + "source_clean": True, + "executable_sha256": common.sha256_file(binary), + } + ), + encoding="utf-8", + ) + (build_dir / KernelBuildConstants.PROVENANCE_NAME).write_text( + json.dumps( + { + "format": 1, + **kernel_inputs, + "kernel_sha256": common.sha256_file(build_dir / "vmlinux"), + "config_sha256": common.sha256_file(generated_config), + } + ), + encoding="utf-8", + ) + (build_dir / InitramfsBuildConstants.PROVENANCE_NAME).write_text( + json.dumps( + { + "format": 1, + "inputs": build.initramfs_provenance_inputs(), + "initramfs_sha256": common.sha256_file(build_dir / "initramfs.cpio.gz"), + "package_manifest_sha256": common.sha256_file( + build_dir / "initramfs.cpio.gz.packages.json" + ), + } + ), + encoding="utf-8", + ) + paths = { + "build": build_dir, + "source": source_dir, + "openvmm": openvmm_dir, + "binary": binary, + } + return paths, kernel_inputs, revision + + +class CliTests(unittest.TestCase): + def test_usage_documents_every_command_and_option(self): + # doc/usage.md is the CLI reference: every command has a row in its + # command table and a section that names exactly the options of the + # command's --help usage. + option = re.compile(r"(? str: + stdout = io.StringIO() + with contextlib.redirect_stdout(stdout), self.assertRaises(SystemExit): + nvx.parse_args([*command, "--help"]) + return stdout.getvalue().split("\n\n", 1)[0] + + def subcommands(usage: str) -> list[str]: + # A subcommand list reads "{a,b} ...", which argparse may wrap + # before the dots; an option's repeated choices read + # "{a,b} [{a,b} ...]". + match = re.search(r"(? str: + match = re.search( + rf"^{'#' * level} `{re.escape(title)}`\n(.*?)(?=^#{{2,{level}}} |\Z)", + document, + re.MULTILINE | re.DOTALL, + ) + self.assertIsNotNone(match, f"doc/usage.md has no section for {title}") + return match.group(1) if match else "" + + table = document.split("## Commands\n", 1)[1].split("\n## ", 1)[0] + commands = subcommands(usage_of()) + self.assertEqual( + set(re.findall(r"^\| `([a-z0-9-]+)` \|", table, re.MULTILINE)), + set(commands), + ) + for name in commands: + usage = usage_of(name) + nested = subcommands(usage) + section(name, 3) + targets = ( + [((name, child), 4) for child in nested] if nested else [((name,), 3)] + ) + for command, level in targets: + title = " ".join(command) + with self.subTest(command=title): + self.assertEqual( + set(option.findall(section(title, level))), + set(option.findall(usage_of(*command))), + ) + + def test_guest_selection_includes_azure_linux(self): + args = nvx.parse_args(["build-initramfs", "--guest", "azurelinux"]) + self.assertEqual(args.guest, "azurelinux") + self.assertEqual( + guests.guest_descriptor(args.guest).initramfs_name, + "initramfs-azurelinux.cpio.gz", + ) + + def test_run_help_and_usage_state_each_guest_memory_default(self): + stdout = io.StringIO() + with contextlib.redirect_stdout(stdout), self.assertRaises(SystemExit): + nvx.parse_args(["run", "--help"]) + help_text = " ".join(stdout.getvalue().split()) + usage = (BuildConstants.REPO_ROOT / "doc" / "usage.md").read_text( + encoding="utf-8" + ) + row = next( + line + for line in usage.splitlines() + if line.startswith("| `--memory-mib MIB`") + ) + labels = {"alpine": "Alpine", "ubuntu": "Ubuntu", "azurelinux": "Azure Linux"} + for name in guests.GUEST_NAMES: + default = guests.guest_descriptor(name).default_memory_mib + with self.subTest(guest=name): + self.assertIn(f"{name} {default}", help_text) + self.assertIn(f"{default} for {labels[name]}", row) + + def test_benchmark_exposes_device_restore_profile(self): + args = nvx.parse_args( + [ + "benchmark", + "--suite", + "device-restore-profile", + "--restore-devices", + "console", + "virtiofs", + "--restore-modes", + "deferred", + "--output-dir", + "results", + ] + ) + + benchmark.apply_benchmark_suite_defaults(args) + self.assertEqual(args.restore_devices, ["console", "virtiofs"]) + self.assertEqual(args.restore_modes, ["deferred"]) + self.assertEqual(args.warmups, 1) + self.assertEqual(args.runs, 5) + self.assertEqual(args.output_dir, Path("results")) + self.assertIs(args.handler, nvx.command_benchmark) + + def test_benchmark_exposes_non_python_performance_suite(self): + args = nvx.parse_args( + [ + "benchmark", + "--suite", + "performance", + "--shell-memories", + "64", + "256", + "--output-dir", + "results", + ] + ) + + self.assertEqual(args.suite, "performance") + self.assertEqual(args.shell_memories, [64, 256]) + self.assertEqual(args.virtfs_runs, 3) + self.assertEqual(args.payload_mib, 64) + self.assertEqual(args.network_memory_mib, 256) + self.assertEqual(args.host_cpu_reserve, 2) + self.assertEqual(args.output_dir, Path("results")) + self.assertIs(args.handler, nvx.command_benchmark) + + def test_device_io_uses_canonical_sampling_defaults(self): + args = nvx.parse_args(["benchmark", "--suite", "device-io"]) + + benchmark.apply_benchmark_suite_defaults(args) + + self.assertEqual(args.warmups, 5) + self.assertEqual(args.runs, 30) + self.assertEqual(args.device_io_duration_seconds, 10.0) + self.assertEqual(args.device_io_size_mib, 512) + self.assertEqual(args.device_io_port, 5201) + self.assertFalse(hasattr(args, "device_io_abi")) + + with self.assertRaises(SystemExit): + nvx.parse_args( + ["benchmark", "--suite", "device-io", "--device-io-abi", "1"] + ) + + def test_benchmark_exposes_restore_only_shell_suite(self): + args = nvx.parse_args( + [ + "benchmark", + "--suite", + "shell-snapshot-restore", + "--shell-memories", + "512", + "--processors", + "8", + "--output-dir", + "results", + ] + ) + + self.assertEqual(args.suite, "shell-snapshot-restore") + self.assertEqual(args.shell_memories, [512]) + self.assertEqual(args.processors, 8) + self.assertIs(args.handler, nvx.command_benchmark) + + def test_benchmark_exposes_restore_vcpu_matrix(self): + args = nvx.parse_args( + [ + "benchmark", + "--suite", + "snapshot-restore-vcpu", + "--processors", + "8", + "--memory-mib", + "512", + "--output-dir", + "results", + ] + ) + + self.assertEqual(args.suite, "snapshot-restore-vcpu") + self.assertEqual(args.processors, 8) + self.assertEqual(args.memory_mib, 512) + self.assertIs(args.handler, nvx.command_benchmark) + + def test_benchmark_exposes_snapshot_profile_matrix(self): + args = nvx.parse_args( + [ + "benchmark", + "--suite", + "snapshot-profile", + "--backend", + "mshv", + "--cache-state", + "cold", + ] + ) + + self.assertEqual(args.suite, "snapshot-profile") + self.assertIsNone(args.shell_memories) + benchmark.apply_benchmark_suite_defaults(args) + self.assertEqual(args.shell_memories, [128, 256, 512, 1024]) + self.assertEqual(args.cache_state, "cold") + self.assertFalse(args.snapshot_profile) + self.assertIs(args.handler, nvx.command_benchmark) + + def test_benchmark_preserves_canonical_shell_memory_defaults(self): + args = nvx.parse_args(["benchmark", "--suite", "performance"]) + + benchmark.apply_benchmark_suite_defaults(args) + + self.assertEqual(args.shell_memories, [128, 256, 512]) + + def test_benchmark_positive_integer_diagnostics_are_stable(self): + for value, message in ( + ("not-an-int", "argument --runs: invalid positive_int value: 'not-an-int'"), + ("0", "argument --runs: must be at least 1"), + ): + with self.subTest(value=value): + with ( + patch("sys.stderr", new_callable=io.StringIO) as errors, + self.assertRaises(SystemExit) as exit_context, + ): + nvx.parse_args(["benchmark", "--runs", value]) + + self.assertEqual(exit_context.exception.code, 2) + self.assertIn(message, errors.getvalue()) + + def test_release_commands_keep_their_cli_contract(self): + download = nvx.parse_args( + ["download", "--repository", "example/nvx", "--hypervisor", "auto"] + ) + self.assertEqual(download.command, "download") + self.assertEqual(download.repository, "example/nvx") + self.assertEqual(download.hypervisor, "auto") + self.assertIs(download.handler, nvx.command_download) + + collect = nvx.parse_args(["collect-sources"]) + self.assertEqual(collect.command, "collect-sources") + self.assertIs(collect.handler, nvx.command_collect_sources) + + package = nvx.parse_args( + [ + "package", + "--version", + "1.2.3", + "--destination", + "output", + "--include-source", + "--force", + ] + ) + self.assertEqual(package.command, "package") + self.assertEqual(package.version, "1.2.3") + self.assertEqual(package.destination, Path("output")) + self.assertTrue(package.include_source) + self.assertFalse(package.binary_only) + self.assertTrue(package.force) + self.assertIs(package.handler, nvx.command_package) + + verify = nvx.parse_args(["verify"]) + self.assertEqual(verify.command, "verify") + self.assertIs(verify.handler, nvx.command_verify) + + openvmm_unit_tests = nvx.parse_args(["test-openvmm-unit"]) + self.assertEqual(openvmm_unit_tests.command, "test-openvmm-unit") + self.assertIs(openvmm_unit_tests.handler, nvx.command_test_openvmm_unit) + + openvmm_tests = nvx.parse_args(["test-openvmm", "--backend", "mshv"]) + self.assertEqual(openvmm_tests.backend, "mshv") + self.assertIs(openvmm_tests.handler, nvx.command_test_openvmm) + + def test_openvmm_help_describes_petri_vmm_tests(self): + with ( + patch("sys.stdout", new_callable=io.StringIO) as output, + self.assertRaises(SystemExit) as exit_context, + ): + nvx.parse_args(["--help"]) + + self.assertEqual(exit_context.exception.code, 0) + help_text = output.getvalue() + self.assertIn("run OpenVMM Petri VMM tests", help_text) + self.assertNotIn("run OpenVMM microVM integration tests", help_text) + + def test_guest_selection_cli_contract(self): + default_build = nvx.parse_args(["build-guest"]) + self.assertEqual(default_build.guest, "alpine") + self.assertFalse(default_build.debug_kernel) + + debug_guest = nvx.parse_args(["build-guest", "--debug-kernel"]) + self.assertTrue(debug_guest.debug_kernel) + self.assertTrue(nvx._build_config(debug_guest).debug_kernel) + self.assertTrue(nvx.parse_args(["build", "--debug-kernel"]).debug_kernel) + + self.assertFalse(nvx.parse_args(["build-kernel"]).debug) + self.assertTrue(nvx.parse_args(["build-kernel", "--debug"]).debug) + + all_guests = nvx.parse_args(["build-guest", "--guest", "all"]) + self.assertEqual(all_guests.guest, "all") + + ubuntu_initramfs = nvx.parse_args(["build-initramfs", "--guest", "ubuntu"]) + self.assertEqual(ubuntu_initramfs.guest, "ubuntu") + + azurelinux_initramfs = nvx.parse_args( + ["build-initramfs", "--guest", "azurelinux"] + ) + with patch("nvx.build_docker_initramfs") as build_docker_initramfs: + nvx.command_build_initramfs(azurelinux_initramfs) + build_docker_initramfs.assert_called_once_with( + build.DockerBuildConfig(artifact_destination=BuildConstants.BUILD_DIR), + "azurelinux", + ) + + distro = nvx.parse_args( + [ + "build-distro-layer", + "--guest", + "ubuntu", + "--output", + "ubuntu.erofs", + "--replace", + ] + ) + self.assertEqual(distro.guest, "ubuntu") + self.assertEqual(distro.output, Path("ubuntu.erofs")) + self.assertTrue(distro.replace) + + with self.assertRaises(SystemExit): + nvx.parse_args(["run", "--guest", "all"]) + + def test_docker_initramfs_rejects_native_guest(self): + with self.assertRaisesRegex( + common.ScriptError, "Alpine Linux.*do not require Docker" + ): + build.build_docker_initramfs(build.DockerBuildConfig(), "alpine") + + def test_docker_initramfs_requires_expected_artifacts(self): + with tempfile.TemporaryDirectory() as temporary: + config = build.DockerBuildConfig(artifact_destination=Path(temporary)) + with ( + patch.object(build, "require_tool"), + patch.object(build, "run_checked") as run_checked, + self.assertRaisesRegex( + common.ScriptError, + "initramfs-azurelinux.cpio.gz", + ), + ): + build.build_docker_initramfs(config, "azurelinux") + command = run_checked.call_args.args[0] + self.assertEqual( + command[command.index("--target") + 1], + "azurelinux-initramfs-artifacts", + ) + + def test_docker_artifacts_require_expected_files(self): + with tempfile.TemporaryDirectory() as temporary: + config = build.DockerBuildConfig(artifact_destination=Path(temporary)) + with ( + patch.object(build, "require_tool"), + patch.object(build, "run_checked"), + self.assertRaisesRegex(common.ScriptError, "vmlinux"), + ): + build.build_docker_artifacts(config, "azurelinux") + + def test_ubuntu_run_selects_artifact_and_default_memory(self): + args = nvx.parse_args(["run", "--guest", "ubuntu", "--dry-run"]) + + def require(path: Path, _description: str) -> Path: + return path + + with ( + patch.object(nvx, "require_file", side_effect=require), + patch.object( + nvx, + "_format_command", + return_value="formatted", + ) as format_command, + ): + nvx.command_run(args) + + command = format_command.call_args.args[0] + self.assertEqual(command[command.index("--memory") + 1], "512M") + self.assertEqual( + Path(command[command.index("--initrd") + 1]).name, + "initramfs-ubuntu.cpio.gz", + ) + + def test_azurelinux_run_selects_artifact_and_default_memory(self): + args = nvx.parse_args(["run", "--guest", "azurelinux", "--dry-run"]) + + def require(path: Path, _description: str) -> Path: + return path + + with ( + patch.object(nvx, "require_file", side_effect=require), + patch.object( + nvx, + "_format_command", + return_value="formatted", + ) as format_command, + ): + nvx.command_run(args) + + command = format_command.call_args.args[0] + self.assertEqual(command[command.index("--memory") + 1], "512M") + self.assertEqual( + Path(command[command.index("--initrd") + 1]).name, + "initramfs-azurelinux.cpio.gz", + ) + + def test_restore_rejects_ubuntu_guest_selection(self): + args = nvx.parse_args( + [ + "run", + "--guest", + "ubuntu", + "--restore-snapshot", + "snapshot", + "--dry-run", + ] + ) + with self.assertRaisesRegex(common.ScriptError, "snapshot already fixes"): + nvx.command_run(args) + + def test_sandbox_rejects_systemd_entrypoint(self): + for entrypoint in nvx.SYSTEMD_ENTRYPOINTS: + with self.subTest(entrypoint=entrypoint): + args = nvx.parse_args( + [ + "sandbox", + "--entrypoint", + entrypoint, + "--layer", + "distro,distro.erofs,11111111-1111-1111-1111-111111111111", + "--scratch", + "scratch.ext4", + ] + ) + with self.assertRaisesRegex( + common.ScriptError, + "systemd entrypoints", + ): + nvx.command_sandbox(args) + + def test_sandbox_allows_non_systemd_sbin_init(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + layer = root / "distro.erofs" + scratch = root / "scratch.ext4" + layer.write_bytes(b"distro") + scratch.write_bytes(b"scratch") + args = nvx.parse_args( + [ + "sandbox", + "--entrypoint", + "/sbin/init", + "--layer", + f"distro,{layer},11111111-1111-1111-1111-111111111111", + "--scratch", + str(scratch), + "--dry-run", + ] + ) + + def require(path: Path, _description: str) -> Path: + return path + + with ( + patch.object(nvx, "require_file", side_effect=require), + patch.object(nvx, "_format_command", return_value="formatted"), + ): + nvx.command_sandbox(args) + + def test_sandbox_rejects_systemd_from_bound_distro_metadata(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + layer = root / "distro.erofs" + scratch = root / "scratch.ext4" + layer.write_bytes(b"distro") + scratch.write_bytes(b"scratch") + layer.with_name(f"{layer.name}.manifest.json").write_text( + json.dumps( + { + "format": 1, + "artifact": layer.name, + "artifact_sha256": common.sha256_file(layer), + "packages": [{"name": "systemd"}], + } + ), + encoding="utf-8", + ) + args = nvx.parse_args( + [ + "sandbox", + "--entrypoint", + "/sbin/init", + "--layer", + f"distro,{layer},11111111-1111-1111-1111-111111111111", + "--scratch", + str(scratch), + ] + ) + with self.assertRaisesRegex(common.ScriptError, "systemd images"): + nvx.command_sandbox(args) + + def test_sandbox_command_parses_typed_launch_contract(self): + args = nvx.parse_args( + [ + "sandbox", + "--layer", + "distro,distro.erofs,11111111-1111-1111-1111-111111111111", + "--scratch", + "scratch.ext4", + "--entrypoint", + "/bin/workload", + "--arg=--serve", + "--memory-max", + "268435456", + "--pids-max", + "64", + "--workload-user", + "1000:1001", + ] + ) + + self.assertEqual(args.layer[0].role, "distro") + self.assertEqual(args.scratch, Path("scratch.ext4")) + self.assertEqual(args.sandbox_arg, ["--serve"]) + self.assertEqual(args.memory_max, 268435456) + self.assertEqual(args.pids_max, 64) + self.assertEqual(args.workload_user, (1000, 1001)) + self.assertIs(args.handler, nvx.command_sandbox) + + def test_sandbox_command_parses_state_aware_operations(self): + provision = nvx.parse_args( + [ + "sandbox", + "provision", + "--state-dir", + "state", + "--layer", + "distro,distro.erofs,11111111-1111-1111-1111-111111111111", + "--scratch", + "scratch.ext4", + ] + ) + self.assertEqual(provision.sandbox_operation, "provision") + self.assertEqual(provision.state_dir, Path("state")) + + execute = nvx.parse_args( + [ + "sandbox", + "exec", + "--state-dir", + "state", + "--entrypoint", + "/bin/sh", + "--arg=-c", + "--arg=echo managed", + "--exec-timeout-ms", + "5000", + ] + ) + self.assertEqual(execute.sandbox_operation, "exec") + self.assertEqual(execute.sandbox_arg, ["-c", "echo managed"]) + self.assertEqual(execute.exec_timeout_ms, 5000) + self.assertFalse(execute.inherit_default_environment) + + configured = nvx.parse_args( + [ + "sandbox", + "exec", + "--state-dir", + "state", + "--cwd", + "/work tree", + "--environment", + "EMPTY=", + "--environment", + "VALUE=space = value", + "--inherit-default-environment", + ] + ) + self.assertEqual(configured.cwd, "/work tree") + self.assertEqual( + configured.environment, + ["EMPTY=", "VALUE=space = value"], + ) + self.assertTrue(configured.inherit_default_environment) + + report = nvx.parse_args( + [ + "sandbox", + "exec", + "--state-dir", + "state", + "--outcome-report", + "exec-outcome.json", + ] + ) + self.assertEqual(report.outcome_report, Path("exec-outcome.json")) + + def test_sandbox_timeout_must_be_finite_and_positive(self): + for value in ("0", "-1", "nan", "inf"): + stderr = io.StringIO() + with ( + self.subTest(value=value), + contextlib.redirect_stderr(stderr), + self.assertRaises(SystemExit), + ): + nvx.parse_args( + ["sandbox", "start", "--state-dir", "state", "--timeout", value] + ) + self.assertIn("must be finite and greater than 0", stderr.getvalue()) + + def test_sandbox_managed_operations_reject_dry_run_before_state_access(self): + lifecycle_methods = ( + ("provision", "provision"), + ("start", "start"), + ("exec", "exec_workload"), + ("stop", "stop"), + ("deprovision", "deprovision"), + ) + with tempfile.TemporaryDirectory() as temporary: + state = Path(temporary) / "state" + state.mkdir() + marker = state / "marker" + marker.write_text("preserved", encoding="utf-8") + + for operation, lifecycle_method in lifecycle_methods: + with self.subTest(operation=operation): + args = nvx.parse_args( + [ + "sandbox", + operation, + "--state-dir", + str(state), + "--dry-run", + ] + ) + with ( + patch.object( + sandbox_lifecycle, + lifecycle_method, + ) as lifecycle, + self.assertRaisesRegex( + common.ScriptError, + "--dry-run is only valid for sandbox run", + ), + ): + nvx.command_sandbox(args) + lifecycle.assert_not_called() + self.assertEqual(marker.read_text(encoding="utf-8"), "preserved") + + def test_sandbox_exec_forwards_explicit_empty_environment_and_cwd(self): + with tempfile.TemporaryDirectory() as temporary: + environment_file = Path(temporary) / "environment.json" + environment_file.write_text("[]", encoding="utf-8") + args = nvx.parse_args( + [ + "sandbox", + "exec", + "--state-dir", + "state", + "--cwd", + "/work", + "--environment-file", + str(environment_file), + ] + ) + result = sandbox_lifecycle.ManagedExecResult(0, "exit", b"", b"") + with ( + patch.object( + sandbox_lifecycle, + "exec_workload", + return_value=result, + ) as execute, + self.assertRaises(SystemExit) as exit_context, + ): + nvx.command_sandbox(args) + + self.assertEqual(exit_context.exception.code, 0) + execute.assert_called_once_with( + Path("state"), + ("/bin/sh",), + timeout_ms=0, + response_timeout=60.0, + cwd="/work", + environment=(), + inherit_default_environment=False, + ) + + def test_sandbox_exec_forwards_environment_inheritance(self): + cases: tuple[tuple[list[str], tuple[str, ...] | None], ...] = ( + (["--environment", "A=1"], ("A=1",)), + ( + ["--environment", "A=1", "--environment", "TERM=dumb"], + ("A=1", "TERM=dumb"), + ), + # Without entries the workload gets the defaults whatever the flag says. + ([], None), + ) + for environment_args, environment in cases: + with self.subTest(environment=environment): + args = nvx.parse_args( + [ + "sandbox", + "exec", + "--state-dir", + "state", + *environment_args, + "--inherit-default-environment", + ] + ) + result = sandbox_lifecycle.ManagedExecResult(0, "exit", b"", b"") + with ( + patch.object( + sandbox_lifecycle, + "exec_workload", + return_value=result, + ) as execute, + self.assertRaises(SystemExit), + ): + nvx.command_sandbox(args) + execute.assert_called_once_with( + Path("state"), + ("/bin/sh",), + timeout_ms=0, + response_timeout=60.0, + cwd=None, + environment=environment, + inherit_default_environment=True, + ) + + def test_sandbox_exec_rejects_ambiguous_or_one_shot_environment(self): + both = nvx.parse_args( + [ + "sandbox", + "exec", + "--state-dir", + "state", + "--environment", + "A=1", + "--environment-file", + "environment.json", + ] + ) + with self.assertRaisesRegex(common.ScriptError, "mutually exclusive"): + nvx.command_sandbox(both) + + one_shot = nvx.parse_args(["sandbox", "run", "--environment", "A=1"]) + with self.assertRaisesRegex(common.ScriptError, "require.*exec"): + nvx.command_sandbox(one_shot) + + for operation in ("run", "start"): + inheriting = nvx.parse_args( + ["sandbox", operation, "--inherit-default-environment"] + ) + with ( + self.subTest(operation=operation), + self.assertRaisesRegex(common.ScriptError, "require.*exec"), + ): + nvx.command_sandbox(inheriting) + + def test_sandbox_environment_file_is_bounded_before_state_access(self): + with tempfile.TemporaryDirectory() as temporary: + path = Path(temporary) / "environment.json" + path.write_bytes(b" " * (1024 * 1024 + 1)) + args = nvx.parse_args( + [ + "sandbox", + "exec", + "--state-dir", + "absent-state", + "--environment-file", + str(path), + ] + ) + with ( + patch.object(sandbox_lifecycle, "exec_workload") as execute, + self.assertRaisesRegex(common.ScriptError, "exceeds.*byte limit"), + ): + nvx.command_sandbox(args) + execute.assert_not_called() + + def test_sandbox_environment_is_validated_before_state_access(self): + too_many = [ + value + for index in range(257) + for value in ("--environment", f"KEY{index}=value") + ] + cases = ( + (too_many, "exceeds 256 entries"), + (["--environment", "NO_EQUALS"], "non-empty KEY=VALUE"), + (["--environment", "=empty-key"], "non-empty KEY=VALUE"), + (["--environment", "KEY=embedded\0nul"], "non-empty KEY=VALUE"), + ( + ["--environment", "KEY=" + "\N{SNOWMAN}" * 1365], + "at most 4096 bytes", + ), + ( + ["--environment", "DUPLICATE=one", "--environment", "DUPLICATE=two"], + "duplicate key", + ), + ) + for environment_args, message in cases: + with self.subTest(message=message): + args = nvx.parse_args( + [ + "sandbox", + "exec", + "--state-dir", + "absent-state", + *environment_args, + ] + ) + with ( + patch.object(sandbox_lifecycle, "exec_workload") as execute, + self.assertRaisesRegex(common.ScriptError, message), + ): + nvx.command_sandbox(args) + execute.assert_not_called() + + def test_sandbox_environment_file_entries_are_validated_before_state_access(self): + with tempfile.TemporaryDirectory() as temporary: + path = Path(temporary) / "environment.json" + path.write_text( + json.dumps(["DUPLICATE=one", "DUPLICATE=two"]), + encoding="utf-8", + ) + args = nvx.parse_args( + [ + "sandbox", + "exec", + "--state-dir", + "absent-state", + "--environment-file", + str(path), + ] + ) + with ( + patch.object(sandbox_lifecycle, "exec_workload") as execute, + self.assertRaisesRegex(common.ScriptError, "duplicate key"), + ): + nvx.command_sandbox(args) + execute.assert_not_called() + + def test_network_requires_explicit_portable_profile(self): + args = nvx.parse_args( + [ + "run", + "--net", + "10.0.0.2/24", + "--network-profile", + "portable", + ] + ) + self.assertEqual(args.network_profile, "portable") + + missing_profile = nvx.parse_args(["run", "--net", "10.0.0.2/24"]) + with self.assertRaisesRegex(common.ScriptError, "--net and --network-profile"): + nvx.command_run(missing_profile) + + missing_network = nvx.parse_args(["run", "--network-profile", "portable"]) + with self.assertRaisesRegex(common.ScriptError, "--net and --network-profile"): + nvx.command_run(missing_network) + + def test_run_and_sandbox_forward_network_arguments(self): + network_arguments = ( + "--net 10.0.0.2/24 --network-profile portable " + "--network-egress deny --network-ingress deny " + "--network-egress-allow 140.82.112.0/20:tcp:443 " + "--network-egress-deny 10.0.0.0/8 --host-loopback allow " + "--network-proxy 10.0.0.1:3128 --host-loopback-forward tcp:8080:80" + ).split() + commands = [ + ["run", "--dry-run", *network_arguments], + [ + "sandbox", + "--dry-run", + "--layer", + "distro,distro.erofs,11111111-1111-1111-1111-111111111111", + "--scratch", + "scratch.ext4", + *network_arguments, + ], + ] + + def require(path: Path, _description: str) -> Path: + return path + + for arguments in commands: + with self.subTest(command=arguments[0]): + args = nvx.parse_args(arguments) + with ( + patch.object(nvx, "require_file", side_effect=require), + patch.object(sandbox, "require_file", side_effect=require), + patch.object( + nvx, "_format_command", return_value="formatted" + ) as format_command, + ): + args.handler(args) + + command = format_command.call_args.args[0] + self.assertEqual( + command[command.index("--net") :], + network_arguments, + ) + + def test_run_lowers_structured_egress_policy_before_launch(self): + with tempfile.TemporaryDirectory() as temporary: + policy = Path(temporary) / "policy.json" + policy.write_text( + json.dumps( + { + "allow": [ + { + "to": [ + { + "cidr": "192.0.2.0/24", + "except": ["192.0.2.128/25"], + } + ], + "ports": [ + { + "protocol": "tcp", + "port": 8000, + "endPort": 8001, + } + ], + }, + {"to": [{"cidr": "192.0.2.200/32"}]}, + ], + "deny": [ + { + "to": [{"cidr": "192.0.2.0/24"}], + "ports": [{"protocol": "tcp", "port": 8001}], + } + ], + } + ), + encoding="utf-8", + ) + args = nvx.parse_args( + [ + "run", + "--dry-run", + "--network-egress", + "deny", + "--network-egress-policy-file", + str(policy), + ] + ) + + def require(path: Path, _description: str) -> Path: + return path + + with ( + patch.object(nvx, "require_file", side_effect=require), + patch.object( + nvx, "_format_command", return_value="formatted" + ) as format_command, + ): + nvx.command_run(args) + + command = format_command.call_args.args[0] + self.assertNotIn("--network-egress-policy-file", command) + self.assertEqual(command.count("--network-egress-allow"), 2) + self.assertIn("192.0.2.0/25:tcp:8000-8001", command) + self.assertIn("192.0.2.200/32", command) + self.assertEqual(command.count("--network-egress-deny"), 1) + self.assertIn("192.0.2.0/24:tcp:8001", command) + + def test_policy_file_rejects_ambiguous_flags_before_artifact_access(self): + cases = ( + [ + "--network-egress", + "deny", + "--network-egress-policy-file", + "policy.json", + "--network-egress-allow", + "192.0.2.1", + ], + ["--network-egress-policy-file", "policy.json"], + ) + for extra in cases: + with self.subTest(extra=extra): + args = nvx.parse_args(["run", "--dry-run", *extra]) + with ( + patch.object(nvx, "require_file") as require, + self.assertRaises(common.ScriptError), + ): + nvx.command_run(args) + require.assert_not_called() + + def test_sandbox_rejects_policy_mixing_before_validating_layer_files(self): + for operation in ("run", "provision"): + with self.subTest(operation=operation): + args = nvx.parse_args( + [ + "sandbox", + operation, + "--layer", + "distro,absent.erofs,00000000-0000-4000-8000-000000000001", + "--scratch", + "absent.ext4", + "--network-egress", + "deny", + "--network-egress-policy-file", + "absent.json", + "--network-egress-allow", + "192.0.2.1", + ] + ) + with ( + patch.object(nvx.SandboxLaunch, "validated") as validated, + self.assertRaisesRegex(common.ScriptError, "cannot be combined"), + ): + nvx.command_sandbox(args) + validated.assert_not_called() + + def test_managed_provision_persists_lowered_policy_not_source_path(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + layer = root / "distro.erofs" + scratch = root / "scratch.ext4" + policy = root / "policy.json" + state = root / "state" + layer.write_bytes(b"layer") + scratch.write_bytes(b"scratch") + policy.write_text( + json.dumps( + { + "allow": [ + { + "to": [ + { + "cidr": "192.0.2.0/24", + "except": ["192.0.2.128/25"], + } + ], + } + ] + } + ), + encoding="utf-8", + ) + args = nvx.parse_args( + [ + "sandbox", + "provision", + "--state-dir", + str(state), + "--layer", + f"distro,{layer},11111111-1111-1111-1111-111111111111", + "--scratch", + str(scratch), + "--network-egress", + "deny", + "--network-egress-policy-file", + str(policy), + ] + ) + + nvx.command_sandbox(args) + config = json.loads( + (state / sandbox_lifecycle.CONFIG_NAME).read_text(encoding="utf-8") + ) + policy.unlink() + + self.assertEqual(config["network_egress"], "deny") + self.assertEqual(config["network_egress_allow"], ["192.0.2.0/25"]) + self.assertEqual(config["network_egress_deny"], []) + self.assertNotIn("network_egress_policy_file", config) + + def test_public_cli_provision_persists_policy_after_source_removal(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + layer = root / "distro.erofs" + scratch = root / "scratch.ext4" + policy = root / "policy.json" + state = root / "state" + layer.write_bytes(b"layer") + scratch.write_bytes(b"scratch") + policy.write_text( + json.dumps( + { + "allow": [ + { + "to": [ + { + "cidr": "192.0.2.0/24", + "except": ["192.0.2.128/25"], + } + ], + "ports": [ + { + "protocol": "tcp", + "port": 8000, + "endPort": 8001, + } + ], + } + ], + "deny": [ + { + "to": [{"cidr": "192.0.2.0/24"}], + "ports": [{"protocol": "tcp", "port": 8001}], + } + ], + } + ), + encoding="utf-8", + ) + result = subprocess.run( + [ + sys.executable, + str(Path(nvx.__file__).resolve()), + "sandbox", + "provision", + "--state-dir", + str(state), + "--layer", + f"distro,{layer},11111111-1111-1111-1111-111111111111", + "--scratch", + str(scratch), + "--network-egress", + "deny", + "--network-egress-policy-file", + str(policy), + ], + cwd=BuildConstants.REPO_ROOT, + capture_output=True, + timeout=10, + ) + self.assertEqual( + result.returncode, + 0, + result.stderr.decode("utf-8", "replace"), + ) + policy.unlink() + config = json.loads( + (state / sandbox_lifecycle.CONFIG_NAME).read_text(encoding="utf-8") + ) + + self.assertEqual( + config["network_egress_allow"], + ["192.0.2.0/25:tcp:8000-8001"], + ) + self.assertEqual( + config["network_egress_deny"], + ["192.0.2.0/24:tcp:8001"], + ) + self.assertNotIn(str(policy), json.dumps(config)) + + def test_invalid_policy_has_no_managed_state_side_effect(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + layer = root / "distro.erofs" + scratch = root / "scratch.ext4" + policy = root / "policy.json" + state = root / "state" + layer.write_bytes(b"layer") + scratch.write_bytes(b"scratch") + policy.write_text( + '{"allow":[{"to":[{"cidr":"192.0.2.0/24"}],' + '"ports":[{"protocol":"icmp","port":8}]}]}', + encoding="utf-8", + ) + args = nvx.parse_args( + [ + "sandbox", + "provision", + "--state-dir", + str(state), + "--layer", + f"distro,{layer},11111111-1111-1111-1111-111111111111", + "--scratch", + str(scratch), + "--network-egress", + "deny", + "--network-egress-policy-file", + str(policy), + ] + ) + + with self.assertRaises(common.ScriptError): + nvx.command_sandbox(args) + self.assertFalse(state.exists()) + + def test_managed_non_launch_operations_reject_network_policy_options(self): + args = nvx.parse_args( + [ + "sandbox", + "start", + "--state-dir", + "state", + "--network-egress", + "deny", + "--network-egress-policy-file", + "policy.json", + ] + ) + with ( + patch.object(sandbox_lifecycle, "start") as start, + self.assertRaisesRegex( + common.ScriptError, "only valid for sandbox run or provision" + ), + ): + nvx.command_sandbox(args) + start.assert_not_called() + + def test_run_parses_denied_filesystem_paths(self): + args = nvx.parse_args( + [ + "run", + "--mount", + "/mnt/share,share,rw", + "--mount-deny", + "share/secrets", + "--mount-deny", + "share/private", + ] + ) + self.assertEqual( + args.mount_deny, + [Path("share/secrets"), Path("share/private")], + ) + + def test_run_forwards_the_share_access_policy(self): + args = nvx.parse_args( + [ + "run", + "--mount", + "/mnt/share,share,rw", + "--mount-write", + "share/out", + "--mount-allow", + "share/logs/payloads", + "--mount-deny", + "share/logs", + "--dry-run", + ] + ) + with ( + patch.object(nvx, "require_file", return_value=Path("artifact")), + patch.object( + nvx, "_format_command", return_value="formatted" + ) as format_command, + ): + nvx.command_run(args) + + command = format_command.call_args.args[0] + self.assertEqual( + [ + (value, command[index + 1]) + for index, value in enumerate(command) + if value.startswith("--mount-") + ], + [ + ("--mount-deny", str(Path("share/logs"))), + ("--mount-allow", str(Path("share/logs/payloads"))), + ("--mount-write", str(Path("share/out"))), + ], + ) + + def test_run_forwards_mount_owner_and_requires_mount(self): + args = nvx.parse_args( + ["run", "--mount", "/mnt/share,share,rw", "--mount-owner", "caller"] + + ["--dry-run"] + ) + with ( + patch.object(nvx, "require_file", return_value=Path("artifact")), + patch.object( + nvx, "_format_command", return_value="formatted" + ) as format_command, + ): + if sys.platform == "win32": + with self.assertRaisesRegex(common.ScriptError, "Linux host"): + nvx.command_run(args) + format_command.assert_not_called() + else: + nvx.command_run(args) + command = format_command.call_args.args[0] + self.assertEqual(command[command.index("--mount-owner") + 1], "caller") + + default = nvx.parse_args(["run", "--mount", "/mnt/share,share,rw"]) + self.assertIsNone(default.mount_owner) + unmounted = nvx.parse_args(["run", "--mount-owner", "vmm", "--dry-run"]) + with self.assertRaisesRegex(common.ScriptError, "requires --mount"): + nvx.command_run(unmounted) + with self.assertRaises(SystemExit), contextlib.redirect_stderr(io.StringIO()): + nvx.parse_args( + ["run", "--mount", "/mnt/share,share", "--mount-owner", "root"] + ) + + def test_run_forwards_bounded_outcome_report(self): + args = nvx.parse_args(["run", "--outcome-report", "outcome.json", "--dry-run"]) + with ( + patch.object(nvx, "require_file", return_value=Path("artifact")), + patch.object( + nvx, "_format_command", return_value="formatted" + ) as format_command, + ): + nvx.command_run(args) + + command = format_command.call_args.args[0] + self.assertEqual( + command[command.index("--microvm-report") + 1], + "outcome.json", + ) + + def test_run_forwards_the_cpu_profile(self): + def launched(*extra: str) -> list[str]: + args = nvx.parse_args(["run", *extra, "--dry-run"]) + with ( + patch.object(nvx, "require_file", return_value=Path("artifact")), + patch.object( + nvx, "_format_command", return_value="formatted" + ) as format_command, + ): + nvx.command_run(args) + return format_command.call_args.args[0] + + # Without the option, OpenVMM selects auto itself. + self.assertNotIn("--cpu-profile", launched()) + for profile in ("host", "auto", "intel.alderlake.v1"): + command = launched("--cpu-profile", profile) + self.assertEqual(command[command.index("--cpu-profile") + 1], profile) + restored = launched("--restore-snapshot", "snapshot", "--cpu-profile", "host") + self.assertEqual(restored[restored.index("--cpu-profile") + 1], "host") + + def test_run_explains_a_host_cpu_that_no_profile_serves(self): + tiger_lake = HostCpu("GenuineIntel", 6, 140, 1, "Intel(R) Core(TM) i7-1185G7") + zen3 = HostCpu("AuthenticAMD", 25, 33, 0) + openvmm = "/nvx/openvmm/target/release/openvmm" + + def run( + returncode: int, + cpu_profile: str | None, + cold_boot: bool = True, + host: HostCpu | None = tiger_lake, + ) -> tuple[int, str]: + stderr = io.StringIO() + completed = subprocess.CompletedProcess[bytes]([openvmm], returncode) + with ( + patch.object(nvx.sys, "stderr", stderr), + patch.object(nvx.subprocess, "run", return_value=completed) as launch, + patch.object(nvx, "host_cpu_signature", return_value=host), + ): + status = nvx._run_openvmm( # pyright: ignore[reportPrivateUsage] + [openvmm, "--hypervisor", "kvm"], cpu_profile, cold_boot, "kvm" + ) + # OpenVMM keeps the terminal, its standard error included. + launch.assert_called_once_with([openvmm, "--hypervisor", "kvm"]) + return status, stderr.getvalue() + + status, output = run(1, None) + self.assertEqual(status, 1) + # auto falls back on the CPU, so run says so, and asks for the CPU's + # fingerprint, which this OpenVMM writes for this backend, through the + # CPU profile request form. + self.assertIn( + "nvx: no built-in CPU profile serves this host's CPU, " + "GenuineIntel 6/140/1 (Intel(R) Core(TM) i7-1185G7), so --cpu-profile " + "auto, the default, falls back on it to a CPU profile derived from " + "this host", + output, + ) + self.assertIn("alderlake 6/151 and 6/154", output) + self.assertIn( + f"nvx: {openvmm} --hypervisor kvm --cpu-fingerprint fingerprint.json\n", + output, + ) + self.assertIn(cpu_profile_request_url(tiger_lake), output) + self.assertIn("https://github.com/microsoft/nvx/issues/408", output) + self.assertNotIn("rerun with --cpu-profile host", output) + self.assertEqual(run(1, "auto"), (1, output)) + # Host profiles serve AMD CPUs too, so an AMD host that no built-in + # profile serves falls back too, with the issue that tracks profiles + # for more AMD CPUs, and a host profile that fails on it fails for + # another reason, which OpenVMM explains. + status, output = run(1, None, host=zen3) + self.assertIn("AuthenticAMD 25/33/0, so --cpu-profile auto", output) + self.assertIn(cpu_profile_request_url(zen3), output) + self.assertIn("https://github.com/microsoft/nvx/issues/409", output) + self.assertNotIn("issues/408", output) + self.assertEqual(run(1, "host", host=zen3), (1, "")) + # Host profiles serve only Intel and AMD CPUs, so another vendor's + # host still fails, with either request. + hygon = HostCpu("HygonGenuine", 24, 0, 1) + status, output = run(1, None, host=hygon) + self.assertIn( + "HygonGenuine 24/0/1, so a cold boot with --cpu-profile auto, the " + "default, fails on it with E_PROFILE_HOST_UNKNOWN", + output, + ) + self.assertIn("host CPU profiles serve only Intel and AMD CPUs", output) + self.assertNotIn("cpu-profile.yml", output) + self.assertEqual(run(1, "host", host=hygon), (1, output)) + # Successful runs, restores, other requests and CPUs, and an unknown + # CPU get no guidance. + for returncode, cpu_profile, cold_boot, host in ( + (0, None, True, tiger_lake), + (1, None, False, tiger_lake), + (1, "host", True, tiger_lake), + (1, "intel.alderlake.v1", True, tiger_lake), + (1, None, True, HostCpu("GenuineIntel", 6, 154, 3)), + (1, None, True, HostCpu("AuthenticAMD", 25, 1, 1)), + (1, None, True, None), + ): + with self.subTest( + returncode=returncode, + cpu_profile=cpu_profile, + cold_boot=cold_boot, + host=host, + ): + self.assertEqual( + run(returncode, cpu_profile, cold_boot, host), (returncode, "") + ) + + def test_run_quotes_the_openvmm_path_in_the_cpu_profile_guidance(self): + # The guidance's fingerprint command is for copying, so a path with a + # space is quoted as the shell of this platform reads it. + openvmm = str(Path("/nvx checkout") / "openvmm") + stderr = io.StringIO() + with ( + patch.object(nvx.sys, "stderr", stderr), + patch.object( + nvx.subprocess, + "run", + return_value=subprocess.CompletedProcess[bytes]([openvmm], 1), + ), + patch.object( + nvx, + "host_cpu_signature", + return_value=HostCpu("GenuineIntel", 6, 140, 1), + ), + ): + nvx._run_openvmm( # pyright: ignore[reportPrivateUsage] + [openvmm, "--hypervisor", "kvm"], None, True, "kvm" + ) + quoted = ( + subprocess.list2cmdline([openvmm]) + if os.name == "nt" + else shlex.quote(openvmm) + ) + self.assertNotEqual(quoted, openvmm) + self.assertIn( + f"nvx: {quoted} --hypervisor kvm --cpu-fingerprint fingerprint.json\n", + stderr.getvalue(), + ) + + def test_run_passes_its_backend_to_the_cpu_profile_guidance(self): + args = nvx.parse_args(["run", "--hypervisor", "kvm"]) + with ( + patch.object(nvx, "require_file", return_value=Path("artifact")), + patch.object(nvx, "_run_openvmm", return_value=0) as run_openvmm, + contextlib.redirect_stdout(io.StringIO()), + self.assertRaises(SystemExit), + ): + nvx.command_run(args) + command, cpu_profile, cold_boot, hypervisor = run_openvmm.call_args.args + self.assertEqual(command[command.index("--hypervisor") + 1], "kvm") + self.assertEqual((cpu_profile, cold_boot, hypervisor), (None, True, "kvm")) + + def test_benchmark_refuses_a_cpu_that_no_built_in_profile_serves(self): + # The command guards the benchmark coordinator, which it runs after. + self.assertIs(nvx.run_benchmark, benchmark.run) + granite_rapids = HostCpu("GenuineIntel", 6, 173, 1, "Intel(R) Xeon(R) 6973P-C") + icelake = HostCpu("GenuineIntel", 6, 106, 6) + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + + def checkout(name: str, *generations: CpuGeneration) -> str: + # An OpenVMM checkout that pins profiles of these generations, with + # the fields that NVX reads. + directory = Path(temporary.name) / name + profiles = directory / OPENVMM_CPU_PROFILES + profiles.mkdir(parents=True) + for generation in generations: + document = { + "vendor": generation.vendor, + "generation": { + "name": generation.name, + "cpus": [ + { + "family": cpu.family, + "model": cpu.model, + "steppings": [ + cpu.steppings.start, + cpu.steppings.stop - 1, + ], + } + for cpu in generation.cpus + ], + }, + } + (profiles / f"{generation.name}.json").write_text( + json.dumps(document), encoding="utf-8" + ) + return str(directory) + + pinned = checkout("pinned", *CPU_GENERATIONS) + + def run_command( + host: HostCpu | None, *arguments: str, openvmm_dir: str = pinned + ) -> tuple[int, str, list[argparse.Namespace]]: + stderr = io.StringIO() + launched: list[argparse.Namespace] = [] + + def run(args: argparse.Namespace) -> int: + launched.append(args) + return 0 + + with ( + patch.object(nvx, "host_cpu_signature", return_value=host), + patch.object(nvx, "run_benchmark", side_effect=run), + contextlib.redirect_stderr(stderr), + ): + status = nvx.main( + ["benchmark", "--openvmm-dir", openvmm_dir, *arguments] + ) + return status, stderr.getvalue(), launched + + # The refusal comes before the coordinator builds or measures anything. + status, output, launched = run_command( + granite_rapids, "--suite", "e2e", "--backend", "kvm" + ) + self.assertEqual((status, launched), (1, [])) + self.assertTrue( + output.startswith( + "error: benchmark refuses this host: no built-in CPU profile serves " + "its CPU, GenuineIntel 6/173/1 (Intel(R) Xeon(R) 6973P-C)" + ), + output, + ) + self.assertIn("openvmm --hypervisor kvm --cpu-fingerprint", output) + self.assertIn(cpu_profile_request_url(granite_rapids), output) + # On Windows, both backends name WHP's fingerprint. + status, output, launched = run_command(granite_rapids, "--backend", "both") + self.assertEqual((status, launched), (1, [])) + self.assertIn("openvmm --hypervisor whp --cpu-fingerprint", output) + # A CPU that cannot be identified is refused too. + status, output, launched = run_command(None) + self.assertEqual((status, launched), (1, [])) + self.assertIn("it cannot identify the host's CPU", output) + # The phase 2 suite boots no microVM, so it runs on any CPU, and a CPU + # that a built-in profile serves runs every suite. + for host, arguments in ( + (granite_rapids, ("--suite", "phase2")), + (None, ("--suite", "phase2")), + (icelake, ("--suite", "e2e")), + (icelake, ()), + ): + with self.subTest(host=host, arguments=arguments): + status, output, launched = run_command(host, *arguments) + self.assertEqual((status, output), (0, "")) + self.assertEqual(len(launched), 1) + # The profiles that decide are those of the OpenVMM checkout that + # benchmark builds: one that adds a profile for a CPU benchmarks it, and + # one that lacks a CPU's profile refuses it. + graniterapids = CpuGeneration( + "graniterapids", "GenuineIntel", (CpuModel(6, 173),) + ) + candidate = checkout("candidate", *CPU_GENERATIONS, graniterapids) + status, output, launched = run_command( + granite_rapids, "--suite", "e2e", openvmm_dir=candidate + ) + self.assertEqual((status, output), (0, "")) + self.assertEqual([args.openvmm_dir for args in launched], [Path(candidate)]) + lacking = checkout( + "lacking", + *( + generation + for generation in CPU_GENERATIONS + if generation.name != "icelake-sp" + ), + ) + status, output, launched = run_command(icelake, openvmm_dir=lacking) + self.assertEqual((status, launched), (1, [])) + self.assertIn( + "no built-in CPU profile serves its CPU, GenuineIntel 6/106/6", output + ) + # A checkout whose generations overlap on a CPU is refused there, since + # every cold boot fails on it. + ambiguous = checkout( + "ambiguous", + *CPU_GENERATIONS, + CpuGeneration("icelake-x", "GenuineIntel", (CpuModel(6, 106),)), + ) + status, output, launched = run_command(icelake, openvmm_dir=ambiguous) + self.assertEqual((status, launched), (1, [])) + self.assertIn( + "is in more than one generation of the built-in CPU profiles " + "(icelake-sp, icelake-x)", + output, + ) + # A checkout whose profiles cannot be read is refused, except for the + # phase 2 suite. + missing = str(Path(temporary.name) / "missing") + status, output, launched = run_command(icelake, openvmm_dir=missing) + self.assertEqual((status, launched), (1, [])) + self.assertTrue( + output.startswith( + "error: benchmark cannot confirm that a built-in CPU profile serves " + f"this host's CPU: the OpenVMM checkout {missing} pins no CPU profile" + ), + output, + ) + status, output, launched = run_command( + icelake, "--suite", "phase2", openvmm_dir=missing + ) + self.assertEqual((status, output, len(launched)), (0, "", 1)) + # A KVM worker, which the Windows coordinator reinvokes inside WSL + # without its --openvmm-dir, runs the binaries that the coordinator + # staged after the check, so it skips the check, whatever its checkout. + status, output, launched = run_command( + granite_rapids, "--_kvm-worker", openvmm_dir=missing + ) + self.assertEqual((status, output, len(launched)), (0, "", 1)) + + def test_run_leaves_openvmm_the_terminal_to_restore(self): + """A guest-triggered shutdown can end OpenVMM while its console holds + the terminal in raw mode, and OpenVMM restores the terminal settings + only when its standard error is a terminal, so run must leave it + one.""" + if sys.platform != "linux": + self.skipTest("needs a Linux pseudo-terminal") + import pty + import termios + + openvmm = ( + "import os, termios, tty\n" + # OpenVMM saves the settings through standard error, if it is a + # terminal, and its console makes standard input raw... + "saved = termios.tcgetattr(2) if os.isatty(2) else None\n" + "tty.setraw(0)\n" + # ...until the guest shuts down. + "if saved is not None:\n" + " termios.tcsetattr(2, termios.TCSAFLUSH, saved)\n" + ) + runner = ( + "import sys\n" + f"sys.path.insert(0, {str(Path(__file__).parent)!r})\n" + "import nvx\n" + "sys.exit(nvx._run_openvmm(" + f"[sys.executable, '-c', {openvmm!r}], None, cold_boot=False, " + "hypervisor='kvm'))\n" + ) + primary, secondary = pty.openpty() + try: + before = termios.tcgetattr(secondary) + completed = subprocess.run( + [sys.executable, "-c", runner], + stdin=secondary, + stdout=secondary, + stderr=secondary, + timeout=60, + check=False, + ) + after = termios.tcgetattr(secondary) + output = b"" + while select.select([primary], [], [], 0)[0]: + chunk = os.read(primary, 65536) + if not chunk: + break + output += chunk + finally: + os.close(secondary) + os.close(primary) + self.assertEqual(completed.returncode, 0, output.decode(errors="replace")) + self.assertEqual(after, before, "the terminal was left in raw mode") + + def test_run_exposes_restore_readiness(self): + args = nvx.parse_args( + [ + "run", + "--hypervisor", + "mshv", + "--restore-snapshot", + "snapshot", + "--processors", + "4", + "--restore-processors", + "2", + "--restore-ready-path", + "ready.sock", + "--dry-run", + ] + ) + + with ( + patch.object(nvx, "require_file", return_value=Path("openvmm")) as require, + patch.object( + nvx, "_format_command", return_value="formatted" + ) as format_command, + ): + nvx.command_run(args) + + require.assert_called_once() + command = format_command.call_args.args[0] + self.assertEqual( + command, + [ + "openvmm", + "--single-process", + "--machine", + "microvm", + "--processors", + "4", + "--hypervisor", + "mshv", + "--restore-snapshot", + "snapshot", + "--restore-processors", + "2", + "--restore-ready-path", + "ready.sock", + ], + ) + + missing_snapshot = nvx.parse_args( + ["run", "--restore-ready-path", "ready.sock", "--dry-run"] + ) + with self.assertRaisesRegex( + common.ScriptError, "--restore-ready-path requires --restore-snapshot" + ): + nvx.command_run(missing_snapshot) + + missing_processor_snapshot = nvx.parse_args( + ["run", "--restore-processors", "1", "--dry-run"] + ) + with self.assertRaisesRegex( + common.ScriptError, "--restore-processors requires --restore-snapshot" + ): + nvx.command_run(missing_processor_snapshot) + + invalid_restore_capacity = nvx.parse_args( + [ + "run", + "--processors", + "2", + "--restore-snapshot", + "snapshot", + "--restore-processors", + "4", + "--dry-run", + ] + ) + with self.assertRaisesRegex(common.ScriptError, "cannot exceed"): + nvx.command_run(invalid_restore_capacity) + + legacy = nvx.parse_args(["run", "--restore-snapshot", "snapshot", "--dry-run"]) + with ( + patch.object(nvx, "require_file", return_value=Path("openvmm")), + patch.object( + nvx, "_format_command", return_value="formatted" + ) as format_command, + ): + nvx.command_run(legacy) + self.assertIn("microvm", format_command.call_args.args[0]) + self.assertNotIn("microvm-v2", format_command.call_args.args[0]) + + capture_memory = nvx.parse_args( + [ + "run", + "--memory-mib", + "512", + "--memory-capacity-mib", + "2048", + "--dry-run", + ] + ) + with ( + patch.object(nvx, "require_file", return_value=Path("artifact")), + patch.object( + nvx, "_format_command", return_value="formatted" + ) as format_command, + ): + nvx.command_run(capture_memory) + command = format_command.call_args.args[0] + self.assertEqual(command[command.index("--memory") + 1], "512M") + self.assertEqual(command[command.index("--memory-capacity") + 1], "2048M") + + restore_memory = nvx.parse_args( + [ + "run", + "--restore-snapshot", + "snapshot", + "--restore-memory-mib", + "1024", + "--dry-run", + ] + ) + with ( + patch.object(nvx, "require_file", return_value=Path("openvmm")), + patch.object( + nvx, "_format_command", return_value="formatted" + ) as format_command, + ): + nvx.command_run(restore_memory) + command = format_command.call_args.args[0] + self.assertEqual(command[command.index("--restore-memory") + 1], "1024M") + self.assertNotIn("--memory-capacity", command) + + smp = nvx.parse_args( + ["run", "--machine", "microvm", "--processors", "2", "--dry-run"] + ) + self.assertEqual(smp.machine, "microvm") + self.assertEqual(smp.processors, 2) + + with self.assertRaises(SystemExit): + nvx.parse_args( + ["run", "--machine", "microvm-v2", "--processors", "4", "--dry-run"] + ) + + with self.assertRaises(SystemExit): + nvx.parse_args(["run", "--machine", "microvm-v3", "--dry-run"]) + + def test_openvmm_command_passes_restore_choice_in_build_config(self): + with patch.object(nvx, "build_openvmm") as build_openvmm: + nvx.command_build_openvmm(argparse.Namespace(skip_restore=True)) + + config = build_openvmm.call_args.args[0] + self.assertIsInstance(config, build_config.OpenVmmBuildConfig) + self.assertTrue(config.skip_restore) + self.assertIsNone(config.backend) + + def test_build_commands_pass_backend_choice_in_config(self): + for command in ("build-openvmm", "build"): + for backend in (None, "kvm", "mshv", "whp"): + with self.subTest(command=command, backend=backend): + arguments = [command, "--skip-restore"] + if backend is not None: + arguments.extend(["--backend", backend]) + if command == "build": + arguments.append("--native") + args = nvx.parse_args(arguments) + with ( + patch.object(nvx, "build_openvmm") as openvmm, + patch.object(nvx, "build_all") as combined, + ): + args.handler(args) + + if command == "build": + aggregate = combined.call_args.args[0] + self.assertIsInstance(aggregate, build_config.BuildConfig) + self.assertTrue(aggregate.native_guest) + config = aggregate.openvmm + openvmm.assert_not_called() + else: + config = openvmm.call_args.args[0] + combined.assert_not_called() + self.assertIsInstance(config, build_config.OpenVmmBuildConfig) + self.assertEqual(config.backend, backend) + self.assertTrue(config.skip_restore) + + def test_build_commands_reject_unknown_backends(self): + for command in ("build-openvmm", "build"): + with self.subTest(command=command), self.assertRaises(SystemExit): + nvx.parse_args([command, "--backend", "unknown"]) + + def test_build_commands_pass_specialized_configs_to_consumers(self): + with ( + patch.object(nvx, "build_kernel") as build_kernel, + patch.object(nvx, "build_initramfs") as build_initramfs, + patch.object(nvx, "record_openvmm_provenance") as provenance, + patch.object(nvx, "collect_release_sources") as collect_sources, + ): + nvx.command_build_kernel(argparse.Namespace()) + nvx.command_build_initramfs(argparse.Namespace(guest="alpine")) + nvx.command_record_openvmm_provenance(argparse.Namespace()) + nvx.command_collect_sources(argparse.Namespace()) + + self.assertIsInstance( + build_kernel.call_args.args[0], + build_config.KernelBuildConfig, + ) + self.assertFalse(build_kernel.call_args.args[0].debug) + with patch.object(nvx, "build_kernel") as build_debug_kernel: + nvx.command_build_kernel(argparse.Namespace(debug=True)) + self.assertEqual( + build_debug_kernel.call_args.args[0], + build_config.KernelBuildConfig.debug_variant(), + ) + self.assertIsInstance( + build_initramfs.call_args.args[0], + build_config.InitramfsBuildConfig, + ) + self.assertIsInstance( + provenance.call_args.args[0], + build_config.OpenVmmBuildConfig, + ) + self.assertIsInstance( + collect_sources.call_args.args[0], + build_config.DockerBuildConfig, + ) + + def test_record_openvmm_provenance_command_is_exposed(self): + args = nvx.parse_args(["record-openvmm-provenance"]) + self.assertIs(args.handler, nvx.command_record_openvmm_provenance) + + +class CiTests(unittest.TestCase): + def test_openvmm_unit_tests_run_nextest_and_doctests(self): + with tempfile.TemporaryDirectory() as temporary: + openvmm = Path(temporary) / "openvmm" + openvmm.mkdir() + (openvmm / "Cargo.toml").touch() + fuzz_crates = common.CommandResult( + args=("cargo", "xtask", "fuzz", "list", "--crates"), + returncode=0, + stdout=b"fuzz_alpha\nfuzz_beta\n", + stderr=b"", + ) + + with ( + patch.object(OpenVMMBuildConstants, "DIRECTORY", openvmm), + patch.object(ci, "require_tool", return_value="cargo"), + patch.object(ci, "run_capture", return_value=fuzz_crates) as capture, + patch.object(ci, "run_checked") as run_checked, + ): + ci.run_openvmm_unit_tests() + + capture.assert_called_once_with( + ["cargo", "xtask", "fuzz", "list", "--crates"], + cwd=openvmm, + ) + self.assertEqual(run_checked.call_count, 3) + self.assertEqual( + run_checked.call_args_list[0], + call( + [ + "cargo", + "xflowey", + "restore-packages", + "--no-compat-igvm", + ], + cwd=openvmm, + ), + ) + command = run_checked.call_args_list[1].args[0] + self.assertEqual( + command[:10], + [ + "cargo", + "nextest", + "run", + "--profile", + "agent", + "--workspace", + "--tests", + "--bins", + "--features", + "ci", + ], + ) + excluded_packages = [ + command[index + 1] + for index, argument in enumerate(command) + if argument == "--exclude" + ] + self.assertEqual( + excluded_packages, + [ + *ci.OPENVMM_UNIT_TEST_EXCLUDED_PACKAGES, + "fuzz_alpha", + "fuzz_beta", + ], + ) + self.assertEqual(run_checked.call_args_list[1].kwargs["cwd"], openvmm) + self.assertEqual( + run_checked.call_args_list[2], + call( + [ + "cargo", + "test", + "--locked", + "--doc", + "--workspace", + "--no-fail-fast", + ], + cwd=openvmm, + ), + ) + + def test_openvmm_unit_tests_stop_when_fuzz_crate_query_fails(self): + with tempfile.TemporaryDirectory() as temporary: + openvmm = Path(temporary) / "openvmm" + openvmm.mkdir() + (openvmm / "Cargo.toml").touch() + failed_query = common.CommandResult( + args=("cargo", "xtask", "fuzz", "list", "--crates"), + returncode=2, + stdout=b"", + stderr=b"query failed", + ) + + with ( + patch.object(OpenVMMBuildConstants, "DIRECTORY", openvmm), + patch.object(ci, "require_tool", return_value="cargo"), + patch.object(ci, "run_capture", return_value=failed_query), + patch.object(ci, "run_checked") as run_checked, + self.assertRaisesRegex( + common.ScriptError, + "OpenVMM fuzz crate query exited 2", + ), + ): + ci.run_openvmm_unit_tests() + + run_checked.assert_not_called() + + def test_openvmm_tests_use_nvx_guest_artifacts(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + openvmm = root / "openvmm" + openvmm.mkdir() + (openvmm / "Cargo.toml").touch() + kernel = root / "vmlinux" + initrd = root / "initramfs.cpio.gz" + kernel.touch() + initrd.touch() + backend = "whp" if os.name == "nt" else "kvm" + installed_targets = common.CommandResult( + args=("rustup", "target", "list"), + returncode=0, + stdout=( + "\n".join(OpenVMMBuildConstants.TEST_RUST_TARGETS[backend]) + "\n" + ).encode(), + stderr=b"", + ) + + with ( + patch.object(OpenVMMBuildConstants, "DIRECTORY", openvmm), + patch.object( + ci, + "artifact_path", + side_effect=[kernel, initrd], + ), + patch.object(ci.os, "access", return_value=True), + patch.object(ci.Path, "exists", return_value=False), + patch.object(ci, "require_tool", side_effect=["cargo", "rustup"]), + patch.object( + ci, + "run_capture", + return_value=installed_targets, + ) as run_capture, + patch.object(ci, "run_checked") as run_checked, + patch.dict( + os.environ, + { + "PETRI_CAPABILITIES": "vpci", + "RUNNER_TEMP": os.fspath(root), + }, + ), + ): + ci.run_openvmm_tests(backend) + + run_capture.assert_called_once_with( + [ + "rustup", + "target", + "list", + "--installed", + "--toolchain", + OpenVMMBuildConstants.RUST_TOOLCHAIN, + ] + ) + self.assertEqual(run_checked.call_count, 2) + restore, tests = run_checked.call_args_list + self.assertEqual( + restore.args[0], + ["cargo", "xflowey", "restore-packages", "--no-compat-igvm"], + ) + environment = restore.kwargs["env"] + self.assertEqual( + environment["RUSTUP_TOOLCHAIN"], + OpenVMMBuildConstants.RUST_TOOLCHAIN, + ) + if os.name == "nt": + self.assertNotIn("XDG_CACHE_HOME", environment) + else: + self.assertEqual( + environment["XDG_CACHE_HOME"], + os.fspath(root / "openvmm-cache"), + ) + command = tests.args[0] + self.assertEqual(command[:3], ["cargo", "xflowey", "vmm-tests-run"]) + filter_index = command.index("--filter") + self.assertEqual( + command[filter_index + 1], + ci.OPENVMM_TEST_FILTERS[backend], + ) + self.assertEqual(tests.kwargs["cwd"], openvmm) + self.assertIs(tests.kwargs["env"], environment) + self.assertEqual( + environment["OPENVMM_MICROVM_TEST_KERNEL"], + os.fspath(kernel.resolve()), + ) + self.assertEqual( + environment["OPENVMM_MICROVM_TEST_INITRD"], + os.fspath(initrd.resolve()), + ) + self.assertEqual(environment["PETRI_CAPABILITIES"], "vpci") + if os.name == "nt": + self.assertEqual( + command[command.index("--dir") + 1], + os.fspath(root / backend), + ) + + def test_openvmm_tests_pin_toolchain_without_runner_temp(self): + installed_targets = common.CommandResult( + args=("rustup", "target", "list"), + returncode=0, + stdout=( + "\n".join(OpenVMMBuildConstants.TEST_RUST_TARGETS["whp"]) + "\n" + ).encode(), + stderr=b"", + ) + + with ( + patch.object(ci, "run_capture", return_value=installed_targets), + patch.object(ci, "run_checked") as run_checked, + patch.dict(os.environ, {}, clear=True), + ): + environment = ci._prepare_openvmm_test_environment("whp", "rustup") + + self.assertEqual( + environment, + {"RUSTUP_TOOLCHAIN": OpenVMMBuildConstants.RUST_TOOLCHAIN}, + ) + run_checked.assert_not_called() + + def test_openvmm_tests_prepare_job_local_toolchain_for_missing_targets(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + openvmm = root / "openvmm" + openvmm.mkdir() + (openvmm / "Cargo.toml").touch() + kernel = root / "vmlinux" + initrd = root / "initramfs.cpio.gz" + kernel.touch() + initrd.touch() + backend = "whp" if os.name == "nt" else "kvm" + installed_targets = common.CommandResult( + args=("rustup", "target", "list"), + returncode=0, + stdout=f"{OpenVMMBuildConstants.GUEST_RUST_TARGET}\n".encode(), + stderr=b"", + ) + + with ( + patch.object(OpenVMMBuildConstants, "DIRECTORY", openvmm), + patch.object( + ci, + "artifact_path", + side_effect=[kernel, initrd], + ), + patch.object(ci.os, "access", return_value=True), + patch.object(ci.Path, "exists", return_value=False), + patch.object(ci, "require_tool", side_effect=["cargo", "rustup"]), + patch.object(ci, "run_capture", return_value=installed_targets), + patch.object(ci, "run_checked") as run_checked, + patch.dict(os.environ, {"RUNNER_TEMP": os.fspath(root)}), + ): + ci.run_openvmm_tests(backend) + + self.assertEqual(run_checked.call_count, 4) + install_toolchain, install_targets, restore, tests = ( + run_checked.call_args_list + ) + environment = install_toolchain.kwargs["env"] + self.assertEqual( + environment["RUSTUP_HOME"], + os.fspath(root / "openvmm-rustup"), + ) + self.assertEqual( + environment["RUSTUP_TOOLCHAIN"], + OpenVMMBuildConstants.RUST_TOOLCHAIN, + ) + if os.name != "nt": + self.assertEqual( + environment["XDG_CACHE_HOME"], + os.fspath(root / "openvmm-cache"), + ) + self.assertEqual( + install_toolchain.args[0], + [ + "rustup", + "toolchain", + "install", + OpenVMMBuildConstants.RUST_TOOLCHAIN, + "--profile", + "minimal", + ], + ) + self.assertEqual( + install_targets.args[0], + [ + "rustup", + "target", + "add", + *OpenVMMBuildConstants.TEST_RUST_TARGETS[backend], + "--toolchain", + OpenVMMBuildConstants.RUST_TOOLCHAIN, + ], + ) + for command in (install_targets, restore, tests): + self.assertIs(command.kwargs["env"], environment) + + def test_openvmm_tests_install_missing_pinned_toolchain(self): + missing_toolchain = common.CommandResult( + args=("rustup", "target", "list"), + returncode=1, + stdout=b"", + stderr=( + f"error: toolchain '{OpenVMMBuildConstants.RUST_TOOLCHAIN}' " + "is not installed\n" + ).encode(), + ) + install_toolchain = [ + "rustup", + "toolchain", + "install", + OpenVMMBuildConstants.RUST_TOOLCHAIN, + "--profile", + "minimal", + ] + add_targets = [ + "rustup", + "target", + "add", + *OpenVMMBuildConstants.TEST_RUST_TARGETS["kvm"], + "--toolchain", + OpenVMMBuildConstants.RUST_TOOLCHAIN, + ] + + with tempfile.TemporaryDirectory() as temporary: + for runner_temp in (None, temporary): + with ( + self.subTest(runner_temp=runner_temp), + patch.object(ci, "run_capture", return_value=missing_toolchain), + patch.object(ci, "run_checked") as run_checked, + patch.dict( + os.environ, + {} if runner_temp is None else {"RUNNER_TEMP": runner_temp}, + clear=True, + ), + ): + environment = ci._prepare_openvmm_test_environment("kvm", "rustup") + + self.assertEqual( + [command.args[0] for command in run_checked.call_args_list], + [install_toolchain, add_targets], + ) + for command in run_checked.call_args_list: + self.assertIs(command.kwargs["env"], environment) + self.assertEqual( + environment["RUSTUP_TOOLCHAIN"], + OpenVMMBuildConstants.RUST_TOOLCHAIN, + ) + if runner_temp is None: + self.assertNotIn("RUSTUP_HOME", environment) + else: + self.assertEqual( + environment["RUSTUP_HOME"], + os.fspath( + Path(runner_temp) + / OpenVMMBuildConstants.RUSTUP_DIRECTORY_NAME + ), + ) + + def test_openvmm_tests_define_each_backend_filter(self): + self.assertEqual( + set(ci.OPENVMM_TEST_FILTERS), + set(ci.OPENVMM_TEST_BACKENDS), + ) + self.assertEqual( + set(OpenVMMBuildConstants.TEST_RUST_TARGETS), + set(ci.OPENVMM_TEST_BACKENDS), + ) + self.assertEqual( + ci.OPENVMM_TEST_FILTERS["kvm"], + ci.OPENVMM_KVM_TEST_FILTER, + ) + self.assertIn( + "!test(no_vmbus_prepped_boot_no_vmbus_windows)", + ci.OPENVMM_KVM_TEST_FILTER, + ) + self.assertIn( + "!test(windows_datacenter_core_2022_x64)", + ci.OPENVMM_KVM_TEST_FILTER, + ) + self.assertIn("!test(virtio_net_windows)", ci.OPENVMM_KVM_TEST_FILTER) + self.assertEqual( + set(ci.OPENVMM_KVM_EXCLUDED_TESTS), + { + "multiarch::openvmm_pcat_x64_freebsd_13_2_x64_boot_no_agent", + "multiarch::openvmm_pcat_x64_freebsd_13_2_x64_iso_boot_no_agent", + "multiarch::openvmm_pcat_x64_ubuntu_2404_server_x64_boot", + "multiarch::openvmm_pcat_x64_ubuntu_2504_server_x64_boot", + "multiarch::openvmm_pcat_x64_ubuntu_2504_server_x64_boot_heavy", + }, + ) + for excluded_test in ci.OPENVMM_KVM_EXCLUDED_TESTS: + self.assertIn( + f"!{ci._exact_openvmm_test(excluded_test)}", + ci.OPENVMM_KVM_TEST_FILTER, + ) + self.assertNotIn("!test(openvmm_pcat_x64)", ci.OPENVMM_KVM_TEST_FILTER) + self.assertNotIn( + "openvmm_linux_x64_apicid_offset", + ci.OPENVMM_KVM_TEST_FILTER, + ) + self.assertNotIn( + "openvmm_linux_x64_legacy_xapic", + ci.OPENVMM_KVM_TEST_FILTER, + ) + self.assertEqual( + ci.OPENVMM_TEST_FILTERS["mshv"], + ci.OPENVMM_MSHV_TEST_FILTER, + ) + self.assertIn( + "!test(windows_datacenter_core_2022_x64)", + ci.OPENVMM_MSHV_TEST_FILTER, + ) + self.assertEqual( + set(ci.OPENVMM_MSHV_EXCLUDED_TESTS), + { + "multiarch::openvmm_pcat_x64_freebsd_13_2_x64_boot_no_agent", + "multiarch::openvmm_pcat_x64_freebsd_13_2_x64_iso_boot_no_agent", + "multiarch::openvmm_pcat_x64_ubuntu_2404_server_x64_boot", + "multiarch::openvmm_pcat_x64_ubuntu_2504_server_x64_boot", + "multiarch::openvmm_pcat_x64_ubuntu_2504_server_x64_boot_heavy", + "multiarch::pcie::openvmm_linux_x64_pcie_save_restore", + }, + ) + self.assertNotIn( + "x86_64::openvmm_linux_x64_virtio_blk_device", + ci.OPENVMM_MSHV_EXCLUDED_TESTS, + ) + for excluded_test in ci.OPENVMM_MSHV_EXCLUDED_TESTS: + self.assertIn( + f"!{ci._exact_openvmm_test(excluded_test)}", + ci.OPENVMM_MSHV_TEST_FILTER, + ) + self.assertNotIn("!test(openvmm_pcat_x64)", ci.OPENVMM_MSHV_TEST_FILTER) + for backend, test_filter in ci.OPENVMM_TEST_FILTERS.items(): + for required_test in ci.OPENVMM_REQUIRED_MICROVM_TESTS: + with self.subTest(backend=backend, required_test=required_test): + self.assertIn( + ci._exact_openvmm_test(required_test), + test_filter, + ) + self.assertEqual(len(ci.OPENVMM_WHP_TESTS), 29) + self.assertEqual( + len(set(ci.OPENVMM_WHP_TESTS)), + len(ci.OPENVMM_WHP_TESTS), + ) + for required_test in ci.OPENVMM_REQUIRED_MICROVM_TESTS: + self.assertIn(required_test, ci.OPENVMM_WHP_TESTS) + # Only Linux hosts can build the Linux pipette this TTRPC test boots. + self.assertNotIn("test_ttrpc_interface", ci.OPENVMM_TEST_FILTERS["whp"]) + for backend in ("kvm", "mshv"): + with self.subTest(backend=backend): + self.assertIn("test(ttrpc)", ci.OPENVMM_TEST_FILTERS[backend]) + self.assertNotIn( + "test_ttrpc_interface", + ci.OPENVMM_TEST_FILTERS[backend], + ) + self.assertEqual(ci.OPENVMM_WHP_EXCLUDED_TESTS, ()) + self.assertNotIn( + "multiarch::openvmm_pcat_x64_windows_datacenter_core_2022_x64_boot_heavy", + ci.OPENVMM_WHP_EXCLUDED_TESTS, + ) + self.assertNotIn( + "multiarch::openvmm_pcat_x64_windows_datacenter_core_2022_x64_boot", + ci.OPENVMM_WHP_EXCLUDED_TESTS, + ) + self.assertNotIn( + "multiarch::openvmm_pcat_x64_freebsd_13_2_x64_iso_boot_no_agent", + ci.OPENVMM_WHP_EXCLUDED_TESTS, + ) + self.assertNotIn( + "multiarch::openvmm_pcat_x64_freebsd_13_2_x64_boot_no_agent", + ci.OPENVMM_WHP_EXCLUDED_TESTS, + ) + self.assertNotIn( + "multiarch::pcie::openvmm_uefi_x64_windows_datacenter_core_2022_x64_pcie_nvme_boot", + ci.OPENVMM_WHP_EXCLUDED_TESTS, + ) + self.assertTrue( + set(ci.OPENVMM_WHP_EXCLUDED_TESTS).issubset(ci.OPENVMM_WHP_TESTS) + ) + self.assertEqual( + ci.OPENVMM_TEST_FILTERS["whp"], + ci._join_openvmm_tests( + ci.OPENVMM_WHP_TESTS, + ci.OPENVMM_WHP_EXCLUDED_TESTS, + ), + ) + self.assertEqual( + ci._join_openvmm_tests( + ("suite::boot", "suite::boot_heavy"), + ("suite::boot_heavy",), + ), + ( + "(test(/^suite::boot$/) | test(/^suite::boot_heavy$/))" + " & !test(/^suite::boot_heavy$/)" + ), + ) + + def test_openvmm_tests_reject_unknown_backend(self): + with self.assertRaisesRegex(common.ScriptError, "unsupported.*backend"): + ci.run_openvmm_tests("unknown") + + +class CiConfigurationTests(unittest.TestCase): + def test_ci_needs_reference_declared_jobs(self): + workflow = ( + BuildConstants.REPO_ROOT / ".github" / "workflows" / "ci.yml" + ).read_text(encoding="utf-8") + jobs = set( + re.findall( + r"^ ([A-Za-z0-9_-]+):$", + workflow.split("jobs:\n", 1)[1], + re.MULTILINE, + ) + ) + for job_name in sorted(jobs): + job = _workflow_job(workflow, job_name) + match = re.search(r"^ needs:(.*)$", job, re.MULTILINE) + if match is None: + continue + inline = match.group(1).strip() + if inline: + dependencies = {name.strip() for name in inline.strip("[]").split(",")} + else: + dependencies = set( + re.findall( + r"^ - ([A-Za-z0-9_-]+)$", + job.split(" if:", 1)[0], + re.MULTILINE, + ) + ) + with self.subTest(job=job_name): + self.assertLessEqual( + dependencies, + jobs, + f"undefined job dependencies: {dependencies - jobs}", + ) + required = _workflow_job(workflow, "required-status-check") + self.assertIn(" - aci-edge-sandboxes\n", required) + + def test_development_release_requires_successful_crate_checks(self): + workflow = ( + BuildConstants.REPO_ROOT / ".github" / "workflows" / "ci.yml" + ).read_text(encoding="utf-8") + release_job = _workflow_job(workflow, "release") + self.assertIn(" - aci-edge-sandboxes\n", release_job) + predicate = release_job.split(" if:", 1)[1].split(" runs-on:", 1)[0] + self.assertIn("needs.aci-edge-sandboxes.result == 'success' &&", predicate) + self.assertNotIn("needs.aci-edge-sandboxes.result == 'skipped'", predicate) + + def test_required_ci_result_policy(self): + always_successful = {"quality", "aci-edge-sandboxes", "openvmm-changes"} + # Jobs that succeed whenever the repository's jobs run. + repository = always_successful | set(ci.REQUIRED_CI_CLI_JOBS) + builds = set(ci.REQUIRED_CI_BUILD_JOBS) + openvmm_tests = set(ci.REQUIRED_CI_OPENVMM_TEST_JOBS) + openvmm_artifact_tests = set(ci.REQUIRED_CI_OPENVMM_ARTIFACT_TEST_JOBS) + microvm_tests = set(ci.REQUIRED_CI_MICROVM_TEST_JOBS) + debug_kernel = set(ci.REQUIRED_CI_MICROVM_DEBUG_JOBS) + debug_kernel_push = set(ci.REQUIRED_CI_MICROVM_DEBUG_PUSH_JOBS) + artifacts = {ci.REQUIRED_CI_ARTIFACT_JOB, ci.REQUIRED_CI_DEBUG_KERNEL_JOB} + platforms = set(ci.REQUIRED_CI_PLATFORM_JOBS) + workloads = artifacts | builds | platforms + microvm = microvm_tests | debug_kernel + # Event, same repository, the change detector's outputs (tests, + # workloads, OpenVMM unit tests, OpenVMM VMM tests), and the jobs that + # must succeed; every other required job must be skipped. + cases = ( + ("pull_request", True, (False, False, False, False), repository), + ( + "pull_request", + True, + (False, True, False, False), + repository | workloads | {"performance-gate"}, + ), + ( + "pull_request", + True, + (True, False, True, True), + repository | builds | openvmm_tests, + ), + ( + "pull_request", + True, + (True, True, True, True), + repository + | workloads + | openvmm_tests + | openvmm_artifact_tests + | microvm + | {"performance-gate"}, + ), + # A pull request that changes no input of either OpenVMM suite. + ( + "pull_request", + True, + (True, True, False, False), + repository | workloads | microvm | {"performance-gate"}, + ), + # Guest inputs reach the VMM tests but not the unit tests. + ( + "pull_request", + True, + (True, True, False, True), + repository + | workloads + | openvmm_artifact_tests + | microvm + | {"performance-gate"}, + ), + ("pull_request", False, (False, False, False, False), always_successful), + ( + "pull_request", + False, + (False, True, False, False), + always_successful | artifacts, + ), + ("pull_request", False, (True, False, True, True), always_successful), + ( + "pull_request", + False, + (True, True, True, True), + always_successful | artifacts, + ), + ("push", True, (False, False, False, False), repository), + ("push", True, (False, True, False, False), repository | workloads), + ( + "push", + True, + (True, False, True, True), + repository | builds | openvmm_tests, + ), + ( + "push", + True, + (True, True, True, True), + repository + | workloads + | openvmm_tests + | openvmm_artifact_tests + | microvm + | debug_kernel_push, + ), + ) + + for event_name, same_repository, flags, successful_jobs in cases: + changes = ci.CiChanges(*flags) + with self.subTest( + event_name=event_name, + same_repository=same_repository, + changes=changes, + ): + expected = ci.required_ci_expected_results( + event_name, + same_repository=same_repository, + changes=changes, + ) + self.assertEqual(set(expected), set(ci.REQUIRED_CI_RESULT_ENVIRONMENTS)) + self.assertEqual( + {job for job, result in expected.items() if result == "success"}, + successful_jobs, + ) + self.assertEqual( + ci.required_ci_failures( + event_name, + same_repository=same_repository, + changes=changes, + results=expected, + ), + [], + ) + + for job, expected_result in expected.items(): + unexpected = expected.copy() + unexpected[job] = ( + "skipped" if expected_result == "success" else "success" + ) + with self.subTest(job=job, actual=unexpected[job]): + self.assertEqual( + ci.required_ci_failures( + event_name, + same_repository=same_repository, + changes=changes, + results=unexpected, + ), + [ + f"{job}: expected {expected_result}, " + f"got {unexpected[job]}" + ], + ) + + for job in expected: + failed = expected.copy() + failed[job] = "failure" + with self.subTest(job=job, actual="failure"): + self.assertEqual( + ci.required_ci_failures( + event_name, + same_repository=same_repository, + changes=changes, + results=failed, + ), + [f"{job}: expected {expected[job]}, got failure"], + ) + + def test_required_ci_job_uses_tested_policy(self): + workflow = ( + BuildConstants.REPO_ROOT / ".github" / "workflows" / "ci.yml" + ).read_text(encoding="utf-8") + job = _workflow_job(workflow, "required-status-check") + + self.assertIn("python3 scripts/nvx.py check-required-ci", job) + self.assertIn('--same-repository "${SAME_REPOSITORY}"', job) + self.assertIn( + "github.event.pull_request.head.repo.full_name == github.repository", + job, + ) + self.assertLess( + job.index("uses: actions/checkout@v5"), + job.index("python3 scripts/nvx.py check-required-ci"), + ) + self.assertIn("persist-credentials: false", job) + for environment in ci.REQUIRED_CI_RESULT_ENVIRONMENTS.values(): + with self.subTest(environment=environment): + self.assertIn(f" {environment}:", job) + # Every suite flag reaches the policy as the detector reported it. + detector = _workflow_job(workflow, "openvmm-changes") + for output in ci.CI_CHANGE_OUTPUTS: + environment = output.upper().replace("-", "_") + with self.subTest(output=output): + self.assertIn( + f" {output}: ${{{{ steps.changes.outputs.{output} }}}}", + detector, + ) + self.assertIn( + f" {environment}: " + f"${{{{ needs.openvmm-changes.outputs.{output} }}}}", + job, + ) + self.assertIn(f'--{output} "${{{environment}}}"', job) + + def test_ci_change_classification_selects_suites_by_input(self): + # The pull-request outputs (tests, workloads, OpenVMM unit tests, + # OpenVMM VMM tests) for each class of changed path. + cases = ( + ((), (False, True, False, False)), + (("doc/ci.md", "README.md", "DOC/Guide.MD"), (False, False, False, False)), + (("openvmm",), (True, True, True, True)), + (("rust-toolchain.toml",), (True, True, True, True)), + (("scripts/setup/setup-windows-whp.ps1",), (True, True, True, True)), + (("scripts/setup/README.md",), (True, False, True, True)), + ((".github/workflows/ci.yml",), (True, True, True, True)), + ((".github/actions/validate-runner/action.yml",), (True, True, True, True)), + ((".github/actions/sccache/action.yml",), (True, True, True, True)), + ((".github/actions/setup-curl/curl-shim.rs",), (True, True, True, True)), + (("scripts/nvx.py",), (True, True, True, True)), + (("scripts/nvx_tools/ci.py",), (True, True, True, True)), + (("scripts/nvx_tools/common.py",), (True, True, True, True)), + (("scripts/nvx_tools/build.py",), (True, True, False, True)), + (("scripts/nvx_tools/guests.py",), (True, True, False, True)), + (("kernel/config-microvm",), (True, True, False, True)), + (("kernel/patches/0001-example.patch",), (True, True, False, True)), + (("guest/common/init",), (True, True, False, True)), + (("guest/alpine/nvx-container-enter",), (True, True, False, True)), + (("SOURCE-MANIFEST.json",), (True, True, False, True)), + (("docker/Dockerfile",), (True, True, False, True)), + ( + (".github/actions/build-guest-artifacts/action.yml",), + (True, True, False, True), + ), + (("guest/ubuntu/nvx-bashrc",), (True, True, False, False)), + (("ubuntu/packages.lock.json",), (True, True, False, False)), + (("azurelinux/packages.lock.json",), (True, True, False, False)), + (("aci_edge_sandboxes/src/lib.rs",), (True, True, False, False)), + (("scripts/nvx_tools/microvm_tests.py",), (True, True, False, False)), + ( + (".github/workflows/run-nvx-microvm-tests.yml",), + (True, True, False, False), + ), + ((".github/actions/build-openvmm/action.yml",), (True, True, False, False)), + ((".github/actions/validate-nvx/action.yml",), (False, True, False, False)), + ( + (".github/actions/run-benchmark/action.yml",), + (False, True, False, False), + ), + (("scripts/test_nvx_tools.py",), (False, True, False, False)), + (("data/linux-kvm-virtual-machine.csv",), (False, True, False, False)), + ( + ("scripts/nvx_tools/microvm_tests.py", "kernel/config-microvm"), + (True, True, False, True), + ), + ) + for changed_files, flags in cases: + with self.subTest(changed_files=changed_files): + pull_request = ci.classify_ci_changes("pull_request", changed_files) + self.assertEqual(pull_request, ci.CiChanges(*flags)) + # Pushes keep running both OpenVMM suites with the tests. + run_tests, run_workloads = flags[:2] + self.assertEqual( + ci.classify_ci_changes("push", changed_files), + ci.CiChanges(run_tests, run_workloads, run_tests, run_tests), + ) + self.assertEqual( + ci.classify_ci_changes("pull_request", ["openvmm", ""]), + ci.classify_ci_changes("pull_request", ["openvmm"]), + ) + with self.assertRaisesRegex(ValueError, "unsupported CI event"): + ci.classify_ci_changes("workflow_dispatch", ["openvmm"]) + + def test_ci_change_classification_covers_each_suite_input(self): + github = BuildConstants.REPO_ROOT / ".github" + tools = BuildConstants.REPO_ROOT / "scripts" / "nvx_tools" + workflow = (github / "workflows" / "ci.yml").read_text(encoding="utf-8") + + def local_actions(configuration: str) -> set[str]: + names = set( + re.findall(r"uses: \./\.github/actions/([a-z0-9-]+)", configuration) + ) + pending = list(names) + while pending: + action = (github / "actions" / pending.pop() / "action.yml").read_text( + encoding="utf-8" + ) + for name in local_actions(action) - names: + names.add(name) + pending.append(name) + return names + + def imported_modules(module: str) -> set[str]: + modules = {module} + pending = [module] + while pending: + tree = ast.parse( + (tools / f"{pending.pop()}.py").read_text(encoding="utf-8") + ) + for node in ast.walk(tree): + if not isinstance(node, ast.ImportFrom) or node.level != 1: + continue + names = ( + [node.module.split(".")[0]] + if node.module + else [alias.name for alias in node.names] + ) + for name in set(names) - modules: + modules.add(name) + pending.append(name) + return modules + + command_inputs = { + "scripts/nvx.py", + "scripts/nvx_tools/__init__.py", + ".github/workflows/ci.yml", + *(f"scripts/nvx_tools/{name}.py" for name in imported_modules("ci")), + } + self.assertLessEqual( + {"ci", "common", "build_constants"}, imported_modules("ci") + ) + unit_inputs = command_inputs | { + f".github/actions/{name}/action.yml" + for name in local_actions(_workflow_job(workflow, "openvmm-unit-tests")) + } + vmm_inputs = command_inputs | { + f".github/actions/{name}/action.yml" + for job in ("openvmm-vmm-tests", "artifacts") + for name in local_actions(_workflow_job(workflow, job)) + } + # Every cache key input of the kernel and Alpine initramfs that the + # VMM tests boot, and of every guest that the microVM tests boot. + guest_action = ( + github / "actions" / "build-guest-artifacts" / "action.yml" + ).read_text(encoding="utf-8") + guest_inputs = { + match.group(1): { + path.replace("**", "example").replace("*", "example") + for path in re.findall(r"'([^']+)'", match.group(2)) + } + for match in re.finditer( + r"^\s+([A-Z_]+)_INPUT_HASH: \$\{\{ hashFiles\((.+)\) \}\}$", + guest_action, + re.MULTILINE, + ) + } + self.assertEqual( + set(guest_inputs), + {"KERNEL", "DEBUG_KERNEL", "ALPINE", "UBUNTU", "AZURELINUX"}, + ) + vmm_inputs |= guest_inputs["KERNEL"] | guest_inputs["ALPINE"] + test_inputs = vmm_inputs.union(*guest_inputs.values()) | { + ".github/workflows/run-nvx-microvm-tests.yml", + ".github/workflows/build-openvmm-binary.yml", + *( + f".github/actions/{name}/action.yml" + for configuration in ( + (github / "workflows" / "run-nvx-microvm-tests.yml").read_text( + encoding="utf-8" + ), + (github / "workflows" / "build-openvmm-binary.yml").read_text( + encoding="utf-8" + ), + _workflow_job(workflow, "debug-kernel"), + ) + for name in local_actions(configuration) + ), + } + self.assertNotIn(".github/actions/validate-nvx/action.yml", test_inputs) + for pattern, inputs in ( + (ci.OPENVMM_UNIT_TEST_INPUT_PATHS, unit_inputs), + (ci.OPENVMM_VMM_TEST_INPUT_PATHS, vmm_inputs), + (ci.CI_TEST_INPUT_PATHS, test_inputs), + ): + for path in sorted(inputs): + with self.subTest(pattern=pattern.pattern, path=path): + self.assertIsNotNone(pattern.match(path)) + + def test_ci_change_summary_explains_each_decision(self): + changed_files = [ + "scripts/nvx_tools/microvm_tests.py", + "guest/common/init", + "guest/common/nvx-hostmount", + "guest/common/nvx-snapshot", + "guest/common/nvx-identity-probe", + ] + changes = ci.classify_ci_changes("pull_request", changed_files) + summary = ci.describe_ci_changes("pull_request", changed_files, changes) + self.assertIn("### Artifact-backed tests and benchmarks: run\n", summary) + self.assertIn("### NVX microVM tests: run\n", summary) + self.assertIn( + "### OpenVMM unit tests: skip\nNo input of this suite changed.\n", + summary, + ) + self.assertIn( + "### OpenVMM VMM tests: run\nSuite inputs changed: " + "`guest/common/init`, `guest/common/nvx-hostmount`, " + "`guest/common/nvx-snapshot`, and 1 more.\n", + summary, + ) + push = ci.describe_ci_changes( + "push", changed_files, ci.classify_ci_changes("push", changed_files) + ) + self.assertIn( + "### OpenVMM unit tests: run\n" + "Pushes run every OpenVMM suite whenever the tests run.\n", + push, + ) + documentation = ci.describe_ci_changes( + "pull_request", + ["doc/ci.md"], + ci.classify_ci_changes("pull_request", ["doc/ci.md"]), + ) + self.assertIn( + "### Artifact-backed tests and benchmarks: skip\n" + "Only documentation files changed.\n", + documentation, + ) + self.assertIn( + "### OpenVMM VMM tests: skip\nNo test input changed.\n", documentation + ) + + def test_classify_ci_changes_command_prints_every_output(self): + with tempfile.TemporaryDirectory() as temporary: + changed_files = Path(temporary) / "changed-files" + changed_files.write_text("guest/common/init\n", encoding="utf-8") + summary = Path(temporary) / "summary.md" + summary.write_text("before\n", encoding="utf-8") + args = nvx.parse_args( + [ + "classify-ci-changes", + "--event-name", + "pull_request", + "--changed-files", + os.fspath(changed_files), + "--summary", + os.fspath(summary), + ] + ) + stdout = io.StringIO() + with contextlib.redirect_stdout(stdout): + args.handler(args) + self.assertEqual( + stdout.getvalue().splitlines(), + [ + "run-tests=true", + "run-workloads=true", + "run-openvmm-unit-tests=false", + "run-openvmm-vmm-tests=true", + ], + ) + text = summary.read_text(encoding="utf-8") + self.assertTrue(text.startswith("before\n### ")) + self.assertIn("### OpenVMM VMM tests: run\n", text) + + def test_classify_ci_changes_command_reads_null_delimited_paths(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + changed_files = root / "changed-files" + changed_files.write_bytes( + "doc/notes.md\0kernel/patches/é\r\n.patch\0".encode() + ) + args = nvx.parse_args( + [ + "classify-ci-changes", + "--event-name", + "pull_request", + "--changed-files", + os.fspath(changed_files), + "--null", + "--summary", + os.fspath(root / "summary"), + ] + ) + stdout = io.StringIO() + with contextlib.redirect_stdout(stdout): + args.handler(args) + self.assertEqual( + stdout.getvalue().splitlines(), + [ + "run-tests=true", + "run-workloads=true", + "run-openvmm-unit-tests=false", + "run-openvmm-vmm-tests=true", + ], + ) + summary = (root / "summary").read_bytes().decode("utf-8") + self.assertIn("kernel/patches/é\r\n.patch", summary) + + @unittest.skipUnless(os.name == "posix", "requires bash and POSIX filenames") + def test_ci_change_detector_classifies_real_git_paths(self): + workflow = ( + BuildConstants.REPO_ROOT / ".github" / "workflows" / "ci.yml" + ).read_text(encoding="utf-8") + step = _workflow_step(workflow, "openvmm-changes", "Detect relevant changes") + lines = step.splitlines() + start = lines.index(" run: |") + 1 + script = "\n".join(line[10:] for line in lines[start:]) + classifier = BuildConstants.REPO_ROOT / "scripts" / "nvx.py" + shim = ( + f"python3() {{ {shlex.quote(sys.executable)} " + f'{shlex.quote(os.fspath(classifier))} "${{@:2}}"; }}\n' + ) + cases = ( + ("rename", "doc/input.patch", True), + ("delete", None, True), + ("unicode", "kernel/patches/é.patch", True), + ("newline", "kernel/patches/line\nbreak.patch", True), + ("documentation", "doc/notes.md", False), + ) + for name, destination, test_input in cases: + with self.subTest(name=name), tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + + def git(*arguments: str, directory: Path = root) -> str: + result = subprocess.run( + ["git", *arguments], + cwd=directory, + capture_output=True, + text=True, + check=True, + ) + return result.stdout.strip() + + git("init", "--quiet") + git("config", "user.name", "Reproduction") + git("config", "user.email", "repro@example.invalid") + git("config", "commit.gpgsign", "false") + source = root / "kernel" / "patches" / "input.patch" + source.parent.mkdir(parents=True) + source.write_text("suite input\n", encoding="utf-8") + git("add", ".") + git("commit", "--quiet", "-m", "baseline") + base = git("rev-parse", "HEAD") + if name == "delete": + source.unlink() + else: + assert destination is not None + target = root / destination + target.parent.mkdir(parents=True, exist_ok=True) + if name == "rename": + source.rename(target) + else: + target.write_text("new input\n", encoding="utf-8") + git("add", "--all") + git("commit", "--quiet", "-m", "changed paths") + for event in ("pull_request", "push"): + with self.subTest(event=event): + output = root / f"output-{event}" + environment = os.environ.copy() + environment.update( + EVENT_NAME=event, + BASE_SHA=base, + HEAD_SHA=git("rev-parse", "HEAD"), + GITHUB_OUTPUT=os.fspath(output), + GITHUB_STEP_SUMMARY=os.fspath(root / "summary"), + RUNNER_TEMP=temporary, + ) + result = subprocess.run( + ["bash", "-c", shim + script], + cwd=root, + env=environment, + capture_output=True, + text=True, + check=False, + ) + self.assertEqual(result.returncode, 0, result.stderr) + actual = dict( + line.split("=", 1) + for line in output.read_text(encoding="utf-8").splitlines() + ) + expected = { + "run-tests": str(test_input).lower(), + "run-workloads": str(test_input).lower(), + "run-openvmm-unit-tests": str( + test_input and event == "push" + ).lower(), + "run-openvmm-vmm-tests": str(test_input).lower(), + } + self.assertEqual(actual, expected) + + @unittest.skipUnless(os.name == "posix", "requires bash") + def test_ci_change_detector_runs_the_full_matrix_unless_classified(self): + workflow = ( + BuildConstants.REPO_ROOT / ".github" / "workflows" / "ci.yml" + ).read_text(encoding="utf-8") + step = _workflow_step(workflow, "openvmm-changes", "Detect relevant changes") + lines = step.splitlines() + start = lines.index(" run: |") + 1 + script = "\n".join(line[10:] for line in lines[start:]) + stub = """ +git() { + case "$1" in + cat-file) return "${GIT_CAT_FILE_STATUS}" ;; + diff) + printf '%b' "${CHANGED_FILES}" + return "${GIT_DIFF_STATUS}" + ;; + esac + echo "unexpected git command: $*" >&2 + return 1 +} +python3() { + if [[ -n "${CLASSIFIER:-}" ]]; then + "${CLASSIFIER}" "$@" + return + fi + printf '%b' "${CLASSIFICATION}" + return "${CLASSIFIER_STATUS}" +} +""" + complete = ( + "run-tests=true\\nrun-workloads=true\\n" + "run-openvmm-unit-tests=false\\nrun-openvmm-vmm-tests=false\\n" + ) + everything = dict.fromkeys(ci.CI_CHANGE_OUTPUTS, "true") + classified = { + "run-tests": "true", + "run-workloads": "true", + "run-openvmm-unit-tests": "false", + "run-openvmm-vmm-tests": "false", + } + cases = ( + ("missing base", {"BASE_SHA": ""}, everything), + ("zero base", {"BASE_SHA": "0" * 40}, everything), + ("missing commit", {"GIT_CAT_FILE_STATUS": "1"}, everything), + ("failed diff", {"GIT_DIFF_STATUS": "128"}, everything), + ("failed classifier", {"CLASSIFIER_STATUS": "1"}, everything), + ( + "missing output", + {"CLASSIFICATION": complete.rsplit("run-openvmm-vmm", 1)[0]}, + everything, + ), + ( + "duplicate output", + {"CLASSIFICATION": complete + "run-tests=false\\n"}, + everything, + ), + ( + "malformed output", + {"CLASSIFICATION": complete.replace("=true", "=yes", 1)}, + everything, + ), + ( + "classified", + {"CLASSIFICATION": "notice: ignored\\n" + complete}, + classified, + ), + ( + "real classifier", + { + "CLASSIFIER": sys.executable, + "CHANGED_FILES": "scripts/nvx_tools/microvm_tests.py\\0", + }, + classified, + ), + ) + for name, overrides, expected in cases: + with self.subTest(name=name), tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + environment = os.environ.copy() + environment.update( + { + "EVENT_NAME": "pull_request", + "BASE_SHA": "1" * 40, + "HEAD_SHA": "2" * 40, + "GITHUB_OUTPUT": os.fspath(root / "output"), + "GITHUB_STEP_SUMMARY": os.fspath(root / "summary"), + "RUNNER_TEMP": temporary, + "GIT_CAT_FILE_STATUS": "0", + "GIT_DIFF_STATUS": "0", + "CHANGED_FILES": "openvmm\\0", + "CLASSIFICATION": complete, + "CLASSIFIER_STATUS": "0", + } + ) + environment.update(overrides) + result = subprocess.run( + ["bash", "-c", stub + script], + cwd=BuildConstants.REPO_ROOT, + env=environment, + capture_output=True, + text=True, + check=False, + ) + self.assertEqual(result.returncode, 0, result.stderr) + output = (root / "output").read_text(encoding="utf-8").splitlines() + self.assertEqual(len(output), len(expected), output) + self.assertEqual(dict(line.split("=", 1) for line in output), expected) + summary = (root / "summary").read_text(encoding="utf-8") + if expected is everything: + self.assertIn("### Full matrix: run\n", summary) + else: + self.assertNotIn("Full matrix", summary) + + def test_flowey_downloads_use_retrying_curl(self): + action = ( + BuildConstants.REPO_ROOT + / ".github" + / "actions" + / "setup-curl" + / "action.yml" + ).read_text(encoding="utf-8") + windows_shim = ( + BuildConstants.REPO_ROOT + / ".github" + / "actions" + / "setup-curl" + / "curl-shim.rs" + ).read_text(encoding="utf-8") + workflow = ( + BuildConstants.REPO_ROOT / ".github" / "workflows" / "ci.yml" + ).read_text(encoding="utf-8") + microvm_workflow = ( + BuildConstants.REPO_ROOT + / ".github" + / "workflows" + / "run-nvx-microvm-tests.yml" + ).read_text(encoding="utf-8") + build_action = ( + BuildConstants.REPO_ROOT + / ".github" + / "actions" + / "build-openvmm" + / "action.yml" + ).read_text(encoding="utf-8") + + for option in ( + "--retry 5", + "--retry-all-errors", + "--retry-delay 2", + "--retry-max-time 90", + ): + self.assertEqual(action.count(option), 1) + for argument in ( + '"--retry"', + '"5"', + '"--retry-all-errors"', + '"--retry-delay"', + '"2"', + '"--retry-max-time"', + '"90"', + ): + self.assertIn(argument, windows_shim) + self.assertIn('system_curl="${NVX_SYSTEM_CURL:-$(command -v curl)}"', action) + self.assertIn("Get-Command curl.exe", action) + self.assertIn('Join-Path $ShimDirectory "curl.exe"', action) + self.assertIn("Get-Command rustc.exe", action) + self.assertNotIn("curl.cmd", action) + for job_name in ( + "openvmm-vmm-tests", + "openvmm-unit-tests", + ): + with self.subTest(job_name=job_name): + self.assertEqual( + _workflow_job(workflow, job_name).count( + "uses: ./.github/actions/setup-curl" + ), + 1, + ) + self.assertEqual( + microvm_workflow.count("uses: ./.github/actions/setup-curl"), + 1, + ) + self.assertIn("uses: ./.github/actions/setup-curl", build_action) + + @unittest.skipUnless(os.name == "nt", "Windows-specific curl resolution") + def test_windows_curl_shim_retries_http_500(self): + rustc = shutil.which("rustc") + system_curl = shutil.which("curl.exe") + if rustc is None or system_curl is None: + self.fail("Windows curl shim test requires rustc and curl.exe") + + class RetryHandler(http.server.BaseHTTPRequestHandler): + request_count = 0 + + def do_GET(self) -> None: + type(self).request_count += 1 + if type(self).request_count == 1: + self.send_error(http.HTTPStatus.INTERNAL_SERVER_ERROR) + return + body = b"ok" + self.send_response(http.HTTPStatus.OK) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, format: str, *args: object) -> None: + pass + + server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), RetryHandler) + server_thread = threading.Thread(target=server.serve_forever, daemon=True) + server_thread.start() + try: + with tempfile.TemporaryDirectory() as directory: + shim = Path(directory) / "curl.exe" + shim_source = ( + BuildConstants.REPO_ROOT + / ".github" + / "actions" + / "setup-curl" + / "curl-shim.rs" + ) + subprocess.run( + [ + rustc, + "--edition=2021", + "-C", + "opt-level=s", + "-o", + os.fspath(shim), + os.fspath(shim_source), + ], + check=True, + ) + environment = os.environ.copy() + environment["NVX_SYSTEM_CURL"] = system_curl + environment["PATH"] = directory + os.pathsep + environment["PATH"] + port = server.server_address[1] + result = subprocess.run( + [ + os.fspath(shim), + "--fail", + "-L", + f"http://127.0.0.1:{port}/asset", + ], + check=False, + capture_output=True, + env=environment, + ) + finally: + server.shutdown() + server.server_close() + server_thread.join(timeout=5) + + self.assertFalse( + server_thread.is_alive(), + "HTTP test server did not stop within 5 seconds", + ) + self.assertEqual(result.returncode, 0, result.stderr.decode(errors="replace")) + self.assertEqual(result.stdout, b"ok") + self.assertEqual(RetryHandler.request_count, 2) + + def test_ci_shares_openvmm_inputs_and_binary_artifacts(self): + workflow = ( + BuildConstants.REPO_ROOT / ".github" / "workflows" / "ci.yml" + ).read_text(encoding="utf-8") + build_workflow = ( + BuildConstants.REPO_ROOT + / ".github" + / "workflows" + / "build-openvmm-binary.yml" + ).read_text(encoding="utf-8") + microvm_workflow = ( + BuildConstants.REPO_ROOT + / ".github" + / "workflows" + / "run-nvx-microvm-tests.yml" + ).read_text(encoding="utf-8") + platform_workflow = ( + BuildConstants.REPO_ROOT / ".github" / "workflows" / "run-platform.yml" + ).read_text(encoding="utf-8") + build_action = ( + BuildConstants.REPO_ROOT + / ".github" + / "actions" + / "build-openvmm" + / "action.yml" + ).read_text(encoding="utf-8") + + for configuration in (workflow, build_action): + self.assertIn( + "openvmm/flowey-persist/flowey_lib_common__download_gh_release", + configuration, + ) + self.assertIn( + "openvmm/flowey-persist/flowey_lib_common__cache", + configuration, + ) + self.assertIn( + "openvmm-inputs-v1-${{ runner.os }}-${{ runner.arch }}-", + configuration, + ) + producers = { + "build-openvmm-linux-gnu": ( + "kvm", + "openvmm-linux-gnu", + ), + "build-openvmm-linux-musl": ( + "mshv", + "openvmm-linux-musl", + ), + "build-openvmm-windows-msvc": ( + "whp", + "openvmm-windows-msvc", + ), + } + self.assertEqual( + workflow.count("uses: ./.github/workflows/build-openvmm-binary.yml"), + len(producers), + ) + for job_name, (backend, artifact) in producers.items(): + with self.subTest(producer=job_name): + job = _workflow_job(workflow, job_name) + self.assertIn(f"backend: {backend}", job) + self.assertIn(f"artifact: {artifact}", job) + self.assertNotIn("target:", job) + self.assertNotIn("build-mode:", job) + self.assertNotIn("strategy:", job) + + consumers = { + "nvx-microvm-tests-kvm": ( + "build-openvmm-linux-gnu", + "openvmm-linux-gnu", + "run-nvx-microvm-tests.yml", + ), + "nvx-microvm-tests-mshv": ( + "build-openvmm-linux-gnu", + "openvmm-linux-gnu", + "run-nvx-microvm-tests.yml", + ), + "nvx-microvm-tests-whp": ( + "build-openvmm-windows-msvc", + "openvmm-windows-msvc", + "run-nvx-microvm-tests.yml", + ), + "nvx-microvm-debug-kvm": ( + "build-openvmm-linux-gnu", + "openvmm-linux-gnu", + "run-nvx-microvm-tests.yml", + ), + "nvx-microvm-debug-mshv": ( + "build-openvmm-linux-gnu", + "openvmm-linux-gnu", + "run-nvx-microvm-tests.yml", + ), + "nvx-microvm-debug-whp": ( + "build-openvmm-windows-msvc", + "openvmm-windows-msvc", + "run-nvx-microvm-tests.yml", + ), + "platform-kvm": ( + "build-openvmm-linux-gnu", + "openvmm-linux-gnu", + "run-platform.yml", + ), + "platform-mshv": ( + "build-openvmm-linux-musl", + "openvmm-linux-musl", + "run-platform.yml", + ), + "platform-whp": ( + "build-openvmm-windows-msvc", + "openvmm-windows-msvc", + "run-platform.yml", + ), + } + for job_name, (producer, artifact, reusable_workflow) in consumers.items(): + with self.subTest(consumer=job_name): + job = _workflow_job(workflow, job_name) + debug_kernel = ( + "debug-kernel, " + if job_name.startswith("nvx-microvm-debug-") + else "" + ) + cli = ( + "nvx-cli-windows" + if producer == "build-openvmm-windows-msvc" + else "nvx-cli-linux" + ) + self.assertIn( + f"needs: [artifacts, {debug_kernel}{producer}, {cli}, " + "openvmm-changes]", + job, + ) + self.assertIn(f"needs.{producer}.result == 'success'", job) + self.assertIn(f"needs.{cli}.result == 'success'", job) + self.assertIn(f"uses: ./.github/workflows/{reusable_workflow}", job) + self.assertIn(f"openvmm-artifact: {artifact}", job) + for unrelated_producer in producers.keys() - {producer}: + self.assertNotIn(unrelated_producer, job) + + self.assertIn("build/openvmm.provenance.json", build_workflow) + self.assertIn( + "name: ${{ inputs.artifact }}-executable", + build_workflow, + ) + self.assertIn( + "name: ${{ inputs.artifact }}-provenance", + build_workflow, + ) + self.assertEqual( + build_workflow.count("uses: ./.github/actions/build-openvmm"), + 1, + ) + self.assertIn("backend: ${{ inputs.backend }}", build_workflow) + self.assertNotIn("target: ${{ inputs.target }}", build_workflow) + self.assertNotIn("build-mode: ${{ inputs.build-mode }}", build_workflow) + self.assertEqual(build_workflow.count("overwrite: true"), 2) + self.assertEqual(build_workflow.count("retention-days: 1"), 2) + for consumer_workflow, download_count in ( + (microvm_workflow, 4), + (platform_workflow, 3), + ): + self.assertIn("path: openvmm/target/release", consumer_workflow) + self.assertIn("path: build", consumer_workflow) + self.assertEqual( + consumer_workflow.count("uses: actions/download-artifact@v8"), + download_count, + ) + self.assertIn( + "run: chmod +x openvmm/target/release/openvmm", + consumer_workflow, + ) + for configuration in ( + workflow, + build_workflow, + microvm_workflow, + platform_workflow, + ): + self.assertNotIn("path: .", configuration) + self.assertIn("openvmm-binary-v6-${{ inputs.backend }}-", build_action) + self.assertNotIn("openvmm-binary-v5-", build_action) + self.assertNotIn("inputs.target", build_action) + self.assertNotIn("inputs.build-mode", build_action) + self.assertNotIn("rustup target add", build_action) + self.assertNotIn("X86_64_UNKNOWN_LINUX_MUSL", build_action) + self.assertEqual( + build_action.count( + 'python3 scripts/nvx.py build-openvmm --backend "${{ inputs.backend }}"' + ), + 1, + ) + self.assertEqual( + build_action.count( + 'python scripts\\nvx.py build-openvmm --backend "${{ inputs.backend }}"' + ), + 1, + ) + self.assertIn("Verify OpenVMM provenance on Linux", build_action) + self.assertIn("Verify OpenVMM provenance on Windows", build_action) + self.assertNotIn("nvx-microvm-tests-v1", workflow) + self.assertNotIn("cargo-v2-", build_action) + self.assertNotIn("openvmm-tests-v2-", workflow) + self.assertNotIn("Restore OpenVMM test build", workflow) + self.assertNotIn("Save OpenVMM test build", workflow) + self.assertEqual( + build_workflow.count("uses: ./.github/actions/sccache"), + 2, + ) + for job_name, cache_namespace in ( + ("openvmm-vmm-tests", "vmm-tests"), + ("openvmm-unit-tests", "unit-tests"), + ): + with self.subTest(job_name=job_name): + job = _workflow_job(workflow, job_name) + self.assertEqual( + job.count("uses: ./.github/actions/sccache"), + 2, + ) + self.assertIn( + f"key: openvmm-inputs-v1-${{{{ runner.os }}}}-" + f"${{{{ runner.arch }}}}-{cache_namespace}-" + "${{ steps.openvmm.outputs.sha }}", + job, + ) + self.assertIn( + f"key: openvmm-cargo-v1-${{{{ runner.os }}}}-" + f"${{{{ runner.arch }}}}-{cache_namespace}-" + "${{ hashFiles('openvmm/Cargo.lock') }}", + job, + ) + self.assertIn( + "restore-keys: |\n" + " openvmm-inputs-v1-${{ runner.os }}-" + "${{ runner.arch }}-", + job, + ) + self.assertIn( + "restore-keys: |\n" + " openvmm-cargo-v1-${{ runner.os }}-" + "${{ runner.arch }}-", + job, + ) + self.assertNotIn("uses: actions/cache@v5", workflow) + self.assertNotIn("uses: actions/cache@v5", build_workflow) + self.assertNotIn("uses: actions/cache@v5", build_action) + + release_job = _workflow_job(workflow, "release") + performance_gate_job = _workflow_job(workflow, "performance-gate") + performance_persist_job = _workflow_job(workflow, "performance-persist") + for job_name in consumers: + self.assertIn(f" - {job_name}", release_job) + self.assertIn(f"needs.{job_name}.result", release_job) + self.assertIn(f" - {job_name}", performance_persist_job) + self.assertIn(f"needs.{job_name}.result", performance_persist_job) + for job_name in ("platform-kvm", "platform-mshv", "platform-whp"): + self.assertIn(job_name, performance_gate_job) + self.assertIn(f"needs.{job_name}.result", performance_gate_job) + + def test_copilot_setup_restores_ci_cache_entries(self): + github = BuildConstants.REPO_ROOT / ".github" + setup = (github / "workflows" / "copilot-setup-steps.yml").read_text( + encoding="utf-8" + ) + workflow = (github / "workflows" / "ci.yml").read_text(encoding="utf-8") + build_workflow = (github / "workflows" / "build-openvmm-binary.yml").read_text( + encoding="utf-8" + ) + build_action = (github / "actions" / "build-openvmm" / "action.yml").read_text( + encoding="utf-8" + ) + checkout_action = ( + github / "actions" / "checkout-openvmm" / "action.yml" + ).read_text(encoding="utf-8") + + # Guest artifact keys stay in sync because both sides use one action. + guest_artifacts = _workflow_step( + setup, "copilot-setup-steps", "Restore guest artifacts built by CI" + ) + self.assertEqual( + _yaml_field(guest_artifacts, "uses"), + ["./.github/actions/build-guest-artifacts"], + ) + self.assertEqual(_yaml_field(guest_artifacts, "restore-only"), ['"true"']) + + # The OpenVMM binary restore repeats the cache entry that the Linux GNU + # producer saves. A different key, path list, or compression method + # would make every session miss it without failing setup. + producer = _workflow_job(workflow, "build-openvmm-linux-gnu") + [backend] = _yaml_field(producer, "backend") + [binary] = _yaml_field(producer, "binary") + self.assertIn("binary-path: ${{ inputs.binary }}", build_workflow) + saved = _composite_action_step(build_action, "Save OpenVMM binary on Linux") + restored = _workflow_step( + setup, "copilot-setup-steps", "Restore the OpenVMM binary built by CI" + ) + self.assertEqual( + _yaml_field(restored, "key"), + [ + key.replace("${{ inputs.backend }}", backend).replace( + "steps.openvmm.outputs.sha", "steps.openvmm-cache.outputs.sha" + ) + for key in _yaml_field(saved, "key") + ], + ) + self.assertEqual( + _yaml_field(restored, "path"), + [ + path.replace("${{ inputs.binary-path }}", binary) + for path in _yaml_field(saved, "path") + ], + ) + + prepare = _workflow_step( + setup, "copilot-setup-steps", "Prepare OpenVMM cache restore" + ) + for revision_source in (checkout_action, prepare): + self.assertIn("sha=$(git rev-parse :openvmm)", revision_source) + + def tool_links(configuration: str) -> list[str]: + return [ + line.strip() + for line in configuration.splitlines() + if line.strip().startswith("ln -sf ") + ] + + self.assertEqual( + _yaml_field(saved, "PATH"), + ["${{ steps.cache-tools-linux.outputs.path }}"], + ) + self.assertEqual( + _yaml_field(restored, "PATH"), + ["${{ steps.openvmm-cache.outputs.path }}"], + ) + self.assertEqual( + tool_links(prepare), + tool_links( + _composite_action_step(build_action, "Prepare Linux cache tools") + ), + ) + + def test_guest_artifact_restore_only_guards_builds_and_cache_saves(self): + action = ( + BuildConstants.REPO_ROOT + / ".github" + / "actions" + / "build-guest-artifacts" + / "action.yml" + ).read_text(encoding="utf-8") + steps = _composite_action_steps(action) + guarded_steps = [ + step + for step in steps + if "docker build" in step or "uses: actions/cache/save@" in step + ] + + self.assertTrue(guarded_steps) + for step in guarded_steps: + try: + name = _yaml_scalar(step, "name") + except AssertionError: + name = "" + with self.subTest(step=name): + condition = _yaml_scalar(step, "if") + guard = "inputs.restore-only != 'true' && " + self.assertTrue(condition.startswith(guard), condition) + cache_condition = condition.removeprefix(guard) + self.assertIn("cache-hit != 'true'", cache_condition) + if "||" in cache_condition: + self.assertTrue( + _outer_parentheses_enclose(cache_condition), + cache_condition, + ) + + def test_guest_artifact_restore_only_leaves_restore_and_scratch_steps_active(self): + action = ( + BuildConstants.REPO_ROOT + / ".github" + / "actions" + / "build-guest-artifacts" + / "action.yml" + ).read_text(encoding="utf-8") + steps = _composite_action_steps(action) + restore_steps = [ + step for step in steps if "uses: actions/cache/restore@" in step + ] + + self.assertTrue(restore_steps) + scratch = _composite_action_step( + action, "Prepare Ubuntu sandbox scratch template" + ) + for step in (*restore_steps, scratch): + try: + name = _yaml_scalar(step, "name") + except AssertionError: + name = "" + with self.subTest(step=name): + # Other inputs, such as debug-kernel, may select these steps, + # but restore-only must leave them active. + try: + condition = _yaml_scalar(step, "if") + except AssertionError: + continue + self.assertNotIn("restore-only", condition) + + def test_ci_runs_the_debug_kernel_on_kvm_per_pull_request_and_all_on_dev(self): + workflow = ( + BuildConstants.REPO_ROOT / ".github" / "workflows" / "ci.yml" + ).read_text(encoding="utf-8") + microvm_workflow = ( + BuildConstants.REPO_ROOT + / ".github" + / "workflows" + / "run-nvx-microvm-tests.yml" + ).read_text(encoding="utf-8") + + artifacts = _workflow_job(workflow, "artifacts") + self.assertNotIn("debug-kernel", artifacts) + self.assertNotIn("vmlinux-debug", artifacts) + # The debug kernel builds on GitHub-hosted capacity beside the shared + # artifacts, under the same condition, and only the debug jobs wait. + debug_build = _workflow_job(workflow, "debug-kernel") + self.assertIn("runs-on: ubuntu-latest", debug_build) + self.assertIn('debug-kernel: "true"', debug_build) + self.assertIn('guest-images: "false"', debug_build) + self.assertIn("name: guest-debug-kernel", debug_build) + for name in ( + KernelBuildConstants.DEBUG_BINARY_NAME, + KernelBuildConstants.DEBUG_CONFIG_NAME, + ): + self.assertIn(f"build/{name}\n", debug_build) + + def condition(job: str) -> str: + return job[job.index(" if: >-\n") : job.index(" runs-on:")] + + self.assertEqual(condition(debug_build), condition(artifacts)) + for consumer in ("release", "performance-persist"): + with self.subTest(consumer=consumer): + job = _workflow_job(workflow, consumer) + self.assertIn(" - debug-kernel\n", job) + self.assertIn("needs.debug-kernel.result == 'success' &&", job) + kvm = _workflow_job(workflow, "nvx-microvm-debug-kvm") + self.assertIn("github.event.pull_request.head.repo.full_name", kvm) + for job_name in ci.REQUIRED_CI_MICROVM_DEBUG_PUSH_JOBS: + with self.subTest(job=job_name): + job = _workflow_job(workflow, job_name) + self.assertIn("github.event_name == 'push' &&", job) + self.assertNotIn("pull_request", job) + for job_name in ( + *ci.REQUIRED_CI_MICROVM_DEBUG_JOBS, + *ci.REQUIRED_CI_MICROVM_DEBUG_PUSH_JOBS, + ): + with self.subTest(job=job_name): + job = _workflow_job(workflow, job_name) + self.assertIn("debug-kernel: true", job) + self.assertIn("needs.debug-kernel.result == 'success' &&", job) + self.assertIn(job_name, ci.REQUIRED_CI_RESULT_ENVIRONMENTS) + for consumer in ("release", "performance-persist"): + self.assertIn( + f"needs.{job_name}.result == 'success' ||\n" + f" needs.{job_name}.result == 'skipped'", + _workflow_job(workflow, consumer), + ) + + self.assertIn(" debug-kernel:\n", microvm_workflow) + self.assertIn(" type: boolean\n", microvm_workflow) + download = microvm_workflow.index("name: guest-debug-kernel") + self.assertIn( + "if: inputs.debug-kernel", + microvm_workflow[download - 120 : download], + ) + debug_flag = "${{ inputs.debug-kernel && '--debug-kernel' || '' }}" + self.assertEqual(microvm_workflow.count(debug_flag), 2) + # The debug run replaces every production-kernel workload rather than + # adding to it: each step that drives nvx.py without the debug kernel + # flag boots build/vmlinux, so it must skip the debug jobs (#371). + guard = " && !inputs.debug-kernel" + debug_steps: list[str] = [] + production_steps: list[str] = [] + for step in _workflow_steps(microvm_workflow, "test"): + try: + script = _yaml_scalar(step, "run") + except AssertionError: + continue + if "nvx.py" not in script: + continue + name = _yaml_scalar(step, "name") + try: + step_condition = _yaml_scalar(step, "if") + except AssertionError: + step_condition = "" + with self.subTest(step=name): + if debug_flag in script: + debug_steps.append(name) + self.assertNotIn("debug-kernel", step_condition) + continue + production_steps.append(name) + self.assertTrue(step_condition.endswith(guard), step_condition) + selector = step_condition.removesuffix(guard) + if "||" in selector: + self.assertTrue( + _outer_parentheses_enclose(selector), step_condition + ) + self.assertEqual(len(debug_steps), 2) + self.assertTrue(production_steps) + + def test_benchmarks_rely_on_the_microvm_correctness_jobs(self): + # #286: the benchmark action ran a second smp-lapic gate before + # acceptance. The microVM correctness jobs already gate the required + # status, the development release, and performance persistence. + workflow = ( + BuildConstants.REPO_ROOT / ".github" / "workflows" / "ci.yml" + ).read_text(encoding="utf-8") + action = ( + BuildConstants.REPO_ROOT + / ".github" + / "actions" + / "run-benchmark" + / "action.yml" + ).read_text(encoding="utf-8") + + self.assertNotIn("test-microvm", action) + self.assertNotIn("lapic-correctness", action) + for job_name in ci.REQUIRED_CI_MICROVM_TEST_JOBS: + with self.subTest(job=job_name): + self.assertIn(job_name, ci.REQUIRED_CI_RESULT_ENVIRONMENTS) + for consumer in ("release", "performance-persist"): + job = _workflow_job(workflow, consumer) + self.assertIn(f" - {job_name}\n", job) + self.assertIn( + f"needs.{job_name}.result == 'success' ||\n" + f" needs.{job_name}.result == 'skipped'", + job, + ) + + def test_ci_runs_openvmm_vmm_tests_per_backend_and_unit_tests_per_os(self): + workflow = ( + BuildConstants.REPO_ROOT / ".github" / "workflows" / "ci.yml" + ).read_text(encoding="utf-8") + vmm_tests_job = _workflow_job(workflow, "openvmm-vmm-tests") + unit_tests_job = _workflow_job(workflow, "openvmm-unit-tests") + + def legs(job: str) -> list[tuple[str, str, str]]: + return re.findall( + r"^ - name: (.+)\n" + r" backend: (\S+)\n" + r" runner: '(.+)'$", + job, + re.MULTILINE, + ) + + self.assertEqual( + legs(vmm_tests_job), + [ + ( + "Linux / KVM", + "kvm", + '["self-hosted", "linux", "kvm", "virtual-machine"]', + ), + ( + "Linux / MSHV", + "mshv", + '["self-hosted", "linux", "mshv", "virtual-machine"]', + ), + ( + "Windows / WHP", + "whp", + '["self-hosted", "windows", "whp", "virtual-machine"]', + ), + ], + ) + # The unit tests that need /dev/kvm or /dev/mshv are ignored, so one + # Linux runner covers both Linux backends. + self.assertEqual( + legs(unit_tests_job), + [ + ( + "Linux", + "mshv", + '["self-hosted", "linux", "mshv", "virtual-machine"]', + ), + ( + "Windows", + "whp", + '["self-hosted", "windows", "whp", "virtual-machine"]', + ), + ], + ) + self.assertIn( + """ - name: Run OpenVMM unit tests on Linux + if: runner.os != 'Windows' + shell: bash + run: python3 scripts/nvx.py test-openvmm-unit""", + unit_tests_job, + ) + self.assertIn( + """ - name: Run OpenVMM unit tests on Windows + if: runner.os == 'Windows' + shell: powershell + run: python scripts\\nvx.py test-openvmm-unit""", + unit_tests_job, + ) + self.assertEqual( + vmm_tests_job.count( + 'scripts/nvx.py test-openvmm --backend "${{ matrix.backend }}"' + ), + 2, + ) + self.assertEqual( + vmm_tests_job.count( + 'scripts\\nvx.py test-openvmm --backend "${{ matrix.backend }}"' + ), + 1, + ) + self.assertNotIn("test-openvmm-unit", vmm_tests_job) + self.assertEqual(unit_tests_job.count("scripts/nvx.py test-openvmm-unit"), 1) + self.assertEqual( + unit_tests_job.count("scripts\\nvx.py test-openvmm-unit"), + 1, + ) + self.assertNotIn("test-openvmm --backend", unit_tests_job) + + def test_ci_validates_the_cli_once_per_os_after_checking_out_openvmm(self): + # The CLI tests read OpenVMM's pinned CPU profiles from the submodule, + # which a runner's workspace holds at the pinned revision only after + # the job checks it out. The tests do not depend on the backend, so + # one GitHub-hosted job per OS runs them instead of every hypervisor + # job, and the microVM and platform jobs of each OS wait for it. + workflows = BuildConstants.REPO_ROOT / ".github" / "workflows" + workflow = (workflows / "ci.yml").read_text(encoding="utf-8") + self.assertEqual(ci.REQUIRED_CI_CLI_JOBS, ("nvx-cli-linux", "nvx-cli-windows")) + for job_name, runner in ( + ("nvx-cli-linux", "ubuntu-latest"), + ("nvx-cli-windows", "windows-latest"), + ): + with self.subTest(job=job_name): + job = _workflow_job(workflow, job_name) + self.assertIn(f" runs-on: {runner}\n", job) + self.assertNotIn(" needs:", job) + self.assertIn( + "github.event.pull_request.head.repo.full_name " + "== github.repository", + job, + ) + self.assertEqual(job.count("uses: ./.github/actions/validate-nvx"), 1) + for prerequisite in ( + "uses: ./.github/actions/checkout-openvmm", + "uses: actions/setup-python@v6", + ): + self.assertLess( + job.index(prerequisite), + job.index("uses: ./.github/actions/validate-nvx"), + ) + # The release that the self-hosted runners provide. + self.assertIn('python-version: "3.12"', job) + self.assertIn(job_name, ci.REQUIRED_CI_RESULT_ENVIRONMENTS) + windows = _workflow_job(workflow, "nvx-cli-windows") + self.assertIn("rust-toolchain.toml", windows) + self.assertIn("--profile minimal --no-self-update", windows) + self.assertLess( + windows.index("rustup toolchain install"), + windows.index("uses: ./.github/actions/validate-nvx"), + ) + + configurations = { + name: (workflows / name).read_text(encoding="utf-8") + for name in ( + "ci.yml", + "build-openvmm-binary.yml", + "run-nvx-microvm-tests.yml", + "run-platform.yml", + ) + } + self.assertEqual( + sum( + configuration.count("uses: ./.github/actions/validate-nvx") + for configuration in configurations.values() + ), + len(ci.REQUIRED_CI_CLI_JOBS), + ) + for job_name in ( + *ci.REQUIRED_CI_MICROVM_TEST_JOBS, + *ci.REQUIRED_CI_MICROVM_DEBUG_JOBS, + *ci.REQUIRED_CI_MICROVM_DEBUG_PUSH_JOBS, + *ci.REQUIRED_CI_PLATFORM_JOBS, + ): + cli = "nvx-cli-windows" if job_name.endswith("-whp") else "nvx-cli-linux" + with self.subTest(consumer=job_name): + job = _workflow_job(workflow, job_name) + needs = re.search(r"^ needs: \[(.+)\]$", job, re.MULTILINE) + assert needs is not None + self.assertIn(cli, needs.group(1).split(", ")) + self.assertIn(f" needs.{cli}.result == 'success' &&\n", job) + for other in set(ci.REQUIRED_CI_CLI_JOBS) - {cli}: + self.assertNotIn(other, job) + for job_name in ("openvmm-vmm-tests", "openvmm-unit-tests"): + with self.subTest(job=job_name): + self.assertNotIn("nvx-cli", _workflow_job(workflow, job_name)) + for consumer in ("release", "performance-persist", "required-status-check"): + job = _workflow_job(workflow, consumer) + for cli in ci.REQUIRED_CI_CLI_JOBS: + with self.subTest(consumer=consumer, cli=cli): + self.assertIn(f" - {cli}\n", job) + if consumer != "required-status-check": + self.assertIn( + f" needs.{cli}.result == 'success' &&\n", job + ) + + def test_ci_preserves_failed_openvmm_test_diagnostics(self): + workflow = ( + BuildConstants.REPO_ROOT / ".github" / "workflows" / "ci.yml" + ).read_text(encoding="utf-8") + job = _workflow_job(workflow, "openvmm-vmm-tests") + step_name = " - name: Upload OpenVMM test diagnostics" + self.assertIn(step_name, job) + upload = job.split(step_name, 1)[1].split("\n - name:", 1)[0] + + self.assertIn("\n if: failure()", upload) + self.assertIn("uses: actions/upload-artifact@v7", upload) + self.assertIn( + "name: openvmm-vmm-tests-${{ runner.os }}-${{ matrix.backend }}-" + "${{ github.run_id }}-${{ github.run_attempt }}", + upload, + ) + self.assertIn( + "path: ${{ runner.os == 'Windows' && " + "format('{0}/{1}/test_results', runner.temp, matrix.backend) || " + "'openvmm/target/vmm_tests/test_results' }}", + upload, + ) + self.assertIn("if-no-files-found: warn", upload) + self.assertIn("retention-days: 7", upload) + self.assertLess( + job.index(step_name), + job.index(" - name: Report sccache"), + ) + + def test_runner_setup_pins_and_validates_sccache(self): + workflow = ( + BuildConstants.REPO_ROOT / ".github" / "workflows" / "ci.yml" + ).read_text(encoding="utf-8") + action = ( + BuildConstants.REPO_ROOT / ".github" / "actions" / "sccache" / "action.yml" + ).read_text(encoding="utf-8") + validate_runner = ( + BuildConstants.REPO_ROOT + / ".github" + / "actions" + / "validate-runner" + / "action.yml" + ).read_text(encoding="utf-8") + windows_setup = ( + BuildConstants.REPO_ROOT / "scripts" / "setup" / "setup-windows-whp.ps1" + ).read_text(encoding="utf-8") + linux_setup = ( + BuildConstants.REPO_ROOT / "scripts" / "setup" / "setup-linux-runner.sh" + ).read_text(encoding="utf-8") + manifest = ( + BuildConstants.REPO_ROOT / "scripts" / "setup" / "tool-versions.conf" + ).read_text(encoding="utf-8") + + # The manifest pins sccache and its checksums, and both runner setups + # install that release; test_tool_versions.py checks the manifest. + self.assertRegex(manifest, r"(?m)^sccache\.version=[0-9]+\.[0-9]+\.[0-9]+$") + for platform in ("linux-x86_64", "windows-x86_64"): + self.assertRegex( + manifest, + rf"(?m)^sccache\.artifacts\.{platform}\.sha256=[0-9a-f]{{64}}$", + ) + self.assertIn("\n tool_manifest_artifact sccache\n", linux_setup) + self.assertIn( + '\n$sccacheRelease = Get-ToolArtifact $ToolManifest "sccache" $ToolPlatform\n', + windows_setup, + ) + for configuration in (windows_setup, linux_setup): + self.assertIn("RUSTC_WRAPPER", configuration) + self.assertIn("CARGO_INCREMENTAL", configuration) + self.assertIn("SCCACHE_DIR", configuration) + self.assertIn("SCCACHE_CACHE_SIZE", configuration) + self.assertIn("sccache --zero-stats", action) + self.assertIn("sccache --show-stats", action) + self.assertIn("sccache --stop-server", action) + self.assertIn("SCCACHE_IDLE_TIMEOUT", action) + self.assertIn("sccache --version", validate_runner) + self.assertNotRegex( + workflow, + r"(?m)^\s+path: openvmm/target\s*$", + ) + self.assertLess( + linux_setup.index('test -f "${runner_directory}/.runner"'), + linux_setup.index('test -w "$runner_sccache_dir"'), + ) + + def test_linux_unnamed_setup_defers_existing_runner_validation(self): + linux_setup = ( + BuildConstants.REPO_ROOT / "scripts" / "setup" / "setup-linux-runner.sh" + ).read_text(encoding="utf-8") + environment_check = linux_setup.split("check_environment() {\n", 1)[1].split( + "\n}\n", 1 + )[0] + runner_checks = environment_check.split( + " validate_runner_state_paths\n validate_runner_work_paths\n", 1 + )[1] + guard = 'if [ "$configure_runner" = false ] && [ "$check_only" = false ]; then' + self.assertIn(guard, runner_checks) + self.assertLess( + runner_checks.index(guard), + runner_checks.index('test -w "$runner_sccache_dir"'), + ) + if os.name != "posix": + return + + harness = ( + "set -eu\n" + "runner_directory=/missing-runner\n" + "runner_sccache_dir=/missing-runner/_work/_sccache\n" + "validate_runner_state_paths() { echo state-checked; }\n" + "validate_runner_work_paths() { echo work-checked; }\n" + 'run_as_root() { [ "$1" = test ] && [ "$2" = -f ]; }\n' + "check_runner_state() {\n" + " validate_runner_state_paths\n" + " validate_runner_work_paths\n" + f"{runner_checks}\n" + "}\n" + ) + for configure_runner, check_only in ( + ("false", "false"), + ("false", "true"), + ("true", "false"), + ): + with self.subTest(configure_runner=configure_runner, check_only=check_only): + result = subprocess.run( + [ + "sh", + "-c", + harness + f"{configure_runner=}\n{check_only=}\n" + "check_runner_state\n", + ], + capture_output=True, + text=True, + timeout=10, + check=False, + ) + self.assertEqual( + result.returncode == 0, + configure_runner == check_only == "false", + result.stderr, + ) + if result.returncode == 0: + self.assertIn("state-checked", result.stdout) + self.assertIn("work-checked", result.stdout) + + def test_linux_setup_installs_openvmm_perl_modules(self): + setup_directory = BuildConstants.REPO_ROOT / "scripts" / "setup" + configurations = ( + (setup_directory / "setup-linux-runner.sh").read_text(encoding="utf-8"), + (setup_directory / "setup-linux-mshv.sh").read_text(encoding="utf-8"), + ) + + for configuration in configurations: + for package in ( + "perl-FindBin", + "perl-IPC-Cmd", + "perl-Time-Piece", + "perl-lib", + ): + self.assertIn(package, configuration) + + def test_windows_runner_requires_inbox_pcat_firmware(self): + windows_setup = ( + BuildConstants.REPO_ROOT / "scripts" / "setup" / "setup-windows-whp.ps1" + ).read_text(encoding="utf-8") + validate_runner = ( + BuildConstants.REPO_ROOT + / ".github" + / "actions" + / "validate-runner" + / "action.yml" + ).read_text(encoding="utf-8") + + self.assertIn('"Microsoft-Hyper-V"', windows_setup) + for configuration in (windows_setup, validate_runner): + for firmware in ( + "vmfirmwarepcat.dll", + "vmfirmware.dll", + "VmEmulatedDevices.dll", + ): + self.assertIn(firmware, configuration) + + def test_linux_runners_report_the_invariant_tsc_as_evidence(self): + validate_runner = ( + BuildConstants.REPO_ROOT + / ".github" + / "actions" + / "validate-runner" + / "action.yml" + ).read_text(encoding="utf-8") + linux_setup = ( + BuildConstants.REPO_ROOT / "scripts" / "setup" / "setup-linux-runner.sh" + ).read_text(encoding="utf-8") + + # Neither CI nor provisioning rejects a host without an invariant + # TSC: the guest warp probe measures what its guests observe (#265). + step = validate_runner.split(" - name: Report host TSC\n", 1)[1] + step = step.split("\n\n - name: ", 1)[0] + self.assertIn(" if: runner.os != 'Windows'\n", step) + check = "\n".join( + line.removeprefix(" ") + for line in step.split(" run: |\n", 1)[1].splitlines() + ) + function = linux_setup.split("report_invariant_tsc() {\n", 1)[1] + function = "report_invariant_tsc() {\n" + function.split("\n}\n", 1)[0] + function += "\n}\n" + self.assertNotRegex(function, r"\bdie\b") + self.assertLess( + linux_setup.index("report_invariant_tsc /proc/cpuinfo\n"), + linux_setup.index('sudo -n true || die "passwordless sudo is required"'), + ) + if os.name != "posix": + return + + with tempfile.TemporaryDirectory() as temporary: + cpuinfo = Path(temporary) / "cpuinfo" + for flags, invariant in ( + ("fpu tsc constant_tsc nonstop_tsc tsc_known_freq", True), + ("fpu tsc constant_tsc tsc_known_freq", False), + ("fpu tsc constant_tsc nonstop_tsc_x", False), + ): + cpuinfo.write_text( + f"model name\t: Test CPU\nflags\t\t: {flags}\n", + encoding="utf-8", + ) + with self.subTest(flags=flags, check="validate-runner"): + result = subprocess.run( + ["bash", "-c", check.replace("/proc/cpuinfo", str(cpuinfo))], + capture_output=True, + text=True, + timeout=10, + check=False, + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("CPU: Test CPU", result.stdout) + self.assertIn( + "TSC flag nonstop_tsc: " + + ("present" if invariant else "absent"), + result.stdout, + ) + self.assertEqual( + "::notice::Runner host does not expose an invariant TSC" + in result.stdout, + not invariant, + ) + self.assertNotIn("::error::", result.stdout + result.stderr) + with self.subTest(flags=flags, check="setup-linux-runner"): + result = subprocess.run( + [ + "sh", + "-c", + f"set -eu\n{function}report_invariant_tsc '{cpuinfo}'\n", + ], + capture_output=True, + text=True, + timeout=10, + check=False, + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual( + "warning: host does not expose an invariant TSC" + in result.stderr, + not invariant, + ) + + def test_runners_qualify_their_host_time_before_every_job(self): + validate_runner = ( + BuildConstants.REPO_ROOT + / ".github" + / "actions" + / "validate-runner" + / "action.yml" + ).read_text(encoding="utf-8") + + # The doctor follows the host TSC report on Linux: the backend, the CPU + # fingerprint and generation, and the TSC rate stability. Jobs that + # haven't downloaded OpenVMM skip its CPU profile check (H2) and + # preflight (H3), and only jobs that ask for it run the guest warp + # probe (H6). + self.assertLess( + validate_runner.index(" - name: Report host TSC\n"), + validate_runner.index(" - name: Qualify host time on Linux\n"), + ) + for name, shell, command, summary in ( + ( + "Qualify host time on Linux", + "bash", + "python3 scripts/nvx.py doctor", + '--summary "${GITHUB_STEP_SUMMARY}"', + ), + ( + "Qualify host time on Windows", + "powershell", + "python scripts\\nvx.py doctor", + '--summary "$env:GITHUB_STEP_SUMMARY"', + ), + ): + with self.subTest(step=name): + step = validate_runner.split(f" - name: {name}\n", 1)[1] + step = step.split("\n\n", 1)[0] + self.assertIn(f" shell: {shell}\n", step) + self.assertIn(command, step) + self.assertIn('--backend "${{ inputs.backend }}"', step) + self.assertIn("--checks H1 H2 H4\n", step) + self.assertIn( + "${{ inputs.warp-probe == 'true' && 'H6' || '' }}\n", step + ) + fingerprint = ( + '"${RUNNER_TEMP}/nvx-cpu-fingerprint.json"' + if shell == "bash" + else '"$env:RUNNER_TEMP\\nvx-cpu-fingerprint.json"' + ) + self.assertIn( + "${{ inputs.verify-openvmm == 'true' && " + f"'H3 --cpu-fingerprint {fingerprint}' || '--no-openvmm' }}}}\n", + step, + ) + self.assertIn("--ci-schedule\n", step) + self.assertIn(summary, step) + args = nvx.parse_args( + ["doctor", "--backend", "kvm", "--checks", "H1", "H2", "H4", "H3"] + + ["--cpu-fingerprint", "fingerprint.json", "--ci-schedule"] + ) + self.assertEqual(args.checks, ["H1", "H2", "H4", "H3"]) + self.assertEqual(args.cpu_fingerprint, Path("fingerprint.json")) + args = nvx.parse_args( + ["doctor", "--backend", "kvm", "--checks", "H1", "H2", "H4", "H6", "H3"] + + ["--cpu-fingerprint", "fingerprint.json", "--ci-schedule"] + ) + self.assertEqual(args.checks, ["H1", "H2", "H4", "H6", "H3"]) + args = nvx.parse_args( + ["doctor", "--backend", "kvm", "--checks", "H1", "H2", "H4"] + + ["--no-openvmm", "--ci-schedule"] + ) + self.assertTrue(args.no_openvmm) + self.assertTrue(args.ci_schedule) + self.assertIn(" verify-openvmm:\n", validate_runner) + self.assertIn(' default: "false"\n', validate_runner) + warp_probe = validate_runner.split(" warp-probe:\n", 1)[1] + warp_probe = warp_probe.split("\nruns:\n", 1)[0] + self.assertIn("requires\n verify-openvmm\n", warp_probe) + self.assertIn(' default: "false"\n', warp_probe) + # A failed CPU profile check keeps OpenVMM's fingerprint. + upload = validate_runner.split(" - name: Upload the CPU fingerprint\n")[1] + upload = upload.split("\n\n", 1)[0] + self.assertIn("if: failure() && inputs.verify-openvmm == 'true'\n", upload) + self.assertIn("path: ${{ runner.temp }}/nvx-cpu-fingerprint.json\n", upload) + self.assertIn("if-no-files-found: ignore\n", upload) + # Jobs that run OpenVMM qualify the runner after downloading it. + benchmark = ( + BuildConstants.REPO_ROOT + / ".github" + / "actions" + / "run-benchmark" + / "action.yml" + ).read_text(encoding="utf-8") + self.assertNotIn("download-artifact", benchmark) + for workflow_name in ("run-nvx-microvm-tests.yml", "run-platform.yml"): + with self.subTest(workflow=workflow_name): + workflow = ( + BuildConstants.REPO_ROOT / ".github" / "workflows" / workflow_name + ).read_text(encoding="utf-8") + validate = workflow.index(" - name: Validate runner\n") + self.assertEqual(workflow.count(" - name: Validate runner\n"), 1) + self.assertIn( + 'verify-openvmm: "true"', workflow[validate : validate + 220] + ) + # The microVM scenarios probe after every boot and restore; + # the benchmarks boot guests without such probes. + self.assertEqual( + 'warp-probe: "true"' in workflow[validate : validate + 260], + workflow_name == "run-platform.yml", + ) + for step in ( + " - name: Download guest artifacts\n", + " - name: Download OpenVMM executable\n", + " - name: Make OpenVMM executable\n", + ): + self.assertLess(workflow.index(step), validate) + self.assertLess( + validate_runner.index("Validate Linux toolchain"), + validate_runner.index("Qualify host time on Linux"), + ) + + def test_runner_setups_install_backend_native_openvmm_targets(self): + linux_setup = ( + BuildConstants.REPO_ROOT / "scripts" / "setup" / "setup-linux-runner.sh" + ).read_text(encoding="utf-8") + windows_setup = ( + BuildConstants.REPO_ROOT / "scripts" / "setup" / "setup-windows-whp.ps1" + ).read_text(encoding="utf-8") + + for target in ( + "x86_64-unknown-none", + "x86_64-unknown-uefi", + ): + self.assertIn(target, linux_setup) + self.assertIn(target, windows_setup) + self.assertIn("x86_64-unknown-linux-musl", linux_setup) + for cross_platform_tool in ( + "x86_64-pc-windows-gnu", + "gcc-mingw-w64-x86-64-win32", + "mingw64-gcc", + "x86_64-w64-mingw32-gcc", + "x86_64-w64-mingw32-dlltool", + ): + self.assertNotIn(cross_platform_tool, linux_setup) + + def test_repository_pins_one_exact_rust_toolchain(self): + root = BuildConstants.REPO_ROOT + [toolchain] = re.findall( + r'^channel = "([^"]*)"$', + (root / "rust-toolchain.toml").read_text(encoding="utf-8"), + re.MULTILINE, + ) + # A floating channel such as `stable` lets runners drift apart (#402). + self.assertRegex(toolchain, r"^[0-9]+\.[0-9]+\.[0-9]+$") + self.assertEqual(OpenVMMBuildConstants.RUST_TOOLCHAIN, toolchain) + self.assertIn("rust-toolchain.toml", ReleaseBuildConstants.PROJECT_SOURCE_PATHS) + + setup = root / "scripts" / "setup" + # The setup scripts and the Specula runner setup take the release from + # the setup tool manifest, which rustup cannot read. + self.assertEqual( + re.findall( + r"^rust\.toolchain=(.*)$", + (setup / "tool-versions.conf").read_text(encoding="utf-8"), + re.MULTILINE, + ), + [toolchain], + ) + for path, reader in ( + (setup / "setup-linux-runner.sh", "\n RUST_TOOLCHAIN=$tool_value\n"), + (setup / "setup-linux-mshv.sh", "\n RUST_TOOLCHAIN=$tool_value\n"), + ( + setup / "setup-windows-whp.ps1", + '\n$RustToolchain = Get-ToolManifestValue $ToolManifest "rust.toolchain"\n', + ), + ( + root / ".github" / "specula" / "setup-runner.sh", + '\nRUST_VERSION="$(tool_value rust.toolchain "$VERSION_PATTERN")"\n', + ), + ): + with self.subTest(path=path.name): + self.assertIn(reader, path.read_text(encoding="utf-8")) + + def test_ci_takes_rust_toolchain_from_repository_pin(self): + github = BuildConstants.REPO_ROOT / ".github" + workflows = github / "workflows" + + validate_runner = ( + github / "actions" / "validate-runner" / "action.yml" + ).read_text(encoding="utf-8") + for step_name in ("Validate Linux toolchain", "Validate Windows toolchain"): + with self.subTest(step=step_name): + step = _composite_action_step(validate_runner, step_name) + self.assertIn("rust-toolchain.toml", step) + # Report a missing toolchain without writing the trusted RUSTUP_HOME. + self.assertEqual(_yaml_field(step, "RUSTUP_AUTO_INSTALL"), ['"0"']) + + crate_action = ( + github / "actions" / "check-aci-edge-sandboxes" / "action.yml" + ).read_text(encoding="utf-8") + self.assertEqual(_yaml_field(crate_action, "default"), ['""']) + install = _composite_action_step(crate_action, "Install Rust toolchains") + self.assertIn("../rust-toolchain.toml", install) + self.assertIn("REQUESTED_TOOLCHAIN: ${{ inputs.toolchain }}", install) + self.assertEqual(crate_action.count("${{ inputs.toolchain }}"), 1) + + setup = (workflows / "copilot-setup-steps.yml").read_text(encoding="utf-8") + self.assertIn(" - rust-toolchain.toml\n", setup) + self.assertIn( + "rust-toolchain.toml", + _workflow_step( + setup, "copilot-setup-steps", "Resolve pinned tool versions" + ), + ) + rust = _workflow_step(setup, "copilot-setup-steps", "Install Rust toolchains") + self.assertIn('rustup default "${RUST_TOOLCHAIN}"', rust) + self.assertNotIn("stable", rust) + + # A binary built by another compiler must not satisfy the cache. + build_action = (github / "actions" / "build-openvmm" / "action.yml").read_text( + encoding="utf-8" + ) + self.assertEqual( + build_action.count( + "${{ hashFiles('rust-toolchain.toml') }}-${{ steps.openvmm.outputs.sha }}" + ), + 4, + ) + + # A new release reruns every suite that it builds. + for pattern in ( + ci.CI_TEST_INPUT_PATHS, + ci.OPENVMM_UNIT_TEST_INPUT_PATHS, + ci.OPENVMM_VMM_TEST_INPUT_PATHS, + ): + self.assertIsNotNone(pattern.match("rust-toolchain.toml")) + + def test_release_actions_use_deterministic_immutable_tooling(self): + package_action = ( + BuildConstants.REPO_ROOT + / ".github" + / "actions" + / "package-release" + / "action.yml" + ).read_text(encoding="utf-8") + publish_action = ( + BuildConstants.REPO_ROOT + / ".github" + / "actions" + / "publish-development-release" + / "action.yml" + ).read_text(encoding="utf-8") + + self.assertEqual(package_action.count("archive-release"), 2) + self.assertIn( + "python scripts\\nvx.py materialize-kernel-provenance-inputs", + package_action, + ) + self.assertLess( + package_action.index( + "python scripts\\nvx.py materialize-kernel-provenance-inputs" + ), + package_action.index("- name: Package Windows release"), + ) + self.assertNotIn("git checkout-index", package_action) + self.assertNotIn("tar -czf", package_action) + self.assertNotIn("Compress-Archive", package_action) + self.assertIn( + "python3 -u scripts/publish_development_release.py", + publish_action, + ) + self.assertNotIn("--clobber", publish_action) + + def test_ci_artifact_actions_use_node24(self): + configurations = "\n".join( + path.read_text(encoding="utf-8") + for pattern in ("*.yml", "*.yaml") + for path in (BuildConstants.REPO_ROOT / ".github").rglob(pattern) + ) + + self.assertNotRegex(configurations, r"actions/upload-artifact@v[1-5]\b") + self.assertNotRegex(configurations, r"actions/download-artifact@v[1-6]\b") + + def test_benchmark_diagnostics_preserve_each_workflow_attempt(self): + action = ( + Path(__file__).parents[1] + / ".github" + / "actions" + / "run-benchmark" + / "action.yml" + ).read_text(encoding="utf-8") + diagnostics = _composite_action_step(action, "Upload benchmark diagnostics") + name = ( + "benchmark-diagnostics-${{ inputs.platform }}-${{ github.run_id }}" + "-attempt-${{ github.run_attempt }}" + ) + + self.assertIn(" if: always()", diagnostics) + self.assertIn(" uses: actions/upload-artifact@v7", diagnostics) + self.assertIn(" path: data/runs/${{ inputs.platform }}", diagnostics) + self.assertIn(f" name: {name}", diagnostics) + self.assertIn(" if-no-files-found: error", diagnostics) + self.assertIn(" overwrite: false", diagnostics) + self.assertIn(" retention-days: 1", diagnostics) + self.assertLess( + action.index("- name: Upload benchmark diagnostics"), + action.index("- name: Upload benchmark results"), + ) + + artifacts: dict[str, tuple[str, int]] = {} + for attempt in (1, 2): + for platform in ( + "linux-kvm-virtual-machine", + "linux-mshv-virtual-machine", + "windows-whp-virtual-machine", + ): + artifact = ( + name.replace("${{ inputs.platform }}", platform) + .replace("${{ github.run_id }}", "35771477786") + .replace("${{ github.run_attempt }}", str(attempt)) + ) + self.assertNotIn("${{", artifact) + self.assertNotIn(artifact, artifacts) + artifacts[artifact] = (platform, attempt) + self.assertEqual(len(artifacts), 6) + + def test_benchmark_result_handoff_keeps_run_scoped_names(self): + actions = Path(__file__).parents[1] / ".github" / "actions" + action = (actions / "run-benchmark" / "action.yml").read_text(encoding="utf-8") + prepare = (actions / "prepare-performance-results" / "action.yml").read_text( + encoding="utf-8" + ) + results = _composite_action_step(action, "Upload benchmark results") + name = "benchmark-${{ inputs.platform }}-${{ github.run_id }}" + + self.assertIn(f" name: {name}", results) + self.assertIn(" path: data/runs/${{ inputs.platform }}", results) + self.assertIn(" overwrite: true", results) + self.assertNotIn("github.run_attempt", results) + self.assertNotIn("github.run_attempt", prepare) + for platform in ( + "linux-kvm-virtual-machine", + "linux-mshv-virtual-machine", + "windows-whp-virtual-machine", + ): + with self.subTest(platform=platform): + artifact = name.replace("${{ inputs.platform }}", platform) + self.assertIn(f" name: {artifact}", prepare) + + def test_ci_remeasures_only_unstable_lifecycle_results(self): + action = ( + BuildConstants.REPO_ROOT + / ".github" + / "actions" + / "run-benchmark" + / "action.yml" + ).read_text(encoding="utf-8") + + self.assertEqual(action.count("performance validate-openvmm"), 2) + for step, loop, unstable in ( + ( + "Run Linux acceptance test", + "for attempt in 1 2; do", + '"${validation_status}" -ne 75', + ), + ( + "Run Windows acceptance test", + "foreach ($Attempt in 1, 2)", + "$ValidationStatus -ne 75", + ), + ): + with self.subTest(step=step): + script = _composite_action_script(action, step) + self.assertEqual(script.count("performance validate-openvmm"), 1) + self.assertEqual(script.count(loop), 1) + self.assertEqual(script.count(unstable), 1) + self.assertEqual( + script.count("Lifecycle snapshot generation was unstable"), + 1, + ) + self.assertEqual(script.count("--attempt-log"), 1) + self.assertIn("acceptance-attempts.json", script) + # Publication names the attempt that supplied the lifecycle metrics. + for step, log in ( + ( + "Publish Linux performance results", + '--lifecycle-attempt-log "${run_dir}/acceptance-attempts.json"', + ), + ( + "Publish Windows performance results", + '--lifecycle-attempt-log "$RunDir/acceptance-attempts.json"', + ), + ): + with self.subTest(step=step): + script = _composite_action_script(action, step) + self.assertEqual(script.count(log), 1) + self.assertLess(script.index("--lifecycle-input"), script.index(log)) + + def test_windows_benchmarks_use_provisioned_data_volume_scratch(self): + action = ( + BuildConstants.REPO_ROOT + / ".github" + / "actions" + / "run-benchmark" + / "action.yml" + ).read_text(encoding="utf-8") + setup = ( + BuildConstants.REPO_ROOT / "scripts" / "setup" / "setup-windows-whp.ps1" + ).read_text(encoding="utf-8") + prepare = _composite_action_step(action, "Prepare Windows benchmark scratch") + cleanup = _composite_action_step(action, "Remove Windows benchmark scratch") + + self.assertIn('$BenchmarkScratchVariable = "NVX_BENCHMARK_SCRATCH"', setup) + self.assertIn( + '[Environment]::GetEnvironmentVariable("NVX_BENCHMARK_SCRATCH", "Machine")', + prepare, + ) + self.assertIn('"NVX_BENCHMARK_SCRATCH_DIR=$Scratch"', prepare) + self.assertIn("::warning::NVX_BENCHMARK_SCRATCH is not provisioned", prepare) + self.assertIn(" if: always() && runner.os == 'Windows'", cleanup) + self.assertLess( + action.index("- name: Remove Windows benchmark scratch"), + action.index("- name: Upload benchmark diagnostics"), + ) + for step in ( + "Run Windows acceptance test", + "Run Windows performance suite", + "Run Windows multi-vCPU shell restore", + "Run Windows device operation rates", + ): + with self.subTest(step=step): + script = _composite_action_script(action, step) + self.assertIn( + '$ScratchArgs = @("--scratch-dir", $env:NVX_BENCHMARK_SCRATCH_DIR)', + script, + ) + self.assertIn("@ScratchArgs", script) + self.assertLess( + action.index("- name: Prepare Windows benchmark scratch"), + action.index(f"- name: {step}"), + ) + self.assertIn( + " Configure-SccacheEnvironment\n Configure-BenchmarkScratch\n", + setup, + ) + self.assertIn( + "Assert-ServiceDirectoryAcl -Path $SccacheDirectory -Writable\n" + " Assert-BenchmarkScratch\n", + setup, + ) + + @unittest.skipUnless(os.name == "nt", "requires Windows PowerShell") + def test_windows_acceptance_forwards_benchmark_scratch(self): + action = ( + BuildConstants.REPO_ROOT + / ".github" + / "actions" + / "run-benchmark" + / "action.yml" + ).read_text(encoding="utf-8") + script = _composite_action_script(action, "Run Windows acceptance test") + for name, value in ( + ("backend", "whp"), + ("platform", "windows-whp-virtual-machine"), + ("warmups", "1"), + ("runs", "10"), + ("teardown-mode", "guest-exit"), + ): + script = script.replace("${{ inputs." + name + " }}", value) + stub = """ +function python { + $global:LASTEXITCODE = 0 + if ($args[1] -ne "benchmark") { + return + } + $output = $args[[Array]::IndexOf($args, "--output") + 1] + $index = [Array]::IndexOf($args, "--scratch-dir") + $scratch = "" + if ($index -ge 0) { + $scratch = $args[$index + 1] + } + New-Item -ItemType Directory -Path (Split-Path $output) -Force | Out-Null + Set-Content -LiteralPath $output -Encoding UTF8 -Value ( + ConvertTo-Json -Compress @{ scratch = $scratch } + ) +} +""" + for scratch in ("", r"F:\nvx-benchmark-scratch\job"): + with ( + self.subTest(scratch=scratch), + tempfile.TemporaryDirectory() as temporary, + ): + root = Path(temporary).resolve() + environment = os.environ.copy() + environment.pop("NVX_BENCHMARK_SCRATCH_DIR", None) + if scratch: + environment["NVX_BENCHMARK_SCRATCH_DIR"] = scratch + result = subprocess.run( + [ + "powershell.exe", + "-NoProfile", + "-NonInteractive", + "-Command", + stub + script, + ], + cwd=root, + env=environment, + capture_output=True, + text=True, + timeout=30, + check=False, + ) + + self.assertEqual(result.returncode, 0, result.stderr) + accepted = ( + root + / "data" + / "runs" + / "windows-whp-virtual-machine" + / "microvm-v2" + / "1vcpu" + / "acceptance.json" + ) + self.assertEqual( + json.loads(accepted.read_text(encoding="utf-8-sig")), + {"scratch": scratch}, + ) + + @unittest.skipUnless(os.name == "nt", "requires Windows PowerShell") + def test_windows_benchmark_scratch_steps_manage_per_job_directories(self): + action = ( + BuildConstants.REPO_ROOT + / ".github" + / "actions" + / "run-benchmark" + / "action.yml" + ).read_text(encoding="utf-8") + machine_lookup = ( + '[Environment]::GetEnvironmentVariable("NVX_BENCHMARK_SCRATCH", "Machine")' + ) + prepare = _composite_action_script( + action, "Prepare Windows benchmark scratch" + ).replace("${{ inputs.platform }}", "windows-whp-virtual-machine") + cleanup = _composite_action_script(action, "Remove Windows benchmark scratch") + self.assertIn(machine_lookup, prepare) + + def run_step(script: str, environment: dict[str, str]): + return subprocess.run( + ["powershell.exe", "-NoProfile", "-NonInteractive", "-Command", script], + env=environment, + capture_output=True, + text=True, + timeout=30, + check=False, + ) + + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary).resolve() / "scratch" + root.mkdir() + stale = root / "windows-whp-virtual-machine-1-1" + fresh = root / "windows-whp-virtual-machine-2-1" + for directory in (stale, fresh): + directory.mkdir() + (directory / "memory.bin").write_bytes(b"ram") + old = time.time() - 7 * 60 * 60 + os.utime(stale, (old, old)) + github_env = Path(temporary) / "github-env.txt" + github_env.write_text("", encoding="utf-8") + environment = os.environ.copy() + environment.update( + { + "GITHUB_ENV": str(github_env), + "GITHUB_RUN_ID": "35805840174", + "GITHUB_RUN_ATTEMPT": "2", + } + ) + + result = run_step( + prepare.replace(machine_lookup, f"'{root}'"), + environment, + ) + + self.assertEqual(result.returncode, 0, result.stderr) + scratch = root / "windows-whp-virtual-machine-35805840174-2" + self.assertTrue(scratch.is_dir()) + self.assertFalse(stale.exists()) + self.assertTrue(fresh.is_dir()) + self.assertEqual( + github_env.read_text(encoding="utf-8-sig").splitlines(), + [f"NVX_BENCHMARK_SCRATCH_DIR={scratch}"], + ) + + locked = scratch / "locked.bin" + with locked.open("wb"): + result = run_step( + cleanup, + {**environment, "NVX_BENCHMARK_SCRATCH_DIR": str(scratch)}, + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn( + "::warning::Could not remove benchmark scratch", result.stdout + ) + + result = run_step( + cleanup, + {**environment, "NVX_BENCHMARK_SCRATCH_DIR": str(scratch)}, + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertFalse(scratch.exists()) + + github_env.write_text("", encoding="utf-8") + result = run_step(prepare.replace(machine_lookup, "''"), environment) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn( + "::warning::NVX_BENCHMARK_SCRATCH is not provisioned", result.stdout + ) + self.assertEqual(github_env.read_text(encoding="utf-8"), "") + + @unittest.skipUnless(os.name == "nt", "requires Windows PowerShell") + def test_windows_setup_selects_largest_data_volume_for_benchmark_scratch(self): + setup = BuildConstants.REPO_ROOT / "scripts" / "setup" / "setup-windows-whp.ps1" + harness = f""" +Set-StrictMode -Version 2.0 +$ErrorActionPreference = "Stop" +$tokens = $null +$errors = $null +$ast = [System.Management.Automation.Language.Parser]::ParseFile( + '{setup}', + [ref]$tokens, + [ref]$errors +) +if ($errors.Count -ne 0) {{ + throw "setup script has parse errors" +}} +foreach ($name in "Test-SystemVolumePath", "Get-BenchmarkScratchDirectory") {{ + $definition = $ast.Find({{ + param($node) + $node -is [System.Management.Automation.Language.FunctionDefinitionAst] -and + $node.Name -eq $name + }}, $true) + . ([scriptblock]::Create($definition.Extent.Text)) +}} +$BenchmarkScratchVariable = "NVX_TEST_UNSET_{uuid.uuid4().hex}" +$BenchmarkScratchName = "nvx-benchmark-scratch" +$BenchmarkScratchDirectory = $null +$env:SystemDrive = "C:" +$Volumes = @( + [pscustomobject]@{{ DriveLetter = [char]"C"; DriveType = "Fixed"; FileSystem = "NTFS"; Size = 900GB }}, + [pscustomobject]@{{ DriveLetter = [char]"D"; DriveType = "Fixed"; FileSystem = "NTFS"; Size = 64GB }}, + [pscustomobject]@{{ DriveLetter = $null; DriveType = "Fixed"; FileSystem = "NTFS"; Size = 2TB }}, + [pscustomobject]@{{ DriveLetter = [char]"E"; DriveType = "CD-ROM"; FileSystem = ""; Size = 1TB }}, + [pscustomobject]@{{ DriveLetter = [char]"F"; DriveType = "Fixed"; FileSystem = "NTFS"; Size = 512GB }}, + [pscustomobject]@{{ DriveLetter = [char]"G"; DriveType = "Fixed"; FileSystem = "FAT32"; Size = 1TB }} +) +function Get-Volume {{ + $script:Volumes +}} +Write-Output (Get-BenchmarkScratchDirectory) +$Volumes = @($Volumes[0]) +Write-Output ("none=" + ($null -eq (Get-BenchmarkScratchDirectory))) +$BenchmarkScratchDirectory = "H:\\explicit\\scratch" +Write-Output (Get-BenchmarkScratchDirectory) +""" + + result = subprocess.run( + ["powershell.exe", "-NoProfile", "-NonInteractive", "-Command", harness], + capture_output=True, + text=True, + timeout=30, + check=False, + ) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual( + result.stdout.splitlines(), + [ + r"F:\nvx-benchmark-scratch", + "none=True", + r"H:\explicit\scratch", + ], + ) + + def test_cli_validation_runs_every_script_test_module(self): + # Each scripts/test_*.py module must run in both CLI validation jobs; + # test_control_session.py once ran in neither (#229). + action = ( + BuildConstants.REPO_ROOT + / ".github" + / "actions" + / "validate-nvx" + / "action.yml" + ).read_text(encoding="utf-8") + modules = { + path.name + for path in (BuildConstants.REPO_ROOT / "scripts").glob("test_*.py") + } + self.assertIn("test_control_session.py", modules) + for step in ("Validate NVX CLI on Linux", "Validate NVX CLI on Windows"): + with self.subTest(step=step): + script = _composite_action_script(action, step) + invoked = re.findall(r"(?:^|\s)scripts[/\\](test_\w+\.py)", script) + self.assertEqual(sorted(invoked), sorted(modules)) + + def test_ci_runs_the_public_sandbox_acceptance_on_every_backend(self): + workflows = BuildConstants.REPO_ROOT / ".github" / "workflows" + workflow = (workflows / "ci.yml").read_text(encoding="utf-8") + microvm_workflow = (workflows / "run-nvx-microvm-tests.yml").read_text( + encoding="utf-8" + ) + for backend in ("kvm", "mshv", "whp"): + with self.subTest(backend=backend): + job = _workflow_job(workflow, f"nvx-microvm-tests-{backend}") + self.assertIn( + "uses: ./.github/workflows/run-nvx-microvm-tests.yml", job + ) + self.assertIn(f"backend: {backend}", job) + self.assertNotIn("debug-kernel: true", job) + for system in ("Linux", "Windows"): + with self.subTest(system=system): + acceptance = _workflow_step( + microvm_workflow, + "test", + f"Run public managed sandbox acceptance on {system}", + ) + self.assertIn("!inputs.debug-kernel", acceptance) + self.assertIn("--scenario managed-exec-config", acceptance) + self.assertIn("--scenario sandbox-lifecycle", acceptance) + self.assertIn( + '--output-dir "build/test-results/public-exec-${{ inputs.backend }}"', + acceptance, + ) + smoke = _workflow_step( + microvm_workflow, + "test", + f"Run Ubuntu sandbox layer smoke test on {system}", + ) + for option in ( + "--entrypoint /sbin/nvx-sandbox-smoke", + "--arg limits", + "--arg 33554432", + "--arg 8", + "--memory-max 33554432", + "--pids-max 8", + ): + self.assertIn(option, smoke) + upload = _workflow_step( + microvm_workflow, "test", "Upload NVX microVM failure logs" + ) + self.assertIn("build/test-results/public-exec-${{ inputs.backend }}", upload) + + @unittest.skipUnless(os.name == "nt", "requires Windows PowerShell") + def test_windows_cli_validation_stops_at_each_failed_command(self): + action = ( + BuildConstants.REPO_ROOT + / ".github" + / "actions" + / "validate-nvx" + / "action.yml" + ).read_text(encoding="utf-8") + script = _composite_action_script(action, "Validate NVX CLI on Windows") + command_count = sum(line.startswith("python ") for line in script.splitlines()) + self.assertGreater(command_count, 0) + stub = """ +$CommandCount = 0 +function python { + $script:CommandCount += 1 + Write-Output "python-$script:CommandCount" + $global:LASTEXITCODE = 0 + if ($script:CommandCount -eq $script:FailureAt) { + $global:LASTEXITCODE = 37 + } +} +""" + for failure_at in range(command_count + 1): + with self.subTest(failure_at=failure_at): + result = subprocess.run( + [ + "powershell.exe", + "-NoProfile", + "-NonInteractive", + "-Command", + f"$FailureAt = {failure_at}\n{stub}\n{script}", + ], + capture_output=True, + text=True, + timeout=30, + check=False, + ) + self.assertEqual( + result.returncode, 37 if failure_at else 0, result.stderr + ) + self.assertEqual( + result.stdout.splitlines(), + [ + f"python-{index}" + for index in range(1, (failure_at or command_count) + 1) + ], + ) + + @unittest.skipUnless(os.name == "nt", "requires Windows PowerShell") + def test_windows_acceptance_preserves_attempts_and_publishes_only_valid_data(self): + action = ( + BuildConstants.REPO_ROOT + / ".github" + / "actions" + / "run-benchmark" + / "action.yml" + ).read_text(encoding="utf-8") + script = _composite_action_script(action, "Run Windows acceptance test") + for name, value in ( + ("backend", "whp"), + ("platform", "windows-whp-virtual-machine"), + ("warmups", "1"), + ("runs", "10"), + ("teardown-mode", "guest-exit"), + ): + script = script.replace("${{ inputs." + name + " }}", value) + stub = """ +$BenchmarkCount = 0 +$ValidationCount = 0 +function python { + if ($args[1] -eq "benchmark") { + $script:BenchmarkCount += 1 + $output = $args[[Array]::IndexOf($args, "--output") + 1] + New-Item -ItemType Directory -Path (Split-Path $output) -Force | Out-Null + Set-Content -LiteralPath $output -Encoding UTF8 -Value ( + '{"attempt":' + $script:BenchmarkCount + '}' + ) + $global:LASTEXITCODE = $script:BenchmarkStatus + } + elseif ($args[1] -eq "performance") { + $log = $args[[Array]::IndexOf($args, "--attempt-log") + 1] + Add-Content -LiteralPath "attempt-logs.txt" -Value $log + $global:LASTEXITCODE = $script:ValidationStatuses[$script:ValidationCount] + $script:ValidationCount += 1 + } + else { + throw "unexpected command: $args" + } +} +""" + for name, statuses, benchmark_status in ( + ("first-success", (0,), 0), + ("remeasured-success", (75, 0), 0), + ("both-unstable", (75, 75), 0), + ("invalid-data", (1,), 0), + ("benchmark-failure", (0,), 42), + ): + with self.subTest(name=name), tempfile.TemporaryDirectory() as temporary: + root = Path(temporary).resolve() + output = ( + root + / "data" + / "runs" + / "windows-whp-virtual-machine" + / "microvm-v2" + / "1vcpu" + ) + output.mkdir(parents=True) + accepted = output / "acceptance.json" + accepted.write_text('{"attempt":"stale"}', encoding="utf-8") + # A rerun in the same directory must not keep a previous + # run's attempt log or later attempts. + attempt_log = output / "acceptance-attempts.json" + attempt_log.write_text('{"attempts":"stale"}', encoding="utf-8") + (output / "acceptance-attempt-2.json").write_text( + '{"attempt":"stale"}', encoding="utf-8" + ) + setup = ( + "$ValidationStatuses = @(" + + ", ".join(str(status) for status in statuses) + + f")\n$BenchmarkStatus = {benchmark_status}\n" + ) + result = subprocess.run( + [ + "powershell.exe", + "-NoProfile", + "-NonInteractive", + "-Command", + setup + stub + script, + ], + cwd=root, + capture_output=True, + text=True, + timeout=30, + check=False, + ) + expected_status = benchmark_status or statuses[-1] + self.assertEqual(result.returncode, expected_status, result.stderr) + self.assertFalse(attempt_log.exists()) + validations = root / "attempt-logs.txt" + self.assertEqual( + ( + validations.read_text(encoding="utf-8").splitlines() + if validations.exists() + else [] + ), + [ + "data/runs/windows-whp-virtual-machine/microvm-v2/1vcpu/" + "acceptance-attempts.json" + ] + * (0 if benchmark_status else len(statuses)), + ) + self.assertEqual( + sorted(path.name for path in output.glob("acceptance-attempt-*")), + [ + f"acceptance-attempt-{index}.json" + for index in range(1, len(statuses) + 1) + ], + ) + for index in range(1, len(statuses) + 1): + attempt = output / f"acceptance-attempt-{index}.json" + self.assertEqual( + json.loads(attempt.read_text(encoding="utf-8-sig")), + {"attempt": index}, + ) + self.assertEqual(accepted.exists(), expected_status == 0) + if expected_status == 0: + self.assertEqual( + json.loads(accepted.read_text(encoding="utf-8-sig")), + {"attempt": len(statuses)}, + ) + + @unittest.skipUnless(os.name == "posix", "requires bash") + def test_linux_acceptance_preserves_attempts_and_publishes_only_valid_data(self): + action = ( + BuildConstants.REPO_ROOT + / ".github" + / "actions" + / "run-benchmark" + / "action.yml" + ).read_text(encoding="utf-8") + script = _composite_action_script(action, "Run Linux acceptance test") + platform = "linux-kvm-virtual-machine" + for name, value in ( + ("backend", "kvm"), + ("platform", platform), + ("warmups", "1"), + ("runs", "10"), + ("teardown-mode", "guest-exit"), + ): + script = script.replace("${{ inputs." + name + " }}", value) + stub = """ +benchmark_count=0 +validation_count=0 +python3() { + local argument output="" input="" platform="" attempt_log="" previous="" + for argument in "$@"; do + case "${previous}" in + --output) output="${argument}" ;; + --input) input="${argument}" ;; + --platform) platform="${argument}" ;; + --attempt-log) attempt_log="${argument}" ;; + esac + previous="${argument}" + done + if [[ "${2-}" == benchmark ]]; then + benchmark_count=$((benchmark_count + 1)) + echo "benchmark ${platform} ${output}" + mkdir -p "$(dirname "${output}")" + printf '{"attempt":%d}' "${benchmark_count}" >"${output}" + return "${BENCHMARK_STATUS}" + fi + if [[ "${2-} ${3-}" == "performance validate-openvmm" ]]; then + validation_count=$((validation_count + 1)) + echo "validate ${platform} ${input} ${attempt_log}" + return "${VALIDATION_STATUSES[validation_count - 1]}" + fi + echo "unexpected command: $*" >&2 + return 1 +} +""" + environment = os.environ.copy() + environment.update( + { + "NVX_BENCHMARK_CPUS": "0-7", + "NVX_BENCHMARK_HOST_CPU_RESERVE": "0", + "NVX_BENCHMARK_MARKER_TIMEOUT": "40", + } + ) + run_dir = f"data/runs/{platform}/microvm-v2/1vcpu" + for name, statuses, benchmark_status in ( + ("first-success", (0,), 0), + ("remeasured-success", (75, 0), 0), + ("both-unstable", (75, 75), 0), + ("invalid-data", (2,), 0), + ("unstable-then-invalid", (75, 2), 0), + ("benchmark-failure", (0,), 42), + ): + with self.subTest(name=name), tempfile.TemporaryDirectory() as temporary: + root = Path(temporary).resolve() + output = root / run_dir + output.mkdir(parents=True) + accepted = output / "acceptance.json" + accepted.write_text('{"attempt":"stale"}', encoding="utf-8") + # A rerun in the same directory must not keep a previous + # run's attempt log or later attempts. + attempt_log = output / "acceptance-attempts.json" + attempt_log.write_text('{"attempts":"stale"}', encoding="utf-8") + (output / "acceptance-attempt-2.json").write_text( + '{"attempt":"stale"}', encoding="utf-8" + ) + setup = ( + "VALIDATION_STATUSES=(" + + " ".join(str(status) for status in statuses) + + f")\nBENCHMARK_STATUS={benchmark_status}\n" + ) + result = subprocess.run( + [ + "bash", + "--noprofile", + "--norc", + "-eo", + "pipefail", + "-c", + setup + stub + script, + ], + cwd=root, + env=environment, + capture_output=True, + text=True, + timeout=30, + check=False, + ) + expected_status = benchmark_status or statuses[-1] + self.assertEqual(result.returncode, expected_status, result.stderr) + self.assertFalse(attempt_log.exists()) + expected_stdout: list[str] = [] + for index in range(1, len(statuses) + 1): + attempt = f"{run_dir}/acceptance-attempt-{index}.json" + if index > 1: + expected_stdout.append( + "::warning::Lifecycle snapshot generation was unstable " + f"on attempt {index - 1}; remeasuring once" + ) + expected_stdout.append(f"benchmark {platform} {attempt}") + if not benchmark_status: + expected_stdout.append( + f"validate {platform} {attempt} " + f"{run_dir}/acceptance-attempts.json" + ) + self.assertEqual(result.stdout.splitlines(), expected_stdout) + self.assertEqual( + sorted(path.name for path in output.glob("acceptance-attempt-*")), + [ + f"acceptance-attempt-{index}.json" + for index in range(1, len(statuses) + 1) + ], + ) + for index in range(1, len(statuses) + 1): + attempt = output / f"acceptance-attempt-{index}.json" + self.assertEqual( + json.loads(attempt.read_text(encoding="utf-8")), + {"attempt": index}, + ) + self.assertEqual(accepted.exists(), expected_status == 0) + if expected_status == 0: + self.assertEqual( + json.loads(accepted.read_text(encoding="utf-8")), + {"attempt": len(statuses)}, + ) + + +class BuildConstantsTests(unittest.TestCase): + def test_constants_are_grouped_class_namespaces(self): + for namespace in ( + BuildConstants, + KernelBuildConstants, + OpenVMMBuildConstants, + AlpineBuildConstants, + UbuntuBuildConstants, + InitramfsBuildConstants, + DockerBuildConstants, + ZstdBuildConstants, + ReleaseBuildConstants, + ): + with self.subTest(namespace=namespace.__name__): + self.assertEqual(namespace.__module__, "nvx_tools.build_constants") + self.assertTrue( + all( + name.isupper() + for name in vars(namespace) + if not name.startswith("__") + ) + ) + + def test_build_pins_and_artifact_names_have_one_literal_owner(self): + centralized = { + KernelBuildConstants.VERSION, + KernelBuildConstants.URL, + KernelBuildConstants.SHA256, + KernelBuildConstants.BINARY_NAME, + KernelBuildConstants.CONFIG_NAME, + KernelBuildConstants.PROVENANCE_NAME, + KernelBuildConstants.DEBUG_BINARY_NAME, + KernelBuildConstants.DEBUG_CONFIG_NAME, + KernelBuildConstants.DEBUG_PROVENANCE_NAME, + OpenVMMBuildConstants.GNU_RUST_TARGET, + OpenVMMBuildConstants.MUSL_RUST_TARGET, + OpenVMMBuildConstants.WINDOWS_RUST_TARGET, + OpenVMMBuildConstants.PROVENANCE_NAME, + OpenVMMBuildConstants.CONTROL_CONTRACT_REVISION, + AlpineBuildConstants.VERSION, + AlpineBuildConstants.BRANCH, + AlpineBuildConstants.MINIROOTFS_SHA256, + AlpineBuildConstants.INITRAMFS_NAME, + AlpineBuildConstants.PACKAGE_MANIFEST_NAME, + UbuntuBuildConstants.VERSION, + UbuntuBuildConstants.BASE_URL, + UbuntuBuildConstants.BASE_SHA256, + UbuntuBuildConstants.INITRAMFS_NAME, + UbuntuBuildConstants.PACKAGE_MANIFEST_NAME, + UbuntuBuildConstants.DISTRO_NAME, + UbuntuBuildConstants.DISTRO_MANIFEST_NAME, + InitramfsBuildConstants.PROVENANCE_NAME, + ZstdBuildConstants.VERSION, + ZstdBuildConstants.SHA256, + } + duplicates: list[tuple[str, int, str]] = [] + for path in (BuildConstants.REPO_ROOT / "scripts").rglob("*.py"): + if path.name == "build_constants.py" or path.name.startswith("test_"): + continue + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + duplicates.extend( + (path.name, node.lineno, node.value) + for node in ast.walk(tree) + if isinstance(node, ast.Constant) + and isinstance(node.value, str) + and node.value in centralized + ) + self.assertEqual(duplicates, []) + + def test_old_module_level_constants_are_not_reexported(self): + names = ( + "REPO_ROOT", + "BUILD_DIR", + "SOURCE_DIR", + "OPENVMM_DIR", + "DEFAULT_KERNEL_VERSION", + "DEFAULT_KERNEL_URL", + "DEFAULT_KERNEL_SHA256", + "DEFAULT_ALPINE_VERSION", + "DEFAULT_ALPINE_BRANCH", + "DEFAULT_ALPINE_MINIROOTFS_SHA256", + "DEFAULT_UBUNTU_VERSION", + "DEFAULT_UBUNTU_CODENAME", + "DEFAULT_UBUNTU_ARCHITECTURE", + "DEFAULT_UBUNTU_BASE_URL", + "DEFAULT_UBUNTU_BASE_SHA256", + "UBUNTU_PACKAGE_LOCK", + "UBUNTU_EROFS_FORMAT", + "MICROVM_ABI_VERSION", + "CONTROL_SESSION_PROTOCOL_VERSION", + "CONTROL_CONTRACT_REVISION", + "OPENVMM_PROVENANCE_NAME", + "KERNEL_PROVENANCE_NAME", + "INITRAMFS_PROVENANCE_NAME", + "REQUIRED_VIRTIO_CONSOLE_CONFIG", + "REQUIRED_SHARED_STATUS_KERNEL_CONFIG", + "REQUIRED_SANDBOX_KERNEL_CONFIG", + "ZSTD_VERSION", + "ZSTD_ARCHIVE", + "ZSTD_URL", + "ZSTD_SHA256", + "OPENVMM_RUST_TOOLCHAIN", + "OPENVMM_GUEST_RUST_TARGET", + "OPENVMM_UEFI_RUST_TARGET", + "OPENVMM_LINUX_MUSL_RUST_TARGET", + "OPENVMM_RUST_TARGETS", + "APORTS_URL", + "REPOSITORIES", + "UBUNTU_ARCHIVE_KEYRING_URL", + "UBUNTU_ARCHIVE_KEYRING_SHA256", + "UBUNTU_SNAPSHOT_ARCHIVE_URL", + "UBUNTU_SOURCE_INDEXES", + "_UBUNTU_POCKET_SUITES", + "_UBUNTU_COMPONENTS", + "_LAUNCHPAD_ARCHIVE_API", + "_LAUNCHPAD_SERIES_API", + "_GENERATED_OUTPUT_ENTRIES", + "_KNOWN_PACKAGE_CONTROL_ACTIONS", + "PROJECT_SOURCE_PATHS", + "GUEST_RELEASE_NAMES", + "_ZIP_TIMESTAMP", + ) + for module in ( + build, + build_config, + common, + ubuntu, + ci, + collect_alpine_sources, + collect_ubuntu_sources, + release, + benchmark, + archive, + ): + with self.subTest(module=module.__name__): + self.assertFalse(set(names).intersection(vars(module))) + + def test_guest_descriptors_and_source_manifest_share_build_pins(self): + manifest = json.loads( + (BuildConstants.REPO_ROOT / "SOURCE-MANIFEST.json").read_text( + encoding="utf-8" + ) + ) + self.assertEqual(KernelBuildConstants.VERSION, manifest["linux"]["version"]) + self.assertEqual(KernelBuildConstants.URL, manifest["linux"]["upstream_url"]) + self.assertEqual( + KernelBuildConstants.SHA256, + manifest["linux"]["upstream_archive_sha256"], + ) + self.assertEqual( + AlpineBuildConstants.MINIROOTFS_URL, manifest["alpine"]["minirootfs_url"] + ) + self.assertEqual( + AlpineBuildConstants.MINIROOTFS_SHA256, + manifest["alpine"]["minirootfs_sha256"], + ) + for descriptor in (guests.ALPINE_GUEST, guests.UBUNTU_GUEST): + with self.subTest(guest=descriptor.name): + pinned = manifest[descriptor.name] + self.assertEqual(descriptor.release, pinned["version"]) + self.assertEqual(descriptor.architecture, pinned["architecture"]) + self.assertEqual( + f"build/{descriptor.package_manifest_name}", + pinned["package_manifests"][0], + ) + self.assertEqual( + OpenVMMBuildConstants.MICROVM_ABI_VERSION, + manifest["openvmm"]["microvm_abi_version"], + ) + self.assertEqual( + OpenVMMBuildConstants.CONTROL_SESSION_PROTOCOL_VERSION, + manifest["openvmm"]["control_session_protocol_version"], + ) + self.assertEqual( + OpenVMMBuildConstants.CONTROL_CONTRACT_REVISION, + manifest["openvmm"]["control_contract_revision"], + ) + + def test_cache_defaults_are_resolved_for_each_configuration(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + for configured in ("", str(root / "first"), str(root / "second")): + with ( + self.subTest(configured=configured), + patch.dict(os.environ, {"NVX_CACHE_DIR": configured}), + ): + expected = ( + Path(configured) + if configured + else BuildConstants.REPO_ROOT / ".cache" + ).resolve() + self.assertEqual( + build_config.KernelBuildConfig().cache_directory, expected + ) + self.assertEqual(common.cache_root(), expected) + self.assertEqual( + build_config.KernelBuildConfig( + cache_directory=root / "explicit" + ).cache_directory, + root / "explicit", + ) + + def test_openvmm_executable_default_is_resolved_at_configuration_time(self): + for host, binary in (("nt", "openvmm.exe"), ("posix", "openvmm")): + expected = OpenVMMBuildConstants.DIRECTORY / "target" / "release" / binary + with self.subTest(host=host), patch.object(common.os, "name", host): + self.assertEqual(common.openvmm_binary_path(), expected) + self.assertEqual(build_config.OpenVmmBuildConfig().output, expected) + + def test_initramfs_provenance_tracks_constants_source(self): + relative = "scripts/nvx_tools/build_constants.py" + source = BuildConstants.REPO_ROOT / relative + original = build.initramfs_provenance_inputs() + records = cast(list[dict[str, str]], original["source_files"]) + self.assertEqual( + [record for record in records if record["path"] == relative], + [{"path": relative, "sha256": common.sha256_file(source)}], + ) + + def changed_sha256(path: Path) -> str: + return "f" * 64 if path == source else common.sha256_file(path) + + with patch.object(build, "sha256_file", side_effect=changed_sha256): + changed = build.initramfs_provenance_inputs() + self.assertNotEqual(original, changed) + self.assertEqual(original["alpine"], changed["alpine"]) + + def test_constants_are_in_docker_and_release_source_inputs(self): + dockerfile = (BuildConstants.REPO_ROOT / "docker" / "Dockerfile").read_text( + encoding="utf-8" + ) + self.assertIn("COPY scripts/ /repo/scripts/", dockerfile) + self.assertIn("scripts", ReleaseBuildConstants.PROJECT_SOURCE_PATHS) + self.assertTrue( + ( + BuildConstants.REPO_ROOT + / "scripts" + / "nvx_tools" + / "build_constants.py" + ).is_file() + ) + + def test_constants_consumers_import_in_either_order(self): + modules = ( + "nvx_tools.build_constants", + "nvx_tools.guests", + "nvx_tools.common", + "nvx_tools.build_config", + "nvx_tools.build", + "nvx_tools.release", + "nvx", + ) + for ordered in (modules, tuple(reversed(modules))): + with self.subTest(first=ordered[0]): + result = subprocess.run( + [ + sys.executable, + "-c", + "import importlib, sys; " + "[importlib.import_module(name) for name in sys.argv[1:]]", + *ordered, + ], + cwd=BuildConstants.REPO_ROOT / "scripts", + capture_output=True, + text=True, + timeout=30, + check=False, + ) + self.assertEqual(result.returncode, 0, result.stderr) + + +class UbuntuSourceCollectionTests(unittest.TestCase): + def test_malformed_package_manifest_is_reported_as_script_error(self): + with tempfile.TemporaryDirectory() as temporary: + manifest = Path(temporary) / "packages.json" + manifest.write_text("{", encoding="utf-8") + + with self.assertRaises(collect_ubuntu_sources.ScriptError) as context: + collect_ubuntu_sources._source_requirements([manifest]) + + self.assertIn("cannot read Ubuntu package manifest", str(context.exception)) + + +class AlpineSourceCollectionTests(unittest.TestCase): + INSTALL_SCRIPT = b"#!/bin/sh\nexit 0\n" + + @staticmethod + def _aports_commit(cache: Path, files: dict[str, tuple[str, bytes]]) -> str: + environment = { + key: value + for key, value in os.environ.items() + if not key.startswith("GIT_CONFIG_") + } + environment.update( + GIT_AUTHOR_NAME="NVX Tests", + GIT_AUTHOR_EMAIL="nvx-tests@example.com", + GIT_COMMITTER_NAME="NVX Tests", + GIT_COMMITTER_EMAIL="nvx-tests@example.com", + ) + + def git(*args: str, data: bytes | None = None) -> str: + result = subprocess.run( + ["git", "-C", str(cache), *args], + input=data, + env=environment, + capture_output=True, + check=True, + ) + return result.stdout.decode("ascii").strip() + + cache.mkdir(parents=True, exist_ok=True) + git("init", "-q") + git("config", "core.autocrlf", "false") + for path, (mode, contents) in files.items(): + blob = git("hash-object", "-w", "--stdin", data=contents) + git("update-index", "--add", "--cacheinfo", f"{mode},{blob},{path}") + return git("commit-tree", git("write-tree"), "-m", "aports fixture") + + def _symlinked_recipe_commit(self, cache: Path) -> str: + return self._aports_commit( + cache, + { + "main/foo/APKBUILD": ("100644", b"pkgname=foo\n"), + "main/foo/foo.post-install": ("100755", self.INSTALL_SCRIPT), + "main/foo/foo.post-upgrade": ("120000", b"foo.post-install"), + }, + ) + + @staticmethod + def _package_manifest(root: Path, commit: str) -> Path: + manifest = root / "packages.json" + manifest.write_text( + json.dumps( + { + "guest": AlpineBuildConstants.GUEST_NAME, + "alpine_branch": AlpineBuildConstants.BRANCH, + "architecture": AlpineBuildConstants.ARCHITECTURE, + "packages": [ + { + "name": "foo", + "version": "1.0-r0", + "origin": "foo", + "license": "MIT", + "aports_commit": commit, + } + ], + } + ), + encoding="utf-8", + ) + return manifest + + @staticmethod + def _member( + name: str, + kind: bytes = tarfile.REGTYPE, + data: bytes = b"", + *, + linkname: str = "", + mode: int = 0o664, + ) -> tuple[tarfile.TarInfo, bytes]: + member = tarfile.TarInfo(name) + member.type = kind + member.linkname = linkname + member.mode = mode + member.size = len(data) if kind == tarfile.REGTYPE else 0 + return member, data + + @staticmethod + def _archive(*members: tuple[tarfile.TarInfo, bytes]) -> bytes: + buffer = io.BytesIO() + with tarfile.open(fileobj=buffer, mode="w") as archive: + for member, data in members: + archive.addfile(member, io.BytesIO(data) if member.isfile() else None) + return buffer.getvalue() + + def test_malformed_package_manifest_is_reported_as_source_error(self): + with tempfile.TemporaryDirectory() as temporary: + for contents in ("{", "[]"): + manifest = Path(temporary) / "packages.json" + manifest.write_text(contents, encoding="utf-8") + + with ( + self.subTest(contents=contents), + self.assertRaises(collect_alpine_sources.SourceError) as context, + ): + collect_alpine_sources._load_packages([manifest]) + + self.assertIn("package manifest", str(context.exception)) + + def test_recipe_collection_preserves_blob_line_endings(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + cache = root / "aports" + apkbuild = b"pkgname=foo\npkgver=1.0\npkgrel=0\n" + commit = self._aports_commit( + cache, + {"main/foo/APKBUILD": ("100644", apkbuild)}, + ) + subprocess.run( + ["git", "-C", str(cache), "config", "core.autocrlf", "true"], + check=True, + ) + output = root / "alpine" + + with ( + patch.object(collect_alpine_sources, "_prepare_aports"), + contextlib.redirect_stdout(io.StringIO()), + ): + collect_alpine_sources.collect_alpine_sources( + [self._package_manifest(root, commit)], + output, + cache, + skip_upstream=True, + ) + + collected = output / "recipes" / commit / "main/foo/APKBUILD" + self.assertEqual(collected.read_bytes(), apkbuild) + + def test_recipe_symlink_is_collected_as_verified_regular_file(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + cache = root / "aports" + commit = self._symlinked_recipe_commit(cache) + output = root / "alpine" + + with ( + patch.object(collect_alpine_sources, "_prepare_aports"), + contextlib.redirect_stdout(io.StringIO()), + ): + collect_alpine_sources.collect_alpine_sources( + [self._package_manifest(root, commit)], + output, + cache, + skip_upstream=True, + ) + + upgrade = output / "recipes" / commit / "main/foo/foo.post-upgrade" + self.assertFalse(upgrade.is_symlink()) + self.assertEqual(upgrade.read_bytes(), self.INSTALL_SCRIPT) + source_manifest = json.loads( + (output / "manifest.json").read_text(encoding="utf-8") + ) + self.assertEqual( + source_manifest["format"], AlpineBuildConstants.SOURCE_MANIFEST_FORMAT + ) + self.assertEqual( + source_manifest["executables"], + [ + f"recipes/{commit}/main/foo/foo.post-install", + f"recipes/{commit}/main/foo/foo.post-upgrade", + ], + ) + self.assertIn( + f"recipes/{commit}/main/foo/foo.post-upgrade", + dict(common.verify_sha256_sums(output).files), + ) + if os.name == "posix": + self.assertEqual(upgrade.stat().st_mode & 0o777, 0o755) + + def test_collection_replaces_stale_output_and_passes_release_validation(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + cache = root / "aports" + commit = self._symlinked_recipe_commit(cache) + manifest = self._package_manifest(root, commit) + source_dir = root / "sources" + output = source_dir / "alpine" + stale_files = ( + output / "recipes" / ("0" * 40) / "main/stale/APKBUILD", + output / "upstream" / ("0" * 40) / "main/stale/stale.tar.gz", + ) + for stale in stale_files: + stale.parent.mkdir(parents=True) + stale.write_bytes(b"stale") + (output / "manifest.json").write_text("{}", encoding="utf-8") + (output / "SHA256SUMS").write_text("", encoding="ascii") + + def fetch_upstream(bundle: Path, _alpine_version: str) -> None: + upstream = bundle / "upstream" / commit / "main" / "foo" + upstream.mkdir(parents=True) + (upstream / "foo-1.0.tar.gz").write_bytes(b"upstream") + + with ( + patch.object(collect_alpine_sources, "_prepare_aports"), + patch.object( + collect_alpine_sources, + "_fetch_upstream_sources", + side_effect=fetch_upstream, + ), + patch.object(BuildConstants, "SOURCE_DIR", source_dir), + contextlib.redirect_stdout(io.StringIO()), + ): + collect_alpine_sources.collect_alpine_sources([manifest], output, cache) + executables = release._validate_alpine_sources([manifest]) + + self.assertEqual( + executables, + { + f"recipes/{commit}/main/foo/foo.post-install", + f"recipes/{commit}/main/foo/foo.post-upgrade", + }, + ) + for stale in stale_files: + self.assertFalse(stale.parent.parent.exists(), stale) + self.assertEqual( + sorted(path.name for path in output.iterdir()), + ["SHA256SUMS", "manifest.json", "recipes", "upstream"], + ) + + def test_recipe_extraction_materializes_symlinks_inside_recipe(self): + member = self._member + archive = self._archive( + member("main/", tarfile.DIRTYPE, mode=0o775), + member("main/foo/", tarfile.DIRTYPE, mode=0o775), + member("main/foo/patches/", tarfile.DIRTYPE, mode=0o775), + member("main/foo/APKBUILD", data=b"pkgname=foo\n"), + member("main/foo/foo.post-install", data=self.INSTALL_SCRIPT, mode=0o775), + member( + "main/foo/foo.post-upgrade", + tarfile.SYMTYPE, + linkname="foo.post-install", + ), + member( + "main/foo/foo.pre-upgrade", + tarfile.SYMTYPE, + linkname="./foo.post-upgrade", + ), + member("main/foo/patches/fix.patch", data=b"patch\n"), + member( + "main/foo/fix.patch", + tarfile.SYMTYPE, + linkname="patches/../patches/fix.patch", + ), + member( + "main/foo/patches/APKBUILD", tarfile.SYMTYPE, linkname="../APKBUILD" + ), + ) + with tempfile.TemporaryDirectory() as temporary: + destination = Path(temporary) + + executables = collect_alpine_sources._safe_extract( + archive, destination, "main/foo" + ) + + self.assertEqual( + executables, + [ + "main/foo/foo.post-install", + "main/foo/foo.post-upgrade", + "main/foo/foo.pre-upgrade", + ], + ) + recipe = destination / "main" / "foo" + expected = { + "APKBUILD": (b"pkgname=foo\n", 0o644), + "foo.post-install": (self.INSTALL_SCRIPT, 0o755), + "foo.post-upgrade": (self.INSTALL_SCRIPT, 0o755), + "foo.pre-upgrade": (self.INSTALL_SCRIPT, 0o755), + "fix.patch": (b"patch\n", 0o644), + "patches/fix.patch": (b"patch\n", 0o644), + "patches/APKBUILD": (b"pkgname=foo\n", 0o644), + } + for relative, (contents, mode) in expected.items(): + with self.subTest(path=relative): + path = recipe / relative + self.assertFalse(path.is_symlink()) + self.assertEqual(path.read_bytes(), contents) + if os.name == "posix": + self.assertEqual(path.stat().st_mode & 0o777, mode) + + def test_recipe_extraction_accepts_implicit_directories(self): + member = self._member + archive = self._archive( + member("main/foo/APKBUILD", data=b"pkgname=foo\n"), + member("main/foo/scripts/install", data=self.INSTALL_SCRIPT, mode=0o775), + member( + "main/foo/foo.post-install", + tarfile.SYMTYPE, + linkname="scripts/install", + ), + member( + "main/foo/scripts/APKBUILD", tarfile.SYMTYPE, linkname="../APKBUILD" + ), + ) + with tempfile.TemporaryDirectory() as temporary: + destination = Path(temporary) + + executables = collect_alpine_sources._safe_extract( + archive, destination, "main/foo" + ) + + self.assertEqual( + executables, + ["main/foo/foo.post-install", "main/foo/scripts/install"], + ) + recipe = destination / "main" / "foo" + for relative, contents in ( + ("foo.post-install", self.INSTALL_SCRIPT), + ("scripts/APKBUILD", b"pkgname=foo\n"), + ): + with self.subTest(path=relative): + path = recipe / relative + self.assertFalse(path.is_symlink()) + self.assertEqual(path.read_bytes(), contents) + if os.name == "posix": + for directory in (destination / "main", recipe, recipe / "scripts"): + with self.subTest(directory=directory.name): + self.assertEqual(directory.stat().st_mode & 0o777, 0o755) + + def test_recipe_extraction_rejects_unsafe_members(self): + member = self._member + recipe = ( + member("main/", tarfile.DIRTYPE), + member("main/foo/", tarfile.DIRTYPE), + member("main/foo/sub/", tarfile.DIRTYPE), + member("main/foo/APKBUILD", data=b"pkgname=foo\n"), + member("main/foo/sub/file", data=b"file\n"), + ) + cases: dict[str, tuple[tuple[tarfile.TarInfo, bytes], ...]] = { + "absolute path": (member("/main/foo/file"),), + "parent path": (member("main/foo/../../file"),), + "outside recipe": (member("main/bar/file"),), + "duplicate path": (member("main/foo/APKBUILD"),), + "member below a file": (member("main/foo/APKBUILD/file"),), + "member below a symlink": ( + member("main/foo/alias", tarfile.SYMTYPE, linkname="sub"), + member("main/foo/alias/file"), + ), + "hard link": ( + member("main/foo/link", tarfile.LNKTYPE, linkname="main/foo/APKBUILD"), + ), + "fifo": (member("main/foo/pipe", tarfile.FIFOTYPE),), + "character device": (member("main/foo/device", tarfile.CHRTYPE),), + "absolute symlink": ( + member("main/foo/link", tarfile.SYMTYPE, linkname="/etc/passwd"), + ), + "escaping symlink": ( + member("main/foo/link", tarfile.SYMTYPE, linkname="../../../passwd"), + ), + "sibling recipe symlink": ( + member("main/foo/link", tarfile.SYMTYPE, linkname="../bar/APKBUILD"), + ), + "dangling symlink": ( + member("main/foo/link", tarfile.SYMTYPE, linkname="missing"), + ), + "symlink through a missing directory": ( + member( + "main/foo/link", + tarfile.SYMTYPE, + linkname="missing/../APKBUILD", + ), + ), + "directory symlink": ( + member("main/foo/link", tarfile.SYMTYPE, linkname="sub"), + ), + "symlink loop": ( + member("main/foo/first", tarfile.SYMTYPE, linkname="second"), + member("main/foo/second", tarfile.SYMTYPE, linkname="first"), + ), + "symlinked directory traversal": ( + member("main/foo/link", tarfile.SYMTYPE, linkname="alias/file"), + member("main/foo/alias", tarfile.SYMTYPE, linkname="sub"), + ), + } + for name, members in cases.items(): + with self.subTest(case=name), tempfile.TemporaryDirectory() as temporary: + destination = Path(temporary) / "recipes" + + with self.assertRaises(collect_alpine_sources.SourceError): + collect_alpine_sources._safe_extract( + self._archive(*recipe, *members), + destination, + "main/foo", + ) + + self.assertEqual(list(destination.iterdir()), []) + + def test_collection_guards_output_before_replacing_it(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + manifest = self._package_manifest(root, "0" * 40) + output = root / "alpine" + (output / "recipes").mkdir(parents=True) + notes = output / "notes.txt" + notes.write_text("keep", encoding="utf-8") + not_directory = root / "file" + not_directory.write_text("keep", encoding="utf-8") + + for target, message in ( + (output, "unexpected entries: notes.txt"), + (not_directory, "is not a directory"), + ): + with ( + self.subTest(output=target.name), + patch.object(collect_alpine_sources, "_prepare_aports") as prepare, + self.assertRaisesRegex( + collect_alpine_sources.SourceError, + message, + ), + ): + collect_alpine_sources.collect_alpine_sources( + [manifest], + target, + root / "aports", + skip_upstream=True, + ) + prepare.assert_not_called() + self.assertTrue(notes.is_file()) + self.assertTrue((output / "recipes").is_dir()) + + notes.unlink() + with ( + patch.object(collect_alpine_sources, "_prepare_aports"), + patch.object( + collect_alpine_sources.shutil, + "rmtree", + side_effect=PermissionError(13, "Permission denied"), + ), + self.assertRaisesRegex( + collect_alpine_sources.SourceError, + "cannot remove previous Alpine source output.*remove it manually", + ), + ): + collect_alpine_sources.collect_alpine_sources( + [manifest], + output, + root / "aports", + skip_upstream=True, + ) + + def test_collection_refuses_symlinked_output(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + output = root / "alpine" + (output / "recipes").mkdir(parents=True) + link = root / "alpine-link" + try: + link.symlink_to(output, target_is_directory=True) + except OSError as error: + self.skipTest(f"symlinks are unavailable: {error}") + + with ( + patch.object(collect_alpine_sources, "_prepare_aports") as prepare, + self.assertRaisesRegex( + collect_alpine_sources.SourceError, + "Alpine source output is not a directory", + ), + ): + collect_alpine_sources.collect_alpine_sources( + [self._package_manifest(root, "0" * 40)], + link, + root / "aports", + skip_upstream=True, + ) + + prepare.assert_not_called() + self.assertTrue((output / "recipes").is_dir()) + + def test_collection_refuses_output_junctions(self): + if sys.platform != "win32": + self.skipTest("directory junctions exist only on Windows") + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + manifest = self._package_manifest(root, "0" * 40) + elsewhere = root / "elsewhere" + (elsewhere / "recipes").mkdir(parents=True) + kept = elsewhere / "recipes" / "kept" + kept.write_bytes(b"kept") + output = root / "alpine" + output.mkdir() + cases = ( + ( + root / "junction", + elsewhere, + root / "junction", + "Alpine source output is not a directory", + ), + ( + output / "recipes", + elsewhere / "recipes", + output, + "Alpine source generated path is not a directory", + ), + ) + for link, target, collected, message in cases: + subprocess.run( + ["cmd", "/c", "mklink", "/J", str(link), str(target)], + capture_output=True, + check=True, + ) + with ( + self.subTest(link=link.name), + patch.object(collect_alpine_sources, "_prepare_aports"), + self.assertRaisesRegex(collect_alpine_sources.SourceError, message), + ): + collect_alpine_sources.collect_alpine_sources( + [manifest], + collected, + root / "aports", + skip_upstream=True, + ) + self.assertEqual(kept.read_bytes(), b"kept") + + def test_upstream_fetch_restores_bundle_ownership_on_exit(self): + with ( + tempfile.TemporaryDirectory() as temporary, + patch.object(collect_alpine_sources, "_run") as run, + ): + collect_alpine_sources._fetch_upstream_sources(Path(temporary), "3.24") + + command = run.call_args.args[0] + self.assertEqual(command[:4], ["docker", "run", "--rm", "--volume"]) + script = command[-1] + self.assertIn("owner=$(stat -c '%u:%g' /bundle)", script) + self.assertIn('chown -R "$owner" /bundle/upstream', script) + self.assertLess( + script.index("trap restore_owner EXIT"), + script.index("mkdir -p /bundle/upstream"), + ) + + +class BuildTests(unittest.TestCase): + def test_build_config_owns_standard_runtime_paths(self): + config = build_config.BuildConfig() + alpine = config.initramfs_config("alpine") + ubuntu_config = config.initramfs_config("ubuntu") + distro = config.distro_layer_config() + + self.assertIsInstance(config.docker, build_config.DockerBuildConfig) + self.assertIsInstance(config.kernel, build_config.KernelBuildConfig) + self.assertIsInstance(config.openvmm, build_config.OpenVmmBuildConfig) + self.assertIsInstance(alpine, build_config.InitramfsBuildConfig) + self.assertIsInstance(ubuntu_config, build_config.InitramfsBuildConfig) + self.assertIsInstance(distro, build_config.DistroLayerBuildConfig) + self.assertIsNone(config.openvmm.backend) + self.assertEqual(config.selected_guests(), ("alpine",)) + self.assertEqual(config.kernel.work, BuildConstants.BUILD_DIR / "linux") + self.assertEqual(config.kernel.output, BuildConstants.BUILD_DIR / "vmlinux") + self.assertEqual( + alpine.work, + BuildConstants.BUILD_DIR / "initramfs-alpine-work", + ) + self.assertEqual( + alpine.output, + BuildConstants.BUILD_DIR / "initramfs.cpio.gz", + ) + self.assertEqual( + ubuntu_config.output, + BuildConstants.BUILD_DIR / "initramfs-ubuntu.cpio.gz", + ) + self.assertEqual( + distro.output, + BuildConstants.BUILD_DIR / "ubuntu-distro.erofs", + ) + self.assertEqual(config.docker.artifact_destination, BuildConstants.BUILD_DIR) + self.assertEqual( + config.docker.linux_source_destination, + BuildConstants.SOURCE_DIR / "linux", + ) + + def test_detects_openvmm_build_platform_without_runtime_devices(self): + cases: tuple[ + tuple[ + str, + build_config.OpenVmmBackend | None, + build_config.OpenVmmPlatform, + str, + build_config.OpenVmmBuildMode, + ], + ..., + ] = ( + ( + "win32", + None, + "windows-msvc", + "x86_64-pc-windows-msvc", + "native", + ), + ( + "win32", + "whp", + "windows-msvc", + "x86_64-pc-windows-msvc", + "native", + ), + ( + "linux", + None, + "linux-gnu", + "x86_64-unknown-linux-gnu", + "native", + ), + ( + "linux", + "kvm", + "linux-gnu", + "x86_64-unknown-linux-gnu", + "native", + ), + ( + "linux", + "mshv", + "linux-musl", + "x86_64-unknown-linux-musl", + "musl", + ), + ) + for host, backend, expected_platform, expected_target, expected_mode in cases: + with self.subTest(host=host, backend=backend): + with ( + patch.object(build.sys, "platform", host), + patch.object(build.os, "access") as access, + ): + config = build_config.OpenVmmBuildConfig() + platform = build.detect_openvmm_platform(backend) + + self.assertEqual(platform, expected_platform) + self.assertEqual(config.openvmm_target(platform), expected_target) + self.assertEqual(config.openvmm_build_mode(platform), expected_mode) + access.assert_not_called() + + def test_rejects_unsupported_openvmm_build_platforms(self): + cases: tuple[tuple[str, build_config.OpenVmmBackend | None], ...] = ( + ("linux", "whp"), + ("win32", "kvm"), + ("win32", "mshv"), + ("darwin", None), + ("darwin", "kvm"), + ("darwin", "mshv"), + ("darwin", "whp"), + ) + for host, backend in cases: + with self.subTest(host=host, backend=backend): + with ( + patch.object(build.sys, "platform", host), + patch.object(build.os, "access") as access, + self.assertRaisesRegex( + common.ScriptError, + "unsupported", + ), + ): + build.detect_openvmm_platform(backend) + access.assert_not_called() + + def test_linux_openvmm_builds_do_not_require_hypervisor_devices(self): + for devices_usable in (False, True): + for combined in (False, True): + with ( + self.subTest(devices_usable=devices_usable, combined=combined), + tempfile.TemporaryDirectory() as temporary, + ): + openvmm_dir = Path(temporary) / "openvmm" + output = openvmm_dir / "target" / "release" / "openvmm" + output.parent.mkdir(parents=True) + output.write_bytes(b"openvmm") + (openvmm_dir / "Cargo.toml").touch() + config = build_config.OpenVmmBuildConfig( + skip_restore=True, + directory=openvmm_dir, + output=output, + ) + + with ( + patch.object(build.sys, "platform", "linux"), + patch.object( + build.os, "access", return_value=devices_usable + ) as access, + patch.object(build, "run_checked") as run_checked, + patch.object(build, "record_openvmm_provenance"), + patch.object(build, "build_guest") as guest, + ): + if combined: + aggregate = build_config.BuildConfig(openvmm=config) + build.build_all(aggregate) + guest.assert_called_once_with(aggregate) + else: + build.build_openvmm(config) + guest.assert_not_called() + + access.assert_not_called() + run_checked.assert_called_once_with( + [ + "cargo", + "build", + "--release", + "-p", + "openvmm", + "--bin", + "openvmm", + ], + cwd=openvmm_dir, + ) + + def test_openvmm_target_output_matches_build_mode(self): + config = build_config.OpenVmmBuildConfig(directory=Path("checkout")) + cases: tuple[tuple[build_config.OpenVmmPlatform, Path], ...] = ( + ("linux-gnu", Path("target") / "release" / "openvmm"), + ( + "linux-musl", + Path("target") / "x86_64-unknown-linux-musl" / "release" / "openvmm", + ), + ("windows-msvc", Path("target") / "release" / "openvmm.exe"), + ) + for platform, relative_output in cases: + with self.subTest(platform=platform): + self.assertEqual( + config.openvmm_target_output(platform), + config.directory / relative_output, + ) + + def test_native_openvmm_build_normalizes_output_before_recording_provenance(self): + platforms: tuple[tuple[build_config.OpenVmmPlatform, str], ...] = ( + ("linux-gnu", "openvmm"), + ("windows-msvc", "openvmm.exe"), + ) + for platform, executable in platforms: + for existing_output in (False, True): + with ( + self.subTest(platform=platform, existing_output=existing_output), + tempfile.TemporaryDirectory() as temporary, + ): + root = Path(temporary) + openvmm_dir = root / "openvmm" + native_output = openvmm_dir / "target" / "release" / executable + native_output.parent.mkdir(parents=True) + binary = b"newly-built-openvmm" + native_output.write_bytes(binary) + (openvmm_dir / "Cargo.toml").touch() + output = root / "artifacts" / executable + if existing_output: + output.parent.mkdir(parents=True) + output.write_bytes(b"stale-openvmm") + config = build_config.OpenVmmBuildConfig( + skip_restore=True, + build_directory=root / "build", + directory=openvmm_dir, + output=output, + ) + revision = b"0bc357bbcf3a654b63dfb51f1103c5751bf3d31f\n" + results = [ + common.CommandResult(("git",), 0, revision, b""), + common.CommandResult(("git",), 0, revision, b""), + common.CommandResult(("git",), 0, b"", b""), + ] + + with ( + patch.object(build, "run_checked"), + patch.object(common, "run_capture", side_effect=results), + ): + build.build_openvmm(config, platform=platform) + + self.assertEqual(output.read_bytes(), binary) + provenance = json.loads( + ( + config.build_directory + / OpenVMMBuildConstants.PROVENANCE_NAME + ).read_text(encoding="utf-8") + ) + self.assertEqual( + provenance["executable_sha256"], + hashlib.sha256(binary).hexdigest(), + ) + + def test_native_openvmm_build_restores_and_records_provenance(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + openvmm_dir = root / "openvmm" + output = openvmm_dir / "target" / "release" / "openvmm" + output.parent.mkdir(parents=True) + output.write_bytes(b"openvmm") + (openvmm_dir / "Cargo.toml").touch() + config = build_config.OpenVmmBuildConfig( + build_directory=root / "build", + directory=openvmm_dir, + output=output, + ) + + with ( + patch.object(build, "run_checked") as run_checked, + patch.object(build, "record_openvmm_provenance") as provenance, + ): + build.build_openvmm(config, platform="linux-gnu") + + self.assertEqual( + run_checked.call_args_list, + [ + call( + [ + "cargo", + "xflowey", + "restore-packages", + "--no-compat-igvm", + ], + cwd=openvmm_dir, + ), + call( + [ + "cargo", + "build", + "--release", + "-p", + "openvmm", + "--bin", + "openvmm", + ], + cwd=openvmm_dir, + ), + ], + ) + provenance.assert_called_once_with(config) + + def test_openvmm_build_honors_skip_restore(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + openvmm_dir = root / "openvmm" + output = openvmm_dir / "target" / "release" / "openvmm" + output.parent.mkdir(parents=True) + output.write_bytes(b"openvmm") + (openvmm_dir / "Cargo.toml").touch() + config = build_config.OpenVmmBuildConfig( + skip_restore=True, + build_directory=root / "build", + directory=openvmm_dir, + output=output, + ) + + with ( + patch.object(build, "run_checked") as run_checked, + patch.object(build, "record_openvmm_provenance"), + ): + build.build_openvmm(config, platform="linux-gnu") + + self.assertEqual(run_checked.call_count, 1) + self.assertEqual(run_checked.call_args.args[0][:2], ["cargo", "build"]) + + def test_musl_openvmm_build_sets_sysroot_and_normalizes_output(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + openvmm_dir = root / "openvmm" + (openvmm_dir / "Cargo.toml").parent.mkdir(parents=True) + (openvmm_dir / "Cargo.toml").touch() + sysroot = openvmm_dir / ".packages" / "extracted" / "x86_64-sysroot" + (sysroot / "lib").mkdir(parents=True) + (sysroot / "lib" / "libsymcrypt.a").touch() + output = openvmm_dir / "target" / "release" / "openvmm" + config = build_config.OpenVmmBuildConfig( + build_directory=root / "build", + directory=openvmm_dir, + output=output, + backend="mshv", + ) + target_output = config.openvmm_target_output("linux-musl") + target_output.parent.mkdir(parents=True) + target_output.write_bytes(b"musl-openvmm") + + with ( + patch.object(build.sys, "platform", "linux"), + patch.object(build.os, "access", return_value=False) as access, + patch.object(build, "run_checked") as run_checked, + patch.object(build, "record_openvmm_provenance") as provenance, + patch.object(Path, "chmod") as chmod, + ): + build.build_openvmm(config) + + access.assert_not_called() + self.assertEqual(run_checked.call_count, 3) + self.assertEqual( + run_checked.call_args_list[1], + call(["rustup", "target", "add", "x86_64-unknown-linux-musl"]), + ) + cargo_call = run_checked.call_args_list[2] + self.assertIn("x86_64-unknown-linux-musl", cargo_call.args[0]) + environment = cargo_call.kwargs["env"] + resolved_sysroot = ( + openvmm_dir.resolve() / ".packages" / "extracted" / "x86_64-sysroot" + ) + self.assertEqual( + environment["X86_64_UNKNOWN_LINUX_MUSL_OPENSSL_DIR"], + os.fspath(resolved_sysroot), + ) + self.assertEqual( + environment["X86_64_UNKNOWN_LINUX_MUSL_SYMCRYPT_LIB_PATH"], + os.fspath(resolved_sysroot / "lib"), + ) + self.assertEqual(output.read_bytes(), b"musl-openvmm") + chmod.assert_called_once() + provenance.assert_called_once_with(config) + + def test_guest_build_routes_one_config_to_native_or_docker_consumers(self): + with ( + patch.object(build, "build_kernel") as kernel, + patch.object(build, "build_initramfs") as initramfs, + patch.object(build, "build_docker_artifacts") as docker, + ): + native = build_config.BuildConfig(native_guest=True) + build.build_guest(native) + kernel.assert_called_once_with(native.kernel) + initramfs.assert_called_once_with(native.initramfs_config("alpine")) + docker.assert_not_called() + + with ( + patch.object(build, "build_kernel") as kernel, + patch.object(build, "build_initramfs") as initramfs, + patch.object(build, "build_docker_artifacts") as docker, + ): + portable = build_config.BuildConfig() + build.build_guest(portable) + docker.assert_called_once_with(portable.docker, "alpine") + kernel.assert_not_called() + initramfs.assert_not_called() + + with ( + patch.object(build, "build_kernel") as kernel, + patch.object(build, "build_initramfs") as initramfs, + patch.object(build, "build_distro_layer") as distro, + ): + all_guests = build_config.BuildConfig(guest="all", native_guest=True) + with self.assertRaisesRegex( + common.ScriptError, + "Azure Linux initramfs builds require Docker", + ): + build.build_guest(all_guests) + kernel.assert_not_called() + initramfs.assert_not_called() + distro.assert_not_called() + + def test_combined_build_passes_explicit_backend_to_openvmm_build(self): + config = build_config.BuildConfig( + openvmm=build_config.OpenVmmBuildConfig(backend="mshv") + ) + with ( + patch.object(build.sys, "platform", "linux"), + patch.object(build.os, "access", return_value=False) as access, + patch.object(build, "build_guest") as guest, + patch.object(build, "build_openvmm") as openvmm, + ): + build.build_all(config) + + access.assert_not_called() + guest.assert_called_once_with(config) + openvmm.assert_called_once_with(config.openvmm, platform="linux-musl") + + def test_combined_build_validates_platform_before_building_guest(self): + config = build_config.BuildConfig( + openvmm=build_config.OpenVmmBuildConfig(backend="whp") + ) + with ( + patch.object(build.sys, "platform", "linux"), + patch.object(build, "build_guest") as guest, + patch.object(build, "build_openvmm") as openvmm, + self.assertRaisesRegex( + common.ScriptError, "backend 'whp' is unsupported on linux" + ), + ): + build.build_all(config) + + guest.assert_not_called() + openvmm.assert_not_called() + + def test_release_source_collection_passes_docker_build_config(self): + config = build_config.DockerBuildConfig( + linux_source_destination=Path("custom-linux-source") + ) + with ( + patch.object( + release, + "_guest_release_inputs", + return_value=((), (), (), ()), + ) as guest_release_inputs, + patch.object(release, "collect_alpine_sources"), + patch.object(release, "collect_ubuntu_sources"), + patch.object(release, "build_docker_linux_source") as linux_source, + ): + release.collect_release_sources(config) + + guest_release_inputs.assert_called_once_with(include_azurelinux=False) + linux_source.assert_called_once_with(config) + + def test_records_openvmm_revision_cleanliness_and_executable_hash(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + openvmm_dir = root / "openvmm" + executable = openvmm_dir / "target" / "release" / "openvmm" + executable.parent.mkdir(parents=True) + executable.write_bytes(b"openvmm") + revision = b"0bc357bbcf3a654b63dfb51f1103c5751bf3d31f\n" + results = [ + common.CommandResult(("git",), 0, revision, b""), + common.CommandResult(("git",), 0, revision, b""), + common.CommandResult(("git",), 0, b" M src/main.rs\n", b""), + ] + + with ( + patch.object(BuildConstants, "REPO_ROOT", root), + patch.object(common, "run_capture", side_effect=results), + ): + build.record_openvmm_provenance( + build_config.OpenVmmBuildConfig( + build_directory=root / "build", + directory=openvmm_dir, + output=executable, + ) + ) + + provenance = json.loads( + (root / "build" / OpenVMMBuildConstants.PROVENANCE_NAME).read_text( + encoding="utf-8" + ) + ) + self.assertEqual(provenance["source_revision"], revision.decode().strip()) + self.assertFalse(provenance["source_clean"]) + self.assertEqual( + provenance["executable_sha256"], + hashlib.sha256(b"openvmm").hexdigest(), + ) + + def test_kernel_build_rejects_input_config_mutation_during_build(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + input_config = root / "kernel" / "config-microvm" + input_config.parent.mkdir(parents=True) + input_config.write_text( + "\n".join( + ( + *KernelBuildConstants.REQUIRED_DIRECT_BOOT_CONFIG, + *KernelBuildConstants.REQUIRED_VIRTIO_CONSOLE_CONFIG, + *KernelBuildConstants.REQUIRED_SHARED_STATUS_CONFIG, + *KernelBuildConstants.REQUIRED_SANDBOX_CONFIG, + *KernelBuildConstants.REQUIRED_NETWORK_CONFIG, + *KernelBuildConstants.REQUIRED_TIME_ABI_CONFIG, + *KernelBuildConstants.REQUIRED_HARDENING_CONFIG, + ) + ) + + "\n", + encoding="utf-8", + ) + patch_path = root / "kernel" / "patches" / "example.patch" + patch_path.parent.mkdir() + patch_path.write_text("patch", encoding="utf-8") + source = root / "source" + source.mkdir() + work = root / "work" + output = root / "output" / "vmlinux" + prior_provenance = output.with_name(KernelBuildConstants.PROVENANCE_NAME) + prior_provenance.parent.mkdir() + prior_provenance.write_text("stale", encoding="utf-8") + + with patch.object(BuildConstants, "REPO_ROOT", root): + source_fingerprint = build._kernel_source_fingerprint() + + def run_build(command: object, **_kwargs: object) -> None: + if isinstance(command, list) and command[-1] == "vmlinux": + (work / "vmlinux").write_bytes(b"kernel") + input_config.write_text("CONFIG_CHANGED=y\n", encoding="utf-8") + + with ( + patch.object(BuildConstants, "REPO_ROOT", root), + patch.object(build, "_require_linux"), + patch.object(build, "require_tool", return_value="tool"), + patch.object( + build, + "prepare_kernel_source", + return_value=(source, source_fingerprint), + ), + patch.object(build, "run_checked", side_effect=run_build), + self.assertRaisesRegex( + common.ScriptError, + "inputs changed during the build", + ), + ): + build.build_kernel( + build_config.KernelBuildConfig( + work=work, + output=output, + ) + ) + + self.assertFalse(output.exists()) + self.assertFalse(output.with_name("vmlinux.config").exists()) + self.assertFalse(prior_provenance.exists()) + + def test_materialize_kernel_provenance_inputs_bypasses_mutable_index(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + git_environment = { + key: value + for key, value in os.environ.items() + if not key.startswith("GIT_CONFIG_") + } + config = root / "kernel" / "config-microvm" + patch_path = root / "kernel" / "patches" / "example.patch" + stale_patch = root / "kernel" / "patches" / "stale.patch" + config.parent.mkdir(parents=True) + patch_path.parent.mkdir() + config.write_bytes(b"CONFIG_EXAMPLE=y\n") + patch_path.write_bytes(b"patch\n") + (root / ".gitattributes").write_text( + "\n".join( + ( + "kernel/config-microvm text eol=lf", + "kernel/patches/*.patch text eol=lf", + ) + ) + + "\n", + encoding="ascii", + ) + subprocess.run( + ["git", "init", "-q"], + cwd=root, + env=git_environment, + check=True, + ) + subprocess.run( + ["git", "config", "user.name", "NVX Tests"], + cwd=root, + env=git_environment, + check=True, + ) + subprocess.run( + ["git", "config", "user.email", "nvx-tests@example.com"], + cwd=root, + env=git_environment, + check=True, + ) + subprocess.run( + [ + "git", + "add", + ".gitattributes", + "kernel/config-microvm", + "kernel/patches/example.patch", + ], + cwd=root, + env=git_environment, + check=True, + ) + subprocess.run( + ["git", "commit", "-qm", "kernel provenance fixture"], + cwd=root, + env=git_environment, + check=True, + ) + git_dir = Path( + subprocess.run( + ["git", "rev-parse", "--git-dir"], + cwd=root, + env=git_environment, + check=True, + capture_output=True, + text=True, + ).stdout.strip() + ) + (root / git_dir / "info" / "attributes").write_text( + "\n".join( + ( + "kernel/config-microvm -text", + "kernel/patches/*.patch -text", + ) + ) + + "\n", + encoding="ascii", + ) + config.write_bytes(b"CONFIG_EXAMPLE=y\r\n") + patch_path.write_bytes(b"patch\r\n") + stale_patch.write_bytes(b"stale\r\n") + subprocess.run( + [ + "git", + "add", + "kernel/config-microvm", + "kernel/patches/example.patch", + "kernel/patches/stale.patch", + ], + cwd=root, + env=git_environment, + check=True, + ) + subprocess.run( + [ + "git", + "checkout-index", + "--force", + "--", + "kernel/config-microvm", + "kernel/patches/example.patch", + "kernel/patches/stale.patch", + ], + cwd=root, + env=git_environment, + check=True, + ) + self.assertEqual(config.read_bytes(), b"CONFIG_EXAMPLE=y\r\n") + self.assertEqual(patch_path.read_bytes(), b"patch\r\n") + self.assertEqual(stale_patch.read_bytes(), b"stale\r\n") + + with ( + patch.dict(os.environ, git_environment, clear=True), + patch.object(BuildConstants, "REPO_ROOT", root), + ): + build.materialize_kernel_provenance_inputs() + + self.assertEqual(config.read_bytes(), b"CONFIG_EXAMPLE=y\n") + self.assertEqual(patch_path.read_bytes(), b"patch\n") + self.assertFalse(stale_patch.exists()) + + def test_manifest_tracks_every_kernel_patch(self): + manifest = json.loads( + (BuildConstants.REPO_ROOT / "SOURCE-MANIFEST.json").read_text( + encoding="utf-8" + ) + ) + self.assertEqual( + manifest["linux"]["patches"], + [ + path.relative_to(BuildConstants.REPO_ROOT).as_posix() + for path in build._kernel_patch_files() + ], + ) + self.assertNotIn("guest_agent", manifest) + self.assertEqual( + manifest["openvmm"], + { + "microvm_abi_version": 2, + "control_session_protocol_version": 1, + "control_contract_revision": "nvx-microvm-v2-control-v2", + }, + ) + + def test_guest_descriptors_preserve_alpine_and_select_guest_outputs(self): + alpine = guests.guest_descriptor("alpine") + ubuntu_guest = guests.guest_descriptor("ubuntu") + azurelinux_guest = guests.guest_descriptor("azurelinux") + + self.assertEqual(alpine.initramfs_name, "initramfs.cpio.gz") + self.assertEqual(alpine.default_memory_mib, 128) + self.assertTrue(alpine.sandbox_control) + self.assertEqual( + ubuntu_guest.initramfs_name, + "initramfs-ubuntu.cpio.gz", + ) + self.assertEqual(ubuntu_guest.default_memory_mib, 512) + self.assertFalse(ubuntu_guest.sandbox_control) + self.assertEqual( + azurelinux_guest.initramfs_name, + "initramfs-azurelinux.cpio.gz", + ) + self.assertEqual(azurelinux_guest.default_memory_mib, 512) + self.assertFalse(azurelinux_guest.sandbox_control) + + def test_azurelinux_manifest_and_package_lock_match_build_pins(self): + manifest = json.loads( + (BuildConstants.REPO_ROOT / "SOURCE-MANIFEST.json").read_text( + encoding="utf-8" + ) + )["azurelinux"] + self.assertEqual(manifest["image"], AzureLinuxBuildConstants.IMAGE) + self.assertEqual(manifest["package_lock"], "azurelinux/packages.lock.json") + self.assertEqual( + manifest["package_lock_sha256"], + azurelinux.package_lock_sha256(), + ) + packages = azurelinux.load_package_lock() + self.assertEqual( + {"busybox", "util-linux"} - {package["name"] for package in packages}, + set(), + ) + for package in packages: + self.assertTrue( + package["url"].startswith( + f"{AzureLinuxBuildConstants.REPOSITORY_URL}/Packages/" + ) + ) + attributes = (BuildConstants.REPO_ROOT / ".gitattributes").read_text( + encoding="utf-8" + ) + self.assertIn("azurelinux/packages.lock.json text eol=lf", attributes) + + def test_azurelinux_package_lock_rejects_unpinned_records(self): + document = json.loads( + azurelinux.package_lock_path().read_text(encoding="utf-8") + ) + first = document["packages"][0] + cases: tuple[tuple[str, dict[str, object], str], ...] = ( + ("image", {"image": "mcr.microsoft.com/azurelinux/base/core:3.0"}, "image"), + ( + "url", + { + "packages": [ + {**first, "url": first["url"].replace("/prod/", "/preview/")}, + *document["packages"][1:], + ] + }, + "must be fetched from", + ), + ( + "sha256", + { + "packages": [ + {**first, "sha256": "0" * 63}, + *document["packages"][1:], + ] + }, + "invalid SHA-256", + ), + ( + "order", + {"packages": list(reversed(document["packages"]))}, + "sorted by name", + ), + ) + for name, update, message in cases: + with ( + self.subTest(case=name), + tempfile.TemporaryDirectory() as temporary, + ): + path = Path(temporary) / "packages.lock.json" + path.write_text(json.dumps({**document, **update}), encoding="utf-8") + with self.assertRaisesRegex(common.ScriptError, message): + azurelinux.load_package_lock(path) + + def test_azurelinux_download_packages_verifies_each_locked_rpm(self): + destination = Path("rpms") + with patch.object(azurelinux, "download_verified") as download_verified: + downloaded = azurelinux.download_packages(destination) + + packages = azurelinux.load_package_lock() + self.assertEqual( + download_verified.call_args_list, + [ + call( + package["url"], + destination / package["url"].rsplit("/", maxsplit=1)[-1], + package["sha256"], + ) + for package in packages + ], + ) + self.assertEqual(len(downloaded), len(packages)) + + def test_ubuntu_manifest_and_package_lock_match_build_pins(self): + manifest = json.loads( + (BuildConstants.REPO_ROOT / "SOURCE-MANIFEST.json").read_text( + encoding="utf-8" + ) + ) + ubuntu_manifest = manifest["ubuntu"] + self.assertEqual( + ubuntu_manifest["version"], + UbuntuBuildConstants.VERSION, + ) + self.assertEqual( + ubuntu_manifest["base_url"], + UbuntuBuildConstants.BASE_URL, + ) + self.assertEqual( + ubuntu_manifest["base_sha256"], + UbuntuBuildConstants.BASE_SHA256, + ) + self.assertEqual( + ubuntu_manifest["archive_keyring_url"], + UbuntuBuildConstants.ARCHIVE_KEYRING_URL, + ) + self.assertEqual( + ubuntu_manifest["archive_keyring_sha256"], + UbuntuBuildConstants.ARCHIVE_KEYRING_SHA256, + ) + self.assertEqual( + ubuntu_manifest["package_lock_sha256"], + ubuntu.package_lock_sha256(), + ) + self.assertEqual( + ubuntu_manifest["package_lock_sha256"], + "fcdd30223b96fc26e24fcf4b6763a4a2504e9cd25882e68e89fbf6bf45277ef0", + ) + packages = ubuntu.load_package_lock() + self.assertEqual( + [package["name"] for package in packages], + [ + "busybox-static", + "iputils-ping", + "libcap2", + "libidn2-0", + "libunistring5", + "net-tools", + "netcat-openbsd", + ], + ) + + def test_ubuntu_package_lock_read_errors_are_actionable(self): + with tempfile.TemporaryDirectory() as temporary: + path = Path(temporary) / "packages.json" + path.write_text("{", encoding="utf-8") + + with self.assertRaisesRegex( + common.ScriptError, "failed to read Ubuntu package lock" + ): + ubuntu.load_package_lock(path) + + def test_ubuntu_safe_extractor_rejects_archive_symlink_escape(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + archive_path = root / "unsafe.tar.gz" + with tarfile.open(archive_path, "w:gz") as archive_file: + symlink = tarfile.TarInfo("etc") + symlink.type = tarfile.SYMTYPE + symlink.linkname = "../../outside" + archive_file.addfile(symlink) + payload = b"nameserver 192.0.2.1\n" + resolver = tarfile.TarInfo("etc/resolv.conf") + resolver.size = len(payload) + archive_file.addfile(resolver, io.BytesIO(payload)) + + with self.assertRaisesRegex(common.ScriptError, "escapes"): + ubuntu.safe_extract_tar( + archive_path, + root / "extracted", + label="test archive", + ) + self.assertFalse((root / "outside").exists()) + + def test_ubuntu_safe_extractor_roots_absolute_symlinks(self): + self.assertEqual( + ubuntu._virtual_symlink_target( + PurePosixPath("var/run"), + "/run", + "test archive", + ), + "../run", + ) + self.assertEqual( + ubuntu._virtual_symlink_target( + PurePosixPath("etc/alternatives/awk"), + "/usr/bin/mawk", + "test archive", + ), + "../../usr/bin/mawk", + ) + + def test_ubuntu_safe_extractor_rejects_absolute_and_parent_paths(self): + for member_name in ("/absolute", "../parent"): + with self.subTest(member_name=member_name): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + archive_path = root / "unsafe.tar" + with tarfile.open(archive_path, "w") as archive_file: + payload = b"x" + member = tarfile.TarInfo(member_name) + member.size = len(payload) + archive_file.addfile(member, io.BytesIO(payload)) + with self.assertRaisesRegex( + common.ScriptError, + "absolute path|path traversal", + ): + ubuntu.safe_extract_tar( + archive_path, + root / "extracted", + label="test archive", + ) + + def test_ubuntu_rootfs_digest_and_erofs_uuid_are_deterministic(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + first = root / "first" + second = root / "second" + for directory in (first, second): + (directory / "etc").mkdir(parents=True) + (directory / "etc" / "os-release").write_text( + "ID=ubuntu\n", + encoding="ascii", + ) + os.utime(second / "etc" / "os-release", (1234, 1234)) + + first_digest = ubuntu.rootfs_sha256(first) + second_digest = ubuntu.rootfs_sha256(second) + self.assertEqual(first_digest, second_digest) + self.assertEqual( + ubuntu.erofs_uuid(first_digest), + ubuntu.erofs_uuid(second_digest), + ) + + def test_ubuntu_prepare_root_normalizes_root_directory_mode(self): + with tempfile.TemporaryDirectory() as temporary: + temporary_root = Path(temporary) + work = temporary_root / "work" + expected_root = work / "root" + + def extract( + archive: Path, + destination: Path, + *, + label: str, + ) -> None: + self.assertEqual( + archive.name, + f"ubuntu-base-{UbuntuBuildConstants.VERSION}-base-amd64.tar.gz", + ) + self.assertEqual(label, "Ubuntu Base archive") + status = destination / "var" / "lib" / "dpkg" / "status" + status.parent.mkdir(parents=True) + status.write_text("", encoding="utf-8") + + with ( + patch.object( + ubuntu, + "cache_root", + return_value=temporary_root / "cache", + ), + patch.object(ubuntu, "download_verified"), + patch.object(ubuntu, "safe_extract_tar", side_effect=extract), + patch.object(ubuntu, "_validate_ubuntu_identity"), + patch.object(ubuntu, "load_package_lock", return_value=()), + patch.object(ubuntu, "_validate_package_closure"), + patch.object(ubuntu, "_customize_root"), + patch.object(ubuntu.Path, "chmod", autospec=True) as chmod, + ): + self.assertEqual(ubuntu.prepare_root(work), expected_root) + + chmod.assert_called_once_with(expected_root, 0o755) + + def test_ubuntu_customization_installs_busybox_wget(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + for relative in ("etc", "usr/bin", "usr/sbin"): + (root / relative).mkdir(parents=True) + with ( + patch.object(ubuntu, "_clear_directory"), + patch.object(ubuntu, "_validate_accounts"), + patch.object(ubuntu, "_validate_usr_merge"), + patch.object(ubuntu, "_ensure_symlink") as ensure_symlink, + ): + ubuntu._customize_root(root) + + ensure_symlink.assert_any_call(root, "usr/bin/wget", "busybox") + + def test_distro_layer_refuses_existing_output_without_replace(self): + with tempfile.TemporaryDirectory() as temporary: + output = Path(temporary) / "ubuntu.erofs" + output.touch() + with ( + patch.object(build.sys, "platform", "linux"), + self.assertRaisesRegex(common.ScriptError, "refusing to replace"), + ): + build.build_distro_layer( + build_config.DistroLayerBuildConfig(output=output) + ) + + def test_ubuntu_initramfs_manifest_binds_artifact_digest(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + output = root / "initramfs-ubuntu.cpio.gz" + output.write_bytes(b"ubuntu initramfs") + with patch.object( + ubuntu, + "package_manifest", + return_value={"format": 1, "guest": "ubuntu"}, + ): + build._write_ubuntu_manifest(root, output, {}, "a" * 64) + + manifest = json.loads( + output.with_name(f"{output.name}.packages.json").read_text( + encoding="utf-8" + ) + ) + self.assertEqual(manifest["artifact"], output.name) + self.assertEqual( + manifest["artifact_sha256"], + common.sha256_file(output), + ) + self.assertEqual(manifest["input_sha256"], "a" * 64) + + def test_ubuntu_source_requirements_deduplicate_binary_manifests(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + manifests: list[Path] = [] + for name in ("initramfs.json", "layer.json"): + path = root / name + path.write_text( + json.dumps( + { + "guest": "ubuntu", + "release": UbuntuBuildConstants.VERSION, + "architecture": UbuntuBuildConstants.ARCHITECTURE, + "packages": [ + { + "source_name": "glibc", + "source_version": "2.43-2ubuntu2.3", + } + ], + } + ), + encoding="utf-8", + ) + manifests.append(path) + + requirements = collect_ubuntu_sources._source_requirements(manifests) + + self.assertEqual( + requirements, + ( + { + "source_name": "glibc", + "source_version": "2.43-2ubuntu2.3", + }, + ), + ) + + def test_ubuntu_dsc_validation_matches_source_index(self): + payload = b"source" + digest = hashlib.sha256(payload).hexdigest() + with tempfile.TemporaryDirectory() as temporary: + dsc = Path(temporary) / "example_1.0.dsc" + dsc.write_text( + ( + "Format: 3.0 (quilt)\n" + "Source: example\n" + "Version: 1.0\n" + "Checksums-Sha256:\n" + f" {digest} {len(payload)} example_1.0.orig.tar.xz\n" + ), + encoding="utf-8", + ) + record: collect_ubuntu_sources.SourceRecord = { + "source_name": "example", + "source_version": "1.0", + "directory": "pool/main/e/example", + "index_url": "https://archive.invalid/Sources.xz", + "index_sha256": "0" * 64, + "index_release_url": "https://archive.invalid/InRelease", + "index_release_sha256": "1" * 64, + "files": [ + { + "name": "example_1.0.dsc", + "size": dsc.stat().st_size, + "sha256": common.sha256_file(dsc), + "url": "https://archive.invalid/example_1.0.dsc", + }, + { + "name": "example_1.0.orig.tar.xz", + "size": len(payload), + "sha256": digest, + "url": "https://archive.invalid/example_1.0.orig.tar.xz", + }, + ], + } + + collect_ubuntu_sources._validate_dsc(dsc, record) + record["files"][1]["sha256"] = "f" * 64 + with self.assertRaisesRegex( + common.ScriptError, + "do not match", + ): + collect_ubuntu_sources._validate_dsc(dsc, record) + + def test_ubuntu_source_index_is_authenticated_by_signed_release(self): + dsc_payload = b"dsc" + source_payload = b"source" + sources = ( + "Package: example\n" + "Version: 1.0\n" + "Directory: pool/main/e/example\n" + "Checksums-Sha256:\n" + f" {hashlib.sha256(dsc_payload).hexdigest()} " + f"{len(dsc_payload)} example_1.0.dsc\n" + f" {hashlib.sha256(source_payload).hexdigest()} " + f"{len(source_payload)} example_1.0.orig.tar.xz\n" + ).encode() + compressed = lzma.compress(sources) + index_sha256 = hashlib.sha256(compressed).hexdigest() + inrelease = ( + "-----BEGIN PGP SIGNED MESSAGE-----\n" + "Hash: SHA256\n" + "\n" + "Origin: Ubuntu\n" + f"Codename: {UbuntuBuildConstants.CODENAME}\n" + f"Suite: {UbuntuBuildConstants.CODENAME}\n" + "SHA256:\n" + f" {index_sha256} {len(compressed)} main/source/Sources.xz\n" + "-----BEGIN PGP SIGNATURE-----\n" + "test\n" + "-----END PGP SIGNATURE-----\n" + ) + + def download_release(_url: str, destination: Path) -> None: + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text(inrelease, encoding="utf-8") + + def download_index( + _url: str, + destination: Path, + expected_sha256: str, + ) -> None: + self.assertEqual(expected_sha256, index_sha256) + destination.write_bytes(compressed) + + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + keyring = root / "ubuntu-archive-keyring.gpg" + keyring.write_bytes(b"keyring") + with ( + patch.object( + collect_ubuntu_sources, + "download", + side_effect=download_release, + ), + patch.object( + collect_ubuntu_sources, + "download_verified", + side_effect=download_index, + ), + patch.object( + collect_ubuntu_sources, + "require_tool", + return_value="gpgv", + ), + patch.object(collect_ubuntu_sources, "run_checked") as run_checked, + ): + records, metadata = ( + collect_ubuntu_sources._load_authenticated_source_index( + root / "cache", + keyring, + "https://archive.invalid/ubuntu", + UbuntuBuildConstants.CODENAME, + "main", + ) + ) + + run_checked.assert_called_once() + self.assertEqual(run_checked.call_args.args[0][0], "gpgv") + self.assertIn("--homedir", run_checked.call_args.args[0]) + record = records[("example", "1.0")] + self.assertEqual( + record["index_release_url"], + "https://archive.invalid/ubuntu/dists/resolute/InRelease", + ) + self.assertEqual( + [item["role"] for item in metadata], + ["ubuntu-inrelease", "ubuntu-source-index"], + ) + self.assertEqual( + metadata[1]["authenticated_by"], + record["index_release_url"], + ) + + def test_launchpad_source_record_retains_verifiable_raw_metadata(self): + source_payload = b"source archive" + source_sha256 = hashlib.sha256(source_payload).hexdigest() + self_link = f"{UbuntuBuildConstants.LAUNCHPAD_ARCHIVE_API}/+sourcepub/123" + query = { + "entries": [ + { + "source_package_name": "example", + "source_package_version": "1.0", + "status": "Superseded", + "date_published": "2026-01-01T00:00:00Z", + "date_superseded": "2026-01-03T00:00:00Z", + "pocket": "Updates", + "component_name": "main", + "self_link": self_link, + } + ] + } + source_urls = [ + "https://launchpad.net/ubuntu/+archive/primary/+sourcefiles/" + "example/1.0/example_1.0.dsc", + "https://launchpad.net/ubuntu/+archive/primary/+sourcefiles/" + "example/1.0/example_1.0.orig.tar.xz", + ] + + def download_json(url: str, destination: Path) -> object: + document: object = source_urls if "sourceFileUrls" in url else query + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text( + json.dumps(document, separators=(",", ":")) + "\n", + encoding="utf-8", + ) + return document + + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + snapshot_release = root / "cache" / "snapshot.InRelease" + snapshot_release.parent.mkdir(parents=True) + snapshot_release.write_text("signed release", encoding="utf-8") + snapshot_record: collect_ubuntu_sources.SourceRecord = { + "source_name": "example", + "source_version": "1.0", + "directory": "pool/main/e/example", + "index_url": ( + "https://snapshot.ubuntu.com/ubuntu/20260102T000000Z/" + "dists/resolute-updates/main/source/Sources.xz" + ), + "index_sha256": "2" * 64, + "index_release_url": ( + "https://snapshot.ubuntu.com/ubuntu/20260102T000000Z/" + "dists/resolute-updates/InRelease" + ), + "index_release_sha256": common.sha256_file(snapshot_release), + "files": [ + { + "name": "example_1.0.dsc", + "size": 3, + "sha256": "4" * 64, + "url": ( + "https://snapshot.ubuntu.com/ubuntu/" + "20260102T000000Z/pool/main/e/example/" + "example_1.0.dsc" + ), + }, + { + "name": "example_1.0.orig.tar.xz", + "size": len(source_payload), + "sha256": source_sha256, + "url": ( + "https://snapshot.ubuntu.com/ubuntu/" + "20260102T000000Z/pool/main/e/example/" + "example_1.0.orig.tar.xz" + ), + }, + ], + } + snapshot_metadata: list[collect_ubuntu_sources.SourceMetadata] = [ + { + "role": "ubuntu-inrelease", + "url": snapshot_record["index_release_url"], + "sha256": snapshot_record["index_release_sha256"], + "cache_path": snapshot_release, + "output_name": "snapshot.InRelease", + "authenticated_by": (UbuntuBuildConstants.ARCHIVE_KEYRING_URL), + }, + { + "role": "ubuntu-source-index", + "url": snapshot_record["index_url"], + "sha256": snapshot_record["index_sha256"], + "cache_path": None, + "output_name": None, + "authenticated_by": snapshot_record["index_release_url"], + }, + ] + with ( + patch.object( + collect_ubuntu_sources, + "_download_json", + side_effect=download_json, + ), + patch.object( + collect_ubuntu_sources, + "_load_authenticated_source_index", + return_value=( + {("example", "1.0"): snapshot_record}, + snapshot_metadata, + ), + ), + ): + record, metadata = collect_ubuntu_sources._launchpad_source_record( + root / "cache", + root / "ubuntu-archive-keyring.gpg", + "example", + "1.0", + ) + + self.assertEqual(record, snapshot_record) + self.assertEqual( + [item["role"] for item in metadata], + [ + "ubuntu-inrelease", + "ubuntu-source-index", + "launchpad-publishing-history", + "launchpad-source-file-urls", + ], + ) + for item in metadata: + if item["cache_path"] is None: + continue + self.assertEqual( + item["sha256"], + common.sha256_file(item["cache_path"]), + ) + + output = root / "output" + output.mkdir() + manifest_metadata = collect_ubuntu_sources._materialize_source_metadata( + output, + metadata, + ) + for item, manifest_item in zip(metadata, manifest_metadata, strict=True): + if item["cache_path"] is None: + self.assertNotIn("path", manifest_item) + continue + retained = output / manifest_item["path"] + self.assertEqual( + retained.read_bytes(), + item["cache_path"].read_bytes(), + ) + self.assertEqual( + common.sha256_file(retained), + manifest_item["sha256"], + ) + + def test_ubuntu_source_collection_rejects_unexpected_output_entries(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + output = root / "output" + output.mkdir() + stale = output / "unrelated.txt" + stale.write_text("do not archive", encoding="utf-8") + + with self.assertRaisesRegex( + common.ScriptError, + "unexpected entries: unrelated.txt", + ): + collect_ubuntu_sources.collect_ubuntu_sources( + [], + output, + root / "cache", + ) + + self.assertEqual(stale.read_text(encoding="utf-8"), "do not archive") + + def test_ubuntu_source_collection_validates_output_before_resolving(self): + output = MagicMock(spec=Path) + with patch.object( + collect_ubuntu_sources, + "_validate_source_output", + ) as validate: + + def resolve() -> Path: + validate.assert_called_once_with(output) + raise RuntimeError("stop after ordering assertion") + + output.resolve.side_effect = resolve + with self.assertRaisesRegex(RuntimeError, "ordering assertion"): + collect_ubuntu_sources.collect_ubuntu_sources( + [], + output, + Path("cache"), + ) + + def test_ci_kernel_cache_key_includes_patches(self): + action = ( + BuildConstants.REPO_ROOT + / ".github" + / "actions" + / "build-guest-artifacts" + / "action.yml" + ).read_text(encoding="utf-8") + for cache_input in ( + "SOURCE-MANIFEST.json", + "kernel/config-microvm", + "kernel/patches/**", + "scripts/nvx_tools/build.py", + "scripts/nvx_tools/common.py", + ): + self.assertIn(cache_input, action) + self.assertIn("linux-kernel-v1-", action) + self.assertEqual(action.count("build/vmlinux.provenance.json"), 2) + self.assertEqual(action.count("build/initramfs.provenance.json"), 2) + + def test_ci_guest_cache_keys_include_build_configuration(self): + action = ( + BuildConstants.REPO_ROOT + / ".github" + / "actions" + / "build-guest-artifacts" + / "action.yml" + ).read_text(encoding="utf-8") + cache_names = ( + "KERNEL_INPUT_HASH", + "DEBUG_KERNEL_INPUT_HASH", + "ALPINE_INPUT_HASH", + "UBUNTU_INPUT_HASH", + "AZURELINUX_INPUT_HASH", + ) + # Every artifact, both kernels included, builds inside the image that + # docker/Dockerfile pins, so a toolchain change must miss every cache. + self.assertEqual( + action.count("--file docker/Dockerfile"), + action.count("docker build"), + ) + for cache_name in cache_names: + with self.subTest(cache=cache_name): + cache_input = next( + line + for line in action.splitlines() + if line.strip().startswith(f"{cache_name}:") + ) + self.assertIn("'docker/Dockerfile'", cache_input) + self.assertIn("'scripts/nvx_tools/build_config.py'", cache_input) + self.assertIn("'scripts/nvx_tools/build_constants.py'", cache_input) + + def test_ci_azurelinux_cache_key_covers_build_input_digest(self): + action = ( + BuildConstants.REPO_ROOT + / ".github" + / "actions" + / "build-guest-artifacts" + / "action.yml" + ).read_text(encoding="utf-8") + cache_input = next( + line + for line in action.splitlines() + if line.strip().startswith("AZURELINUX_INPUT_HASH:") + ) + patterns = cache_input.split("'")[1::2] + self.assertIn("scripts/nvx_tools/azurelinux.py", patterns) + for path in azurelinux.input_files(): + relative = path.relative_to(BuildConstants.REPO_ROOT).as_posix() + with self.subTest(path=relative): + self.assertTrue( + any( + relative == pattern + or ( + pattern.endswith("/**") + and relative.startswith(pattern[:-2]) + ) + for pattern in patterns + ) + ) + + def test_kernel_input_config_uses_canonical_lf_line_endings(self): + attributes = (BuildConstants.REPO_ROOT / ".gitattributes").read_text( + encoding="utf-8" + ) + for path in ( + KernelBuildConstants.INPUT_CONFIG, + KernelBuildConstants.DEBUG_CONFIG_FRAGMENT, + ): + with self.subTest(path=path): + self.assertIn( + f"{path.as_posix()} text eol=lf", + attributes.splitlines(), + ) + + def test_ci_debug_kernel_is_opt_in_and_keyed_by_its_fragment(self): + action = ( + BuildConstants.REPO_ROOT + / ".github" + / "actions" + / "build-guest-artifacts" + / "action.yml" + ).read_text(encoding="utf-8") + cache_input = next( + line + for line in action.splitlines() + if line.strip().startswith("DEBUG_KERNEL_INPUT_HASH:") + ) + for path in ( + "kernel/config-microvm", + "kernel/config-microvm-debug", + "kernel/patches/**", + "scripts/nvx_tools/build.py", + "scripts/nvx_tools/build_constants.py", + ): + with self.subTest(path=path): + self.assertIn(f"'{path}'", cache_input) + kernel_input = next( + line + for line in action.splitlines() + if line.strip().startswith("KERNEL_INPUT_HASH:") + ) + self.assertNotIn("config-microvm-debug", kernel_input) + build_step = _composite_action_step(action, "Build Linux debug kernel (Docker)") + self.assertIn("inputs.debug-kernel == 'true'", build_step) + self.assertIn( + f"--target {DockerBuildConstants.DEBUG_KERNEL_TARGET}", build_step + ) + self.assertIn('default: "false"', action) + for name in DockerBuildConstants.DEBUG_KERNEL_ARTIFACT_NAMES: + with self.subTest(artifact=name): + self.assertEqual( + [line.strip() for line in action.splitlines()].count( + f"build/{name}" + ), + 2, + ) + + def test_ci_guest_images_input_leaves_only_the_debug_kernel(self): + action = ( + BuildConstants.REPO_ROOT + / ".github" + / "actions" + / "build-guest-artifacts" + / "action.yml" + ).read_text(encoding="utf-8") + guard = "inputs.guest-images == 'true'" + self.assertIn(" guest-images:\n", action) + self.assertIn('default: "true"', action) + steps = [ + line.strip().removeprefix("- name: ") + for line in action.splitlines() + if line.startswith(" - name: ") + ] + debug_steps = {name for name in steps if "debug kernel" in name} + self.assertEqual(len(debug_steps), 3) + always = {"Resolve guest artifact versions", "Resolve guest cache keys"} + for name in steps: + with self.subTest(step=name): + step = _composite_action_step(action, name) + if name in debug_steps or name in always: + self.assertNotIn("guest-images", step) + else: + self.assertIn(guard, step) + + def test_ci_guest_cache_keys_include_shared_build_modules(self): + action = ( + BuildConstants.REPO_ROOT + / ".github" + / "actions" + / "build-guest-artifacts" + / "action.yml" + ).read_text(encoding="utf-8") + for variable in ("ALPINE_INPUT_HASH", "UBUNTU_INPUT_HASH"): + assignment = next(line for line in action.splitlines() if variable in line) + self.assertIn("'scripts/nvx_tools/common.py'", assignment) + self.assertIn("'scripts/nvx_tools/guests.py'", assignment) + + def test_quality_checks_include_shared_identity_probe(self): + action = ( + BuildConstants.REPO_ROOT + / ".github" + / "actions" + / "check-quality" + / "action.yml" + ).read_text(encoding="utf-8") + + self.assertEqual(action.count("guest/common/nvx-identity-probe"), 2) + + def test_docker_guest_builder_pins_erofs_toolchain(self): + dockerfile = (BuildConstants.REPO_ROOT / "docker" / "Dockerfile").read_text( + encoding="utf-8" + ) + self.assertIn( + "ARG DEBIAN_IMAGE=debian:12-slim@sha256:" + "3783cc01769c7b2b1b83a5c5ad96c815348e28ed7da68e2e3687004faa906251", + dockerfile, + ) + self.assertIn("ARG EROFS_UTILS_VERSION=1.5-1", dockerfile) + self.assertIn("erofs-utils=${EROFS_UTILS_VERSION}", dockerfile) + + def test_azurelinux_initramfs_uses_locked_busybox_applets(self): + dockerfile = (BuildConstants.REPO_ROOT / "docker" / "Dockerfile").read_text( + encoding="utf-8" + ) + base_stage = dockerfile.split("FROM base AS kernel", 1)[0] + self.assertNotIn("busybox", base_stage) + azure_stage = dockerfile.split("FROM base AS azurelinux-initramfs", 1)[1] + self.assertIn("ln -sf ../sbin/busybox /rootfs/usr/bin/busybox", azure_stage) + applets = ( + azure_stage.split("for utility in ", 1)[1] + .split("; do", 1)[0] + .replace("\\\n", " ") + .split() + ) + for applet in ("sh", "arp", "nc", "wget", "mdev", "ifconfig", "route"): + self.assertIn(applet, applets) + # util-linux provides these; the BusyBox applets lack required options. + self.assertNotIn("setpriv", applets) + self.assertNotIn("unshare", applets) + launcher = ( + BuildConstants.REPO_ROOT + / "guest" + / "common" + / "nvx-container-launch-azurelinux" + ).read_text(encoding="utf-8") + self.assertIn("exec unshare --mount --pid --uts --fork --kill-child", launcher) + self.assertIn( + "busybox", [package["name"] for package in azurelinux.load_package_lock()] + ) + + def test_azurelinux_initramfs_gives_nobody_a_home(self): + dockerfile = (BuildConstants.REPO_ROOT / "docker" / "Dockerfile").read_text( + encoding="utf-8" + ) + azure_stage = dockerfile.split("FROM base AS azurelinux-initramfs", 1)[1] + self.assertIn( + "sed -i 's#^\\(nobody:[^:]*:65534:65534:[^:]*:\\)/dev/null:" + "#\\1/nonexistent:#'", + azure_stage, + ) + self.assertIn( + "grep -q '^nobody:[^:]*:65534:65534:[^:]*:/nonexistent:' " + "/rootfs/etc/passwd", + azure_stage, + ) + self.assertIn("install -d -m 0755 /rootfs/nonexistent", azure_stage) + + def test_azurelinux_rootfs_installs_only_locked_rpms(self): + dockerfile = (BuildConstants.REPO_ROOT / "docker" / "Dockerfile").read_text( + encoding="utf-8" + ) + self.assertNotIn("tdnf install", dockerfile) + packages_stage = dockerfile.split("FROM base AS azurelinux-packages", 1)[1] + self.assertIn( + "from nvx_tools.azurelinux import download_packages", + packages_stage.split("\nFROM ", 1)[0], + ) + rootfs_stage = dockerfile.split( + "FROM ${AZURELINUX_IMAGE} AS azurelinux-rootfs", 1 + )[1].split("\nFROM ", 1)[0] + self.assertIn( + "COPY --from=azurelinux-packages /out/rpms/ /tmp/azurelinux-rpms/", + rootfs_stage, + ) + checksig = rootfs_stage.index("rpm --checksig /tmp/azurelinux-rpms/*.rpm") + install = rootfs_stage.index("rpm --upgrade --verbose --hash") + self.assertLess(checksig, install) + + def test_azurelinux_manifest_uses_package_metadata(self): + dockerfile = (BuildConstants.REPO_ROOT / "docker" / "Dockerfile").read_text( + encoding="utf-8" + ) + self.assertIn("rpm -qa --queryformat", dockerfile) + self.assertIn("if name == 'gpg-pubkey':", dockerfile) + self.assertNotIn("'type': 'deb'", dockerfile) + self.assertIn("'packages': json.loads", dockerfile) + + def test_azurelinux_initramfs_packs_normalized_rootfs(self): + dockerfile = (BuildConstants.REPO_ROOT / "docker" / "Dockerfile").read_text( + encoding="utf-8" + ) + azure_stage = dockerfile.split("FROM base AS azurelinux-initramfs", 1)[1] + cleanup = azure_stage.index( + "rm -rf /rootfs/usr/lib/sysimage/tdnf /rootfs/var/cache/tdnf \\\n" + " /rootfs/var/cache/ldconfig /rootfs/var/lib/rpm" + ) + normalize = azure_stage.index( + "find . -exec touch --no-dereference " + f"--date=@{InitramfsBuildConstants.TIMESTAMP} {{}} +" + ) + pack = azure_stage.index("find . -print0 | LC_ALL=C sort -z | cpio") + self.assertLess(cleanup, normalize) + self.assertLess(normalize, pack) + + def test_azurelinux_manifest_records_shared_input_digest(self): + dockerfile = (BuildConstants.REPO_ROOT / "docker" / "Dockerfile").read_text( + encoding="utf-8" + ) + azure_stage = dockerfile.split("FROM base AS azurelinux-initramfs", 1)[1] + self.assertIn("from nvx_tools.azurelinux import input_sha256", azure_stage) + self.assertIn( + "'input_sha256': input_sha256(\n" + " image=os.environ['AZURELINUX_IMAGE'],\n" + " version=os.environ['AZURELINUX_VERSION'],\n" + " ),", + azure_stage, + ) + + def test_azurelinux_input_digest_tracks_guest_sources_and_dockerfile(self): + checkout_digest = azurelinux.input_sha256() + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + for source in azurelinux.input_files(): + destination = root / source.relative_to(BuildConstants.REPO_ROOT) + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(source, destination) + with patch.object(BuildConstants, "REPO_ROOT", root): + baseline = azurelinux.input_sha256() + self.assertEqual(baseline, checkout_digest) + self.assertNotEqual( + azurelinux.input_sha256(image="example.invalid/other@sha256:0"), + baseline, + ) + self.assertNotEqual(azurelinux.input_sha256(version="3.1"), baseline) + for relative in ( + "guest/common/init", + "docker/Dockerfile", + "azurelinux/packages.lock.json", + "scripts/nvx_tools/azurelinux.py", + "scripts/nvx_tools/build_constants.py", + "scripts/nvx_tools/common.py", + ): + with self.subTest(path=relative): + path = root / relative + original = path.read_bytes() + path.write_bytes(original + b"\n# changed\n") + self.assertNotEqual(azurelinux.input_sha256(), baseline) + path.write_bytes(original) + self.assertEqual(azurelinux.input_sha256(), baseline) + unrelated = root / "guest" / "ubuntu" / "nvx-bashrc" + unrelated.parent.mkdir(parents=True, exist_ok=True) + unrelated.write_text("changed\n", encoding="utf-8") + self.assertEqual(azurelinux.input_sha256(), baseline) + + def test_guest_init_runs_virtfs_helper_from_installed_path(self): + installed: list[Path] = [] + + def install(_source: Path, destination: Path) -> dict[str, str]: + installed.append(destination) + return {} + + root = Path("rootfs") + with ( + patch.object(build, "_install", side_effect=install), + patch.object(build, "_build_static_helper", return_value={}), + patch.object(build, "_build_device_io_helper", return_value={}), + ): + build._install_guest_files( + build_config.InitramfsBuildConfig(work=Path("work")), + root, + guests.ALPINE_GUEST, + ) + self.assertIn(root / "sbin" / "nvx-hostmount", installed) + + dockerfile = (BuildConstants.REPO_ROOT / "docker" / "Dockerfile").read_text( + encoding="utf-8" + ) + azure_stage = dockerfile.split("FROM base AS azurelinux-initramfs", 1)[1] + azure_sbin_scripts = azure_stage.split( + "install -m 0755 /repo/guest/common/init /rootfs/init\n", 1 + )[1].split("/rootfs/sbin/", 1)[0] + self.assertIn("/repo/guest/common/nvx-hostmount", azure_sbin_scripts) + init_script = ( + BuildConstants.REPO_ROOT / "guest" / "common" / "init" + ).read_text(encoding="utf-8") + self.assertIn( + '/sbin/nvx-hostmount || fatal "virtfs: failed to mount live host directory"', + init_script, + ) + self.assertNotIn("/usr/sbin/nvx-hostmount", init_script) + + def test_openvmm_ci_downloads_guest_artifacts(self): + workflow = ( + BuildConstants.REPO_ROOT / ".github" / "workflows" / "ci.yml" + ).read_text(encoding="utf-8") + openvmm_tests = _workflow_job(workflow, "openvmm-vmm-tests") + self.assertIn("needs: [artifacts, openvmm-changes]", openvmm_tests) + self.assertIn("needs.artifacts.result == 'success'", openvmm_tests) + self.assertIn("- name: Download guest artifacts", openvmm_tests) + self.assertIn("name: guest-artifacts", openvmm_tests) + self.assertIn("path: build", openvmm_tests) + self.assertIn("openvmm-vmm-tests", ci.REQUIRED_CI_OPENVMM_ARTIFACT_TEST_JOBS) + + def test_apk_add_uses_host_ca_bundle_without_overriding_configuration(self): + root = Path("root") + with ( + patch.object(build.ssl, "get_default_verify_paths") as verify_paths, + patch.object(build, "run_checked") as run, + ): + verify_paths.return_value.cafile = "/etc/host-ca.pem" + with patch.dict(os.environ, {}, clear=True): + build._apk_add(root, "example") + + environment = run.call_args.kwargs["env"] + self.assertEqual(environment["SSL_CERT_FILE"], "/etc/host-ca.pem") + self.assertEqual( + environment["LD_LIBRARY_PATH"], + f"{root / 'lib'}:{root / 'usr' / 'lib'}", + ) + + with patch.dict( + os.environ, + {"SSL_CERT_FILE": "/etc/configured-ca.pem"}, + clear=True, + ): + build._apk_add(root, "example") + + environment = run.call_args.kwargs["env"] + self.assertEqual(environment["SSL_CERT_FILE"], "/etc/configured-ca.pem") + + def test_device_io_helper_is_static_with_nonexecutable_stack(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + work = root / "work" + destination = root / "root" / "sbin" / "nvx-device-io" + work.mkdir() + destination.parent.mkdir(parents=True) + + def compile_helper( + command: list[str | os.PathLike[str]], **_kwargs: object + ) -> None: + output = Path(command[command.index("-o") + 1]) + output.write_bytes(b"static-elf") + + with ( + patch.object(build, "require_tool", return_value="cc"), + patch.object(build, "run_checked", side_effect=compile_helper) as run, + ): + provenance = build._build_device_io_helper(work, destination) + + command = run.call_args.args[0] + self.assertIn("-nostdlib", command) + self.assertIn("-static", command) + self.assertIn("-Wl,-z,noexecstack", command) + self.assertEqual(destination.read_bytes(), b"static-elf") + self.assertEqual( + provenance["binary_sha256"], common.sha256_file(destination) + ) + self.assertEqual(len(provenance["source_sha256"]), 64) + + def test_device_io_provenance_uses_shared_helper_source(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + checkout = root / "nvx" + source = checkout / "guest" / "common" / "nvx-device-io.c" + source.parent.mkdir(parents=True) + source.write_bytes(b"shared helper source") + initrd = root / "initramfs.cpio.gz" + manifest = initrd.with_name(f"{initrd.name}.packages.json") + manifest.write_text( + json.dumps( + { + "helpers": { + "nvx-device-io": { + "source_sha256": common.sha256_file(source), + "binary_sha256": "1" * 64, + } + } + } + ), + encoding="utf-8", + ) + + provenance = benchmark._device_io_helper_provenance( + argparse.Namespace(nvx_dir=checkout), + initrd, + ) + + self.assertEqual(provenance["source"], str(source.resolve())) + self.assertEqual( + provenance["source_sha256"], + common.sha256_file(source), + ) + + def test_sandbox_kernel_config_requires_every_feature(self): + with tempfile.TemporaryDirectory() as temporary: + config = Path(temporary) / ".config" + config.write_text( + "\n".join(KernelBuildConstants.REQUIRED_SANDBOX_CONFIG) + "\n", + encoding="utf-8", + ) + build._assert_sandbox_kernel_config(config) + + for missing in KernelBuildConstants.REQUIRED_SANDBOX_CONFIG: + with self.subTest(missing=missing): + config.write_text( + "\n".join( + setting + for setting in KernelBuildConstants.REQUIRED_SANDBOX_CONFIG + if setting != missing + ) + + "\n", + encoding="utf-8", + ) + with self.assertRaisesRegex(common.ScriptError, missing): + build._assert_sandbox_kernel_config(config) + + def test_network_kernel_config_requires_every_feature(self): + with tempfile.TemporaryDirectory() as temporary: + config = Path(temporary) / ".config" + config.write_text( + "\n".join(KernelBuildConstants.REQUIRED_NETWORK_CONFIG) + "\n", + encoding="utf-8", + ) + build._assert_network_kernel_config(config) + + for missing in KernelBuildConstants.REQUIRED_NETWORK_CONFIG: + with self.subTest(missing=missing): + config.write_text( + "\n".join( + setting + for setting in KernelBuildConstants.REQUIRED_NETWORK_CONFIG + if setting != missing + ) + + "\n", + encoding="utf-8", + ) + with self.assertRaisesRegex( + common.ScriptError, "portable network profile" + ) as raised: + build._assert_network_kernel_config(config) + self.assertIn(missing, str(raised.exception)) + + # An enabled SIT device leaves the "is not set" line out. + config.write_text( + "\n".join( + "CONFIG_IPV6_SIT=y" + if setting == "# CONFIG_IPV6_SIT is not set" + else setting + for setting in KernelBuildConstants.REQUIRED_NETWORK_CONFIG + ) + + "\n", + encoding="utf-8", + ) + with self.assertRaisesRegex(common.ScriptError, "CONFIG_IPV6_SIT"): + build._assert_network_kernel_config(config) + + def test_checked_in_config_provides_the_portable_network_profile(self): + config = BuildConstants.REPO_ROOT / KernelBuildConstants.INPUT_CONFIG + build._assert_network_kernel_config(config) + + def test_checked_in_config_preserves_generic_sandbox_capabilities(self): + config = BuildConstants.REPO_ROOT / "kernel" / "config-microvm" + build._assert_sandbox_kernel_config(config) + configured = config.read_text(encoding="utf-8").splitlines() + for setting in ( + "CONFIG_SECCOMP_FILTER=y", + "CONFIG_UNIX=y", + "# CONFIG_OVERLAY_FS_REDIRECT_ALWAYS_FOLLOW is not set", + ): + self.assertIn(setting, configured) + + def test_shared_status_kernel_config_is_required(self): + with tempfile.TemporaryDirectory() as temporary: + config = Path(temporary) / ".config" + config.write_text( + "\n".join(KernelBuildConstants.REQUIRED_SHARED_STATUS_CONFIG) + "\n", + encoding="utf-8", + ) + build._assert_shared_status_kernel_config(config) + + config.write_text("", encoding="utf-8") + with self.assertRaisesRegex( + common.ScriptError, + KernelBuildConstants.REQUIRED_SHARED_STATUS_CONFIG[0], + ): + build._assert_shared_status_kernel_config(config) + + def test_checked_in_config_meets_time_abi_kernel_contract(self): + config = BuildConstants.REPO_ROOT / KernelBuildConstants.INPUT_CONFIG + build._assert_time_abi_kernel_config(config) + build._assert_hardening_kernel_config(config) + build._assert_watchdog_kernel_config(config, debug=False) + configured = config.read_text(encoding="utf-8").splitlines() + for removed in ( + "CONFIG_CPU_FREQ=y", + "CONFIG_X86_INTEL_PSTATE=y", + "CONFIG_SCHED_MC_PRIO=y", + "CONFIG_HYPERV=y", + "CONFIG_KVM_GUEST=y", + "CONFIG_PARAVIRT_CLOCK=y", + "CONFIG_HALTPOLL_CPUIDLE=y", + "CONFIG_ARCH_CPUIDLE_HALTPOLL=y", + ): + with self.subTest(removed=removed): + self.assertNotIn(removed, configured) + self.assertFalse( + any(line.startswith("CONFIG_CPU_FREQ_") for line in configured) + ) + + def test_time_abi_kernel_config_requires_every_setting(self): + with tempfile.TemporaryDirectory() as temporary: + config = Path(temporary) / ".config" + for missing in KernelBuildConstants.REQUIRED_TIME_ABI_CONFIG: + with self.subTest(missing=missing): + config.write_text( + "\n".join( + setting + for setting in KernelBuildConstants.REQUIRED_TIME_ABI_CONFIG + if setting != missing + ) + + "\n", + encoding="utf-8", + ) + with self.assertRaisesRegex(common.ScriptError, missing): + build._assert_time_abi_kernel_config(config) + + def test_hardening_kernel_config_requires_every_setting(self): + with tempfile.TemporaryDirectory() as temporary: + config = Path(temporary) / ".config" + for missing in KernelBuildConstants.REQUIRED_HARDENING_CONFIG: + with self.subTest(missing=missing): + config.write_text( + "\n".join( + setting + for setting in KernelBuildConstants.REQUIRED_HARDENING_CONFIG + if setting != missing + ) + + "\n", + encoding="utf-8", + ) + with self.assertRaisesRegex(common.ScriptError, re.escape(missing)): + build._assert_hardening_kernel_config(config) + + def test_hardening_kernel_config_rejects_loadable_modules(self): + with tempfile.TemporaryDirectory() as temporary: + config = Path(temporary) / ".config" + config.write_text( + "\n".join( + (*KernelBuildConstants.REQUIRED_HARDENING_CONFIG, "CONFIG_FOO=m") + ) + + "\n", + encoding="utf-8", + ) + with self.assertRaisesRegex(common.ScriptError, "CONFIG_FOO=m"): + build._assert_hardening_kernel_config(config) + + def test_watchdog_kernel_config_separates_production_and_debug(self): + with tempfile.TemporaryDirectory() as temporary: + config = Path(temporary) / ".config" + config.write_text( + "\n".join(KernelBuildConstants.REQUIRED_DEBUG_CONFIG) + "\n", + encoding="utf-8", + ) + build._assert_watchdog_kernel_config(config, debug=True) + with self.assertRaisesRegex(common.ScriptError, "debug-only watchdogs"): + build._assert_watchdog_kernel_config(config, debug=False) + + config.write_text("CONFIG_DEBUG_KERNEL=y\n", encoding="utf-8") + build._assert_watchdog_kernel_config(config, debug=False) + with self.assertRaisesRegex( + common.ScriptError, "CONFIG_SOFTLOCKUP_DETECTOR=y" + ): + build._assert_watchdog_kernel_config(config, debug=True) + + def test_kernel_config_fragment_replaces_base_assignments(self): + base = "\n".join( + ( + "#", + "# Kernel hacking", + "#", + "# CONFIG_DEBUG_KERNEL is not set", + "CONFIG_RCU_CPU_STALL_TIMEOUT=21", + "CONFIG_UNRELATED=y", + ) + ) + fragment = "\n".join( + ( + "# Comment lines and blank lines are ignored.", + "", + "CONFIG_DEBUG_KERNEL=y", + "# CONFIG_DEBUG_MISC is not set", + "CONFIG_RCU_CPU_STALL_TIMEOUT=21", + ) + ) + self.assertEqual( + build.merge_kernel_config_fragment(base, fragment).splitlines(), + [ + "#", + "# Kernel hacking", + "#", + "CONFIG_UNRELATED=y", + "CONFIG_DEBUG_KERNEL=y", + "# CONFIG_DEBUG_MISC is not set", + "CONFIG_RCU_CPU_STALL_TIMEOUT=21", + ], + ) + for invalid, message in ( + ("DEBUG_KERNEL=y\n", "line 1 is invalid"), + ("# only a comment\n", "assigns no symbols"), + ( + "CONFIG_DEBUG_KERNEL=y\n# CONFIG_DEBUG_KERNEL is not set\n", + "more than once", + ), + ): + with self.subTest(fragment=invalid): + with self.assertRaisesRegex(common.ScriptError, message): + build.merge_kernel_config_fragment(base, invalid) + + def test_checked_in_debug_fragment_only_adds_watchdogs(self): + base = (BuildConstants.REPO_ROOT / KernelBuildConstants.INPUT_CONFIG).read_text( + encoding="utf-8" + ) + fragment = ( + BuildConstants.REPO_ROOT / KernelBuildConstants.DEBUG_CONFIG_FRAGMENT + ).read_text(encoding="utf-8") + merged = build.merge_kernel_config_fragment(base, fragment).splitlines() + for setting in KernelBuildConstants.REQUIRED_DEBUG_CONFIG: + with self.subTest(setting=setting): + self.assertIn(setting, merged) + for setting in ( + "# CONFIG_DEBUG_MISC is not set", + "# CONFIG_RCU_TRACE is not set", + "# CONFIG_X86_DEBUG_FPU is not set", + "CONFIG_RCU_CPU_STALL_TIMEOUT=21", + ): + with self.subTest(setting=setting): + self.assertIn(setting, merged) + self.assertNotIn("# CONFIG_DEBUG_KERNEL is not set", merged) + for setting in KernelBuildConstants.REQUIRED_TIME_ABI_CONFIG: + with self.subTest(time_abi=setting): + self.assertIn(setting, merged) + + def test_debug_kernel_build_applies_fragment_and_records_provenance(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + input_config = root / KernelBuildConstants.INPUT_CONFIG + input_config.parent.mkdir(parents=True) + input_config.write_text( + "\n".join( + ( + *KernelBuildConstants.REQUIRED_DIRECT_BOOT_CONFIG, + *KernelBuildConstants.REQUIRED_VIRTIO_CONSOLE_CONFIG, + *KernelBuildConstants.REQUIRED_SHARED_STATUS_CONFIG, + *KernelBuildConstants.REQUIRED_SANDBOX_CONFIG, + *KernelBuildConstants.REQUIRED_NETWORK_CONFIG, + *KernelBuildConstants.REQUIRED_TIME_ABI_CONFIG, + *KernelBuildConstants.REQUIRED_HARDENING_CONFIG, + "# CONFIG_DEBUG_KERNEL is not set", + ) + ) + + "\n", + encoding="utf-8", + ) + fragment = root / KernelBuildConstants.DEBUG_CONFIG_FRAGMENT + fragment.write_text( + "# debug fixture\n" + + "\n".join(KernelBuildConstants.REQUIRED_DEBUG_CONFIG) + + "\n", + encoding="utf-8", + ) + patch_path = root / KernelBuildConstants.PATCH_DIRECTORY / "example.patch" + patch_path.parent.mkdir() + patch_path.write_text("patch", encoding="utf-8") + source = root / "source" + source.mkdir() + output_directory = root / "output" + output_directory.mkdir() + production_provenance = ( + output_directory / KernelBuildConstants.PROVENANCE_NAME + ) + production_provenance.write_text("production", encoding="utf-8") + config = build_config.KernelBuildConfig( + work=root / "work-debug", + output=output_directory / KernelBuildConstants.DEBUG_BINARY_NAME, + debug=True, + ) + + with patch.object(BuildConstants, "REPO_ROOT", root): + source_fingerprint = build._kernel_source_fingerprint() + + def run_build(command: object, **_kwargs: object) -> None: + if isinstance(command, list) and command[-1] == "vmlinux": + (config.work / "vmlinux").write_bytes(b"debug kernel") + + with ( + patch.object(BuildConstants, "REPO_ROOT", root), + patch.object(build, "_require_linux"), + patch.object(build, "require_tool", return_value="tool"), + patch.object( + build, + "prepare_kernel_source", + return_value=(source, source_fingerprint), + ), + patch.object(build, "run_checked", side_effect=run_build), + ): + build.build_kernel(config) + + generated = ( + output_directory / KernelBuildConstants.DEBUG_CONFIG_NAME + ).read_text(encoding="utf-8") + self.assertIn("CONFIG_DEBUG_KERNEL=y", generated.splitlines()) + self.assertNotIn("# CONFIG_DEBUG_KERNEL is not set", generated) + provenance = json.loads( + ( + output_directory / KernelBuildConstants.DEBUG_PROVENANCE_NAME + ).read_text(encoding="utf-8") + ) + self.assertEqual( + provenance["debug_config_fragment"], + { + "path": KernelBuildConstants.DEBUG_CONFIG_FRAGMENT.as_posix(), + "sha256": common.sha256_file(fragment), + }, + ) + self.assertEqual( + provenance["input_config"]["sha256"], common.sha256_file(input_config) + ) + self.assertEqual( + provenance["kernel_sha256"], + hashlib.sha256(b"debug kernel").hexdigest(), + ) + self.assertEqual( + production_provenance.read_text(encoding="utf-8"), "production" + ) + self.assertIn( + "debug_config_fragment_sha256", + (config.work / KernelBuildConstants.BUILD_STAMP_NAME).read_text( + encoding="utf-8" + ), + ) + self.assertEqual( + config.provenance_name, KernelBuildConstants.DEBUG_PROVENANCE_NAME + ) + self.assertEqual( + build_config.KernelBuildConfig().provenance_name, + KernelBuildConstants.PROVENANCE_NAME, + ) + self.assertNotIn("debug_config_fragment", build.kernel_provenance_inputs()) + + def test_guest_build_adds_debug_kernel_only_when_requested(self): + with ( + patch.object(build, "build_kernel") as kernel, + patch.object(build, "build_initramfs"), + patch.object(build, "build_docker_debug_kernel") as docker_debug, + ): + native = build_config.BuildConfig(native_guest=True, debug_kernel=True) + build.build_guest(native) + self.assertEqual( + kernel.call_args_list, + [ + call(native.kernel), + call(build_config.KernelBuildConfig.debug_variant()), + ], + ) + docker_debug.assert_not_called() + + with ( + patch.object(build, "build_docker_artifacts") as docker, + patch.object(build, "build_docker_debug_kernel") as docker_debug, + ): + portable = build_config.BuildConfig(debug_kernel=True) + build.build_guest(portable) + docker.assert_called_once_with(portable.docker, "alpine") + docker_debug.assert_called_once_with(portable.docker) + + with ( + patch.object(build, "build_docker_artifacts"), + patch.object(build, "build_docker_debug_kernel") as docker_debug, + ): + build.build_guest(build_config.BuildConfig()) + docker_debug.assert_not_called() + + debug = build_config.KernelBuildConfig.debug_variant() + self.assertTrue(debug.debug) + self.assertEqual(debug.output.name, KernelBuildConstants.DEBUG_BINARY_NAME) + self.assertEqual( + debug.work.name, KernelBuildConstants.DEBUG_WORK_DIRECTORY_NAME + ) + + def test_guest_files_build_time_probe_with_its_own_flags(self): + source = BuildConstants.REPO_ROOT / "guest" / "common" / "nvx-time-probe.c" + header = source.read_text(encoding="utf-8").splitlines()[:2] + self.assertEqual( + header, + [ + "// Copyright (c) Microsoft Corporation.", + "// Licensed under the MIT License.", + ], + ) + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) / "root" + work = Path(temporary) / "work" + for directory in (root / "sbin", root / "etc", work): + directory.mkdir(parents=True) + commands: list[list[str]] = [] + + def compile_helper( + command: list[str | os.PathLike[str]], **_kwargs: object + ) -> None: + commands.append([os.fspath(part) for part in command]) + output = Path(command[command.index("-o") + 1]) + output.write_bytes(f"static-elf {output.name}".encode()) + + def find_tool(name: str) -> str: + return name + + with ( + patch.object(build, "require_tool", side_effect=find_tool), + patch.object(build, "run_checked", side_effect=compile_helper), + ): + helpers = build._install_guest_files( + build_config.InitramfsBuildConfig(work=work), + root, + guests.guest_descriptor("alpine"), + ) + + probe = InitramfsBuildConstants.TIME_PROBE_NAME + flags_by_source = { + Path(command[-1]).stem: command[1 : command.index("-o")] + for command in commands + } + compiler_by_source = { + Path(command[-1]).stem: command[0] for command in commands + } + self.assertEqual( + flags_by_source[probe], list(InitramfsBuildConstants.TIME_PROBE_CFLAGS) + ) + self.assertIn("-pthread", flags_by_source[probe]) + musl = InitramfsBuildConstants.MUSL_COMPILER + self.assertEqual(compiler_by_source[probe], musl) + for name in InitramfsBuildConstants.STATIC_HELPERS: + with self.subTest(helper=name): + self.assertEqual( + flags_by_source[name], + list(InitramfsBuildConstants.STATIC_HELPER_CFLAGS), + ) + self.assertEqual(compiler_by_source[name], "cc") + for name in InitramfsBuildConstants.MUSL_STATIC_HELPERS: + with self.subTest(helper=name): + self.assertEqual( + flags_by_source[name], + list(InitramfsBuildConstants.STATIC_HELPER_CFLAGS), + ) + self.assertEqual(compiler_by_source[name], musl) + dockerfile = (BuildConstants.REPO_ROOT / "docker" / "Dockerfile").read_text( + encoding="utf-8" + ) + self.assertIn(" musl-tools \\\n", dockerfile) + self.assertEqual(helpers[probe]["source_sha256"], common.sha256_file(source)) + self.assertEqual( + helpers[probe]["binary_sha256"], + hashlib.sha256(f"static-elf {probe}".encode()).hexdigest(), + ) + + def test_docker_debug_kernel_target_exports_debug_artifacts(self): + dockerfile = (BuildConstants.REPO_ROOT / "docker" / "Dockerfile").read_text( + encoding="utf-8" + ) + self.assertIn( + "FROM base AS kernel-debug\n" + "RUN python3 scripts/nvx.py build-kernel --debug\n", + dockerfile, + ) + self.assertIn( + f"FROM scratch AS {DockerBuildConstants.DEBUG_KERNEL_TARGET}", dockerfile + ) + for name in DockerBuildConstants.DEBUG_KERNEL_ARTIFACT_NAMES: + with self.subTest(artifact=name): + self.assertIn( + f"COPY --from=kernel-debug /repo/build/{name} /{name}", dockerfile + ) + + with tempfile.TemporaryDirectory() as temporary: + destination = Path(temporary) + + def export(_command: object, **_kwargs: object) -> None: + for name in DockerBuildConstants.DEBUG_KERNEL_ARTIFACT_NAMES: + (destination / name).write_bytes(b"artifact") + + config = build_config.DockerBuildConfig(artifact_destination=destination) + with ( + patch.object(build, "require_tool", return_value="docker"), + patch.object(build, "run_checked", side_effect=export) as run, + ): + build.build_docker_debug_kernel(config) + command = run.call_args.args[0] + self.assertEqual( + command[command.index("--target") + 1], + DockerBuildConstants.DEBUG_KERNEL_TARGET, + ) + + for name in DockerBuildConstants.DEBUG_KERNEL_ARTIFACT_NAMES: + (destination / name).unlink() + with ( + patch.object(build, "require_tool", return_value="docker"), + patch.object(build, "run_checked"), + self.assertRaisesRegex(common.ScriptError, "did not produce"), + ): + build.build_docker_debug_kernel(config) + + +def _posix_shell() -> str | None: + shell = shutil.which("sh") + if shell is None: + git = shutil.which("git") + git_shell = Path(git).parent.parent / "bin" / "sh.exe" if git else None + if git_shell is not None and git_shell.is_file(): + shell = str(git_shell) + return shell + + +def _shell_function(source: str, name: str) -> str: + start = source.index(f"\n{name}() {{\n") + 1 + return source[start : source.index("\n}\n", start) + 3] + + +class SandboxShareAgentTests(unittest.TestCase): + AGENT = Path(__file__).parents[1] / "guest" / "common" / "nvx-init-agent" + + def setUp(self): + shell = _posix_shell() + if shell is None: + self.skipTest("POSIX shell is unavailable") + self.shell = shell + self.source = self.AGENT.read_text(encoding="utf-8") + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) + self.rootfs = self.root / "rootfs" + self.rootfs.mkdir() + self.runtime = (self.root / "run").as_posix() + + SHARE_FUNCTIONS = ( + "validate_share_target", + "make_share_mount_point", + "record_live_share", + "claim_share_target", + "mount_live_share", + "parse_share_child", + "bind_live_share", + "mount_live_shares", + ) + + def _aggregate(self, *children: str, mode: str = "rw") -> str: + """Returns the bootstrap of an aggregate whose children the + CHILD,TARGET,MODE specifications `children` place.""" + tokens = [ + f"virtfs_dir={self.runtime}/shares", + "virtfs_tag=microvm", + f"virtfs_mode={mode}", + "virtfs_aggregate=1", + *(f"nvx_share={child}" for child in children), + ] + return " ".join(tokens) + + def _run(self, cmdline: str) -> tuple[subprocess.CompletedProcess[str], str]: + mount_log = self.root / "mount.log" + mount_log.unlink(missing_ok=True) + functions = "".join( + _shell_function(self.source, name) for name in self.SHARE_FUNCTIONS + ) + script = ( + "set -eu\n" + "rootfs=$1\ncmdline=$2\nmount_log=$3\nruntime=$4\nshare_mountpoints=\n" + 'fatal() { echo "FATAL: $*" >&2; exit 125; }\n' + 'mount() { printf "%s\\n" "$*" >>"$mount_log"; }\n' + f"{functions}" + "mount_live_shares\n" + "echo \"mountpoints=$(printf '%s' \"$share_mountpoints\" | tr '\\n' ' ')\"\n" + ) + result = subprocess.run( + [ + self.shell, + "-s", + "--", + self.rootfs.as_posix(), + cmdline, + mount_log.as_posix(), + self.runtime, + ], + input=script, + text=True, + capture_output=True, + check=False, + ) + log = mount_log.read_text(encoding="utf-8") if mount_log.exists() else "" + return result, log + + def _run_teardown( + self, script: str, *, shares: tuple[str, ...], failing: str = "" + ) -> tuple[subprocess.CompletedProcess[str], list[str]]: + runtime = self.root / "run" + runtime.mkdir(exist_ok=True) + for name in ("container.pid", "workload-machine-id"): + (runtime / name).write_text("x\n", encoding="utf-8") + log = self.root / "teardown.log" + log.unlink(missing_ok=True) + functions = "".join( + _shell_function(self.source, name) + for name in ("unmount_live_shares", "fatal", "teardown") + ).replace("/sbin/nvx-exit", "nvx_exit") + # The agent records mounted shares one per line, most recent first. + # Windows drops a newline from a process argument, so pass a separator. + mountpoints = "|".join( + f"{self.rootfs.as_posix()}{target}" for target in reversed(shares) + ) + result = subprocess.run( + [ + self.shell, + "-s", + "--", + runtime.as_posix(), + log.as_posix(), + mountpoints, + failing, + ], + input=( + "set -eu\n" + "runtime=$1\nlog=$2\nfailing=$4\n" + "share_mountpoints=$(printf '%s' \"$3\" | tr '|' '\\n')\n" + "rootfs=$runtime/rootfs\nlayers=$runtime/layers\n" + "scratch=$runtime/scratch\n" + 'umount() { printf "umount %s\\n" "$1" >>"$log"; ' + '[ "$1" != "$failing" ]; }\n' + 'mountpoint() { [ "$2" = "$layers/distro" ]; }\n' + 'nvx_exit() { printf "exit %s\\n" "$1" >>"$log"; exit "$1"; }\n' + f"{functions}{script}\n" + ), + text=True, + capture_output=True, + check=False, + ) + lines = log.read_text(encoding="utf-8").splitlines() if log.exists() else [] + return result, lines + + def test_agent_mounts_share_after_lifecycle_checks_for_both_lifecycles(self): + mount_call = self.source.index("\nmount_live_shares\n") + self.assertLess( + self.source.index('fatal "configured workload home is unavailable"'), + mount_call, + ) + self.assertLess( + self.source.index('fatal "unsupported workload lifecycle'), mount_call + ) + machine_id = self.source.index('>"$runtime/workload-machine-id"') + managed_agent = self.source.index("\n /sbin/nvx-managed-agent \\\n") + self.assertLess(mount_call, machine_id) + self.assertLess(machine_id, managed_agent) + self.assertLess(managed_agent, self.source.index("/sbin/nvx-container-launch")) + self.assertNotIn("exec /sbin/nvx-managed-agent", self.source) + self.assertEqual(self.source.count('\nteardown "$status"\n'), 1) + self.assertEqual(self.source.count('\n teardown "$status"\n'), 1) + self.assertLess(managed_agent, self.source.index('\n teardown "$status"\n')) + + def test_agent_teardown_unmounts_share_before_overlay_layers_and_scratch(self): + result, log = self._run_teardown("teardown 7", shares=("/workspace",)) + runtime = (self.root / "run").as_posix() + self.assertEqual(result.returncode, 7, result.stderr) + self.assertEqual( + log, + [ + f"umount {self.rootfs.as_posix()}/workspace", + f"umount {runtime}/rootfs", + f"umount {runtime}/layers/distro", + f"umount {runtime}/scratch", + "exit 7", + ], + ) + self.assertIn("NVX-SANDBOX-EXIT: status=7", result.stdout) + self.assertFalse((self.root / "run" / "workload-machine-id").exists()) + self.assertFalse((self.root / "run" / "container.pid").exists()) + + result, log = self._run_teardown("teardown 0", shares=()) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(log[0], f"umount {runtime}/rootfs") + + def test_agent_teardown_unmounts_shares_in_reverse_mount_order(self): + rootfs = self.rootfs.as_posix() + runtime = (self.root / "run").as_posix() + result, log = self._run_teardown( + "teardown 0", shares=("/workspace", "/opt/hostedtoolcache") + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual( + log[:3], + [ + f"umount {rootfs}/opt/hostedtoolcache", + f"umount {rootfs}/workspace", + f"umount {runtime}/rootfs", + ], + ) + + # A share that fails to unmount does not stop the others. + result, log = self._run_teardown( + "teardown 0", + shares=("/workspace", "/opt/hostedtoolcache"), + failing=f"{rootfs}/opt/hostedtoolcache", + ) + self.assertEqual(result.returncode, 1, result.stdout) + self.assertIn("failed to unmount the live share", result.stderr) + self.assertEqual( + log[:2], + [f"umount {rootfs}/opt/hostedtoolcache", f"umount {rootfs}/workspace"], + ) + self.assertEqual(log[-1], "exit 1") + + def test_agent_teardown_reports_share_unmount_failure(self): + share = f"{self.rootfs.as_posix()}/workspace" + result, log = self._run_teardown( + "teardown 0", shares=("/workspace",), failing=share + ) + self.assertEqual(result.returncode, 1, result.stdout) + self.assertIn("failed to unmount the live share", result.stderr) + self.assertEqual(log[0], f"umount {share}") + self.assertEqual(log[-1], "exit 1") + self.assertIn("NVX-SANDBOX-EXIT: status=1", result.stdout) + + def test_agent_fatal_unmounts_share_before_power_off(self): + result, log = self._run_teardown( + 'fatal "synthetic failure"', shares=("/workspace",) + ) + self.assertEqual(result.returncode, 125, result.stdout) + self.assertIn("NVX-SANDBOX-ERROR: synthetic failure", result.stderr) + self.assertEqual( + log, [f"umount {self.rootfs.as_posix()}/workspace", "exit 125"] + ) + + result, log = self._run_teardown('fatal "early failure"', shares=()) + self.assertEqual(result.returncode, 125, result.stdout) + self.assertEqual(log, ["exit 125"]) + + rootfs = self.rootfs.as_posix() + result, log = self._run_teardown( + 'fatal "late failure"', shares=("/workspace", "/opt/hostedtoolcache") + ) + self.assertEqual(result.returncode, 125, result.stdout) + self.assertEqual( + log, + [ + f"umount {rootfs}/opt/hostedtoolcache", + f"umount {rootfs}/workspace", + "exit 125", + ], + ) + + def test_agent_skips_share_without_bootstrap_tokens(self): + result, log = self._run("console=hvc0 nvx_sandbox=1") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("mountpoints=\n", result.stdout) + self.assertEqual(log, "") + + def test_agent_mounts_share_inside_container_rootfs(self): + (self.rootfs / "opt").mkdir() + for mode in ("rw", "ro"): + with self.subTest(mode=mode): + result, log = self._run( + "nvx_sandbox=1 virtfs_dir=/opt/hostedtoolcache " + f"virtfs_tag=microvm virtfs_mode={mode}" + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertTrue((self.rootfs / "opt" / "hostedtoolcache").is_dir()) + self.assertIn( + f"NVX-SANDBOX-SHARE: target=/opt/hostedtoolcache mode={mode}", + result.stdout, + ) + self.assertEqual( + log.split(), + [ + "-t", + "virtiofs", + "-o", + f"{mode},nosuid,nodev", + "microvm", + f"{self.rootfs.as_posix()}/opt/hostedtoolcache", + ], + ) + self.assertIn( + f"mountpoints={self.rootfs.as_posix()}/opt/hostedtoolcache\n", + result.stdout, + ) + + def test_agent_binds_every_child_of_an_aggregate(self): + rootfs = self.rootfs.as_posix() + result, log = self._run( + "nvx_sandbox=1 " + + self._aggregate("0,/workspace,rw", "1,/opt/hostedtoolcache,ro") + ) + self.assertEqual(result.returncode, 0, result.stderr) + # The aggregate stays outside the container rootfs; each child is bound + # inside it with its own mode. + self.assertEqual( + log.splitlines(), + [ + f"-t virtiofs -o rw,nosuid,nodev microvm {self.runtime}/shares", + f"-o bind {self.runtime}/shares/0 {rootfs}/workspace", + f"-o remount,bind,rw,nosuid,nodev {rootfs}/workspace", + f"-o bind {self.runtime}/shares/1 {rootfs}/opt/hostedtoolcache", + f"-o remount,bind,ro,nosuid,nodev {rootfs}/opt/hostedtoolcache", + ], + ) + self.assertIn( + "NVX-SANDBOX-SHARE: target=/workspace mode=rw child=0\n" + "NVX-SANDBOX-SHARE: target=/opt/hostedtoolcache mode=ro child=1\n", + result.stdout, + ) + # Teardown unmounts the most recent bind first, and the aggregate last. + self.assertIn( + f"mountpoints={rootfs}/opt/hostedtoolcache {rootfs}/workspace " + f"{self.runtime}/shares\n", + result.stdout, + ) + + def test_agent_binds_children_by_the_names_that_nvx_gives_them(self): + rootfs = self.rootfs.as_posix() + workspace = sandbox.aggregate_child_name(0, "/workspace") + tools = sandbox.aggregate_child_name(1, "/opt/tools") + result, log = self._run( + self._aggregate( + f"{workspace},/workspace,rw", f"{tools},/opt/tools,ro", "a.b_C-9,/x,ro" + ) + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn( + f"-o bind {self.runtime}/shares/{workspace} {rootfs}/workspace\n", log + ) + self.assertIn( + f"-o bind {self.runtime}/shares/{tools} {rootfs}/opt/tools\n", log + ) + self.assertIn(f"-o bind {self.runtime}/shares/a.b_C-9 {rootfs}/x\n", log) + self.assertIn( + f"NVX-SANDBOX-SHARE: target=/workspace mode=rw child={workspace}\n", + result.stdout, + ) + # A repeated name is refused, however the child is named. + result, log = self._run( + self._aggregate(f"{workspace},/workspace,rw", f"{workspace},/b,ro") + ) + self.assertEqual(result.returncode, 125, result.stdout) + self.assertIn("child is not unique", result.stderr) + self.assertEqual(log, "") + + def test_hostmount_accepts_only_child_names_inside_the_aggregate(self): + hostmount = self.AGENT.with_name("nvx-hostmount").read_text(encoding="utf-8") + script = ( + 'fail() { echo "FAIL: $*" >&2; exit 1; }\n' + f"{_shell_function(hostmount, 'parse_share')}" + 'parse_share "$1"\n' + 'printf "%s|%s|%s|%s\\n" "$share_child" "$share_target" "$share_mode" ' + '"$share_kind"\n' + ) + + def parse(share: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [self.shell, "-s", "--", share], + input=script, + text=True, + capture_output=True, + check=False, + ) + + name = sandbox.aggregate_child_name(0, "/workspace") + for share, expected in ( + (f"{name},/workspace,rw", f"{name}|/workspace|rw|directory\n"), + ("0,/opt/tools,ro", "0|/opt/tools|ro|directory\n"), + ("a.b_C-9,/x,ro", "a.b_C-9|/x|ro|directory\n"), + # A file child is bound at a file. + ("1,/config/settings.json,ro,file", "1|/config/settings.json|ro|file\n"), + ("1,/results/output.txt,rw,file", "1|/results/output.txt|rw|file\n"), + ): + with self.subTest(share=share): + result = parse(share) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout, expected) + for share in ( + ",/workspace,rw", + ".,/workspace,rw", + "..,/workspace,rw", + "a/b,/workspace,rw", + "a:b,/workspace,rw", + "a*,/workspace,rw", + "0,/workspace,file", + "0,/workspace,rw,dir", + ): + with self.subTest(share=share): + result = parse(share) + self.assertEqual(result.returncode, 1, result.stdout) + self.assertIn("invalid live share", result.stderr) + + def test_hostmount_makes_file_mount_points_without_following_links(self): + hostmount = self.AGENT.with_name("nvx-hostmount").read_text(encoding="utf-8") + script = ( + f"{_shell_function(hostmount, 'make_file_mount_point')}" + 'make_file_mount_point "$1"\n' + ) + + def make(target: Path) -> int: + return subprocess.run( + [self.shell, "-s", "--", target.as_posix()], + input=script, + text=True, + capture_output=True, + check=False, + ).returncode + + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + # A missing file is created with its directories, and an existing + # one is kept as it is. + created = root / "config" / "settings.json" + self.assertEqual(make(created), 0) + self.assertEqual(created.read_bytes(), b"") + existing = root / "existing.json" + existing.write_bytes(b"{}") + self.assertEqual(make(existing), 0) + self.assertEqual(existing.read_bytes(), b"{}") + (root / "directory").mkdir() + self.assertEqual(make(root / "directory"), 1) + if os.name == "nt": + return + # A link is refused before anything is created through it, whether + # or not its target exists. + dangling = root / "dangling.json" + dangling.symlink_to(root / "elsewhere.json") + self.assertEqual(make(dangling), 1) + self.assertFalse((root / "elsewhere.json").exists()) + linked = root / "linked.json" + linked.symlink_to(existing) + self.assertEqual(make(linked), 1) + self.assertEqual(existing.read_bytes(), b"{}") + + def test_agent_binds_a_file_child_at_a_file(self): + rootfs = self.rootfs.as_posix() + (self.rootfs / "opt").mkdir() + (self.rootfs / "opt" / "existing.json").write_text("{}", encoding="utf-8") + result, log = self._run( + "nvx_sandbox=1 " + + self._aggregate( + "0,/workspace,rw", + "1,/etc/app/settings.json,ro,file", + "2,/opt/existing.json,rw,file", + ) + ) + self.assertEqual(result.returncode, 0, result.stderr) + # Each file child gets a file mount point, created inside the container + # rootfs or reused there, and is bound with its own mode. + settings = self.rootfs / "etc" / "app" / "settings.json" + self.assertTrue(settings.is_file()) + self.assertEqual(settings.read_text(encoding="utf-8"), "") + self.assertEqual( + (self.rootfs / "opt" / "existing.json").read_text(encoding="utf-8"), "{}" + ) + self.assertTrue((self.rootfs / "workspace").is_dir()) + self.assertEqual( + log.splitlines(), + [ + f"-t virtiofs -o rw,nosuid,nodev microvm {self.runtime}/shares", + f"-o bind {self.runtime}/shares/0 {rootfs}/workspace", + f"-o remount,bind,rw,nosuid,nodev {rootfs}/workspace", + f"-o bind {self.runtime}/shares/1 {rootfs}/etc/app/settings.json", + f"-o remount,bind,ro,nosuid,nodev {rootfs}/etc/app/settings.json", + f"-o bind {self.runtime}/shares/2 {rootfs}/opt/existing.json", + f"-o remount,bind,rw,nosuid,nodev {rootfs}/opt/existing.json", + ], + ) + self.assertIn( + f"mountpoints={rootfs}/opt/existing.json {rootfs}/etc/app/settings.json " + f"{rootfs}/workspace {self.runtime}/shares\n", + result.stdout, + ) + + # A directory is no file mount point, a file no directory one, and + # the runtime binds the machine ID itself. + (self.rootfs / "dir").mkdir() + (self.rootfs / "file").write_text("x", encoding="utf-8") + for child, message in ( + ("0,/dir,ro,file", "not a regular file"), + ("0,/file/settings.json,ro,file", "not a directory"), + ("0,/file,ro", "not a directory"), + ("0,/etc/machine-id,ro,file", "reserved"), + ("0,/etc/machine-id/child,ro", "reserved"), + ): + with self.subTest(child=child): + result, log = self._run(self._aggregate(child)) + self.assertEqual(result.returncode, 125, result.stdout) + self.assertIn(message, result.stderr) + self.assertEqual(log, "") + + def test_agent_rejects_repeated_children_and_overlapping_targets(self): + for children, message in ( + (("0,/workspace,rw", "0,/opt/tools,ro"), "child is not unique"), + (("0,/workspace,rw", "1,/workspace,ro"), "targets overlap"), + (("0,/workspace,rw", "1,/workspace/cache,ro"), "targets overlap"), + (("0,/opt/tools/node,ro", "1,/opt,rw"), "targets overlap"), + ): + with self.subTest(children=children): + result, log = self._run(self._aggregate(*children)) + self.assertEqual(result.returncode, 125, result.stdout) + self.assertIn(message, result.stderr) + # Every child is checked before anything is mounted. + self.assertEqual(log, "") + + # A shared prefix that is not a path component is not an overlap. + result, log = self._run(self._aggregate("0,/work,rw", "1,/workspace,ro")) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(len(log.splitlines()), 5) + + def test_agent_parses_every_share_before_mounting_any(self): + for cmdline in ( + "virtfs_dir=/workspace virtfs_tag=microvm virtfs_mode=rw " + "virtfs_dir=/opt/tools", + "virtfs_tag=microvm virtfs_mode=ro", + "virtfs_dir=/workspace virtfs_tag=microvm virtfs_mode=rw " + "virtfs_dir= virtfs_tag=microvm virtfs_mode=ro", + "virtfs_aggregate=1", + "nvx_share=0,/workspace,rw", + ): + with self.subTest(cmdline=cmdline): + result, log = self._run(cmdline) + self.assertEqual(result.returncode, 125, result.stdout) + self.assertIn("bootstrap is incomplete", result.stderr) + self.assertEqual(log, "") + + def test_agent_fatal_unmounts_mounted_shares(self): + mount_log = self.root / "mount.log" + functions = "".join( + _shell_function(self.source, name) + for name in ("unmount_live_shares", "fatal", *self.SHARE_FUNCTIONS) + ).replace("/sbin/nvx-exit", "nvx_exit") + result = subprocess.run( + [ + self.shell, + "-s", + "--", + self.rootfs.as_posix(), + self._aggregate("0,/workspace,rw", "1,/opt/tools,ro"), + mount_log.as_posix(), + self.runtime, + ], + input=( + "set -eu\n" + "rootfs=$1\ncmdline=$2\nmount_log=$3\nruntime=$4\n" + "share_mountpoints=\n" + 'mount() { printf "mount %s\\n" "$*" >>"$mount_log"; ' + 'case "$*" in *shares/1*) return 1 ;; esac; }\n' + 'umount() { printf "umount %s\\n" "$1" >>"$mount_log"; }\n' + 'nvx_exit() { printf "exit %s\\n" "$1" >>"$mount_log"; exit "$1"; }\n' + f"{functions}mount_live_shares\n" + ), + text=True, + capture_output=True, + check=False, + ) + self.assertEqual(result.returncode, 125, result.stdout) + self.assertIn("failed to bind live share 1", result.stderr) + rootfs = self.rootfs.as_posix() + self.assertEqual( + mount_log.read_text(encoding="utf-8").splitlines(), + [ + f"mount -t virtiofs -o rw,nosuid,nodev microvm {self.runtime}/shares", + f"mount -o bind {self.runtime}/shares/0 {rootfs}/workspace", + f"mount -o remount,bind,rw,nosuid,nodev {rootfs}/workspace", + f"mount -o bind {self.runtime}/shares/1 {rootfs}/opt/tools", + f"umount {rootfs}/workspace", + f"umount {self.runtime}/shares", + "exit 125", + ], + ) + + def test_agent_fails_closed_for_invalid_share_bootstrap(self): + (self.rootfs / "file").write_text("x", encoding="utf-8") + for cmdline, message in ( + ("virtfs_dir=/workspace", "incomplete"), + ("virtfs_dir=/workspace virtfs_tag=microvm", "incomplete"), + ("virtfs_dir=/workspace virtfs_tag=bad/tag virtfs_mode=rw", "tag"), + ("virtfs_dir=/workspace virtfs_tag=microvm virtfs_mode=rx", "mode"), + ("virtfs_dir=workspace virtfs_tag=microvm virtfs_mode=rw", "absolute"), + ("virtfs_dir=/ virtfs_tag=microvm virtfs_mode=rw", "canonical"), + ("virtfs_dir=/a//b virtfs_tag=microvm virtfs_mode=rw", "canonical"), + ("virtfs_dir=/a/ virtfs_tag=microvm virtfs_mode=rw", "canonical"), + ("virtfs_dir=/a/./b virtfs_tag=microvm virtfs_mode=rw", "canonical"), + ("virtfs_dir=/a/../b virtfs_tag=microvm virtfs_mode=rw", "canonical"), + ("virtfs_dir=/a=b virtfs_tag=microvm virtfs_mode=rw", "reserved char"), + ("virtfs_dir=/proc virtfs_tag=microvm virtfs_mode=rw", "reserved"), + ("virtfs_dir=/sys/fs virtfs_tag=microvm virtfs_mode=rw", "reserved"), + ("virtfs_dir=/dev/shm virtfs_tag=microvm virtfs_mode=rw", "reserved"), + ("virtfs_dir=/.nvx-agent virtfs_tag=microvm virtfs_mode=ro", "reserved"), + ("virtfs_dir=/etc virtfs_tag=microvm virtfs_mode=ro", "reserved"), + ("virtfs_dir=/file virtfs_tag=microvm virtfs_mode=ro", "not a directory"), + ( + "virtfs_dir=/workspace virtfs_tag=microvm virtfs_mode=rw " + "nvx_share=0,/workspace,rw", + "require an aggregate", + ), + ( + "virtfs_dir=/shares virtfs_tag=microvm virtfs_mode=rw " + "virtfs_aggregate=1 nvx_share=0,/workspace,rw", + "must be mounted at", + ), + (self._aggregate(), "has no children"), + (self._aggregate(",/workspace,rw"), "child is invalid"), + (self._aggregate(".,/workspace,rw"), "child is invalid"), + (self._aggregate("..,/workspace,rw"), "child is invalid"), + (self._aggregate("a/b,/workspace,rw"), "child is invalid"), + (self._aggregate("a:b,/workspace,rw"), "child is invalid"), + (self._aggregate("a*,/workspace,rw"), "child is invalid"), + (self._aggregate("0,/workspace"), "child is invalid"), + (self._aggregate("0,/workspace,rx"), "mode"), + (self._aggregate("0,workspace,rw"), "absolute"), + (self._aggregate("0,/a,b,rw"), "reserved char"), + (self._aggregate("0,/workspace,rw", "1,/proc,ro"), "reserved"), + (self._aggregate("0,/file,ro"), "not a directory"), + ): + with self.subTest(cmdline=cmdline): + result, log = self._run(cmdline) + self.assertEqual(result.returncode, 125, result.stdout) + self.assertIn("live share", result.stderr) + self.assertIn(message, result.stderr) + self.assertEqual(log, "") + + def test_agent_rejects_symbolic_link_in_share_target(self): + outside = self.root / "outside" + outside.mkdir() + try: + (self.rootfs / "workspace").symlink_to(outside, target_is_directory=True) + except OSError as error: + self.skipTest(f"symbolic links are unavailable: {error}") + result, log = self._run( + "virtfs_dir=/workspace/project virtfs_tag=microvm virtfs_mode=rw" + ) + self.assertEqual(result.returncode, 125, result.stdout) + self.assertIn("crosses a symbolic link", result.stderr) + self.assertEqual(log, "") + self.assertFalse((outside / "project").exists()) + + +@unittest.skipIf(os.name == "nt", "requires POSIX symbolic links") +class SandboxSmokeShareTests(unittest.TestCase): + SMOKE = Path(__file__).parents[1] / "guest" / "common" / "nvx-sandbox-smoke" + + def setUp(self): + shell = _posix_shell() + if shell is None: + self.skipTest("POSIX shell is unavailable") + self.shell = shell + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) + self.share = self.root / "share" + self.links = self.share / "nvx-links" + self.links.mkdir(parents=True) + (self.share / "nvx-host-marker").write_text("host-to-guest\n", encoding="utf-8") + # The guest resolves nvx-links/outside to this path through ../.. + self.outside = self.root / "nvx-outside-marker" + source = self.SMOKE.read_text(encoding="utf-8") + self.functions = "".join( + _shell_function(source, name) + for name in ( + "check_share", + "check_share_ownership", + "check_share_denied", + "expect_eperm", + "check_share_policy", + "expect_erofs", + "check_share_symlinks", + "check_shares_isolated", + ) + ).replace("/tmp/nvx-tool", '"$scratch/nvx-tool"') + + def _run(self, call: str) -> subprocess.CompletedProcess[str]: + scratch = self.root / "scratch" + scratch.mkdir(exist_ok=True) + return subprocess.run( + [self.shell, "-s", "--", self.share.as_posix(), scratch.as_posix()], + input=f"set -eu\nshare=$1\nscratch=$2\n{self.functions}{call}\n", + text=True, + capture_output=True, + check=False, + ) + + def test_rw_share_creates_links_with_exact_targets(self): + result = self._run('check_share "$share" rw') + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn( + f"NVX-UBUNTU-SANDBOX-SYMLINK-OK target={self.share.as_posix()}", + result.stdout, + ) + self.assertIn("NVX-UBUNTU-SANDBOX-SHARE-OK", result.stdout) + self.assertEqual( + {name: os.readlink(self.links / name) for name in os.listdir(self.links)}, + { + "nvx-tool": "../nvx-tool", + "absolute": "/nvx/absolute/target", + "outside": "../../nvx-outside-marker", + "denied": "../nvx-denied/secret", + "denied-directory": "../nvx-denied", + }, + ) + self.assertEqual( + (self.share / "nvx-guest-marker").read_text(encoding="utf-8"), + "guest-to-host\n", + ) + self.assertTrue((self.share / "nvx-guest-directory").is_dir()) + + def test_links_fail_when_they_expose_host_data(self): + (self.share / "nvx-guest-marker").write_text( + "guest-to-host\n", encoding="utf-8" + ) + self.outside.write_text("host-outside\n", encoding="utf-8") + result = self._run('check_share_symlinks "$share"') + self.assertNotEqual(result.returncode, 0, result.stdout) + self.assertNotIn("NVX-UBUNTU-SANDBOX-SYMLINK-OK", result.stdout) + + for path in self.links.iterdir(): + path.unlink() + self.outside.unlink() + denied = self.share / "nvx-denied" + denied.mkdir() + (denied / "secret").write_text("secret\n", encoding="utf-8") + result = self._run('check_share_symlinks "$share"') + self.assertNotEqual(result.returncode, 0, result.stdout) + self.assertNotIn("NVX-UBUNTU-SANDBOX-SYMLINK-OK", result.stdout) + + def test_ro_share_rejects_links(self): + geteuid = getattr(os, "geteuid", None) + if geteuid is not None and geteuid() == 0: + self.skipTest("root ignores directory write permissions") + self.share.chmod(0o555) + self.addCleanup(self.share.chmod, 0o755) + + result = self._run('check_share "$share" ro') + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("NVX-UBUNTU-SANDBOX-SHARE-OK", result.stdout) + self.assertFalse(os.path.lexists(self.share / "nvx-guest-link")) + + self.share.chmod(0o755) + result = self._run('check_share "$share" ro') + self.assertEqual(result.returncode, 1, result.stdout) + + def test_policy_share_requires_openvmm_to_refuse_writes(self): + (self.share / "nvx-writable").mkdir() + allowed = self.share / "nvx-denied" / "nvx-allowed" + allowed.mkdir(parents=True) + (allowed / "payload").write_text("payload\n", encoding="utf-8") + + # Only OpenVMM's access policy can make the share refuse writes + # outside nvx-writable, so a plain directory fails the checks. + result = self._run('check_share "$share" policy') + self.assertEqual(result.returncode, 1, result.stdout) + self.assertNotIn("NVX-UBUNTU-SANDBOX-POLICY-OK", result.stdout) + + # A refusal other than EROFS, such as a permission check, fails too. + geteuid = getattr(os, "geteuid", None) + if geteuid is not None and geteuid() == 0: + return + (self.share / "nvx-guest-marker").unlink(missing_ok=True) + self.share.chmod(0o555) + self.addCleanup(self.share.chmod, 0o755) + result = self._run('check_share "$share" policy') + self.assertEqual(result.returncode, 1, result.stdout) + self.assertIn("unexpected share error", result.stderr) + self.assertIn("Permission denied", result.stderr) + + def test_rw_share_link_cannot_write_into_ro_share(self): + geteuid = getattr(os, "geteuid", None) + if geteuid is not None and geteuid() == 0: + self.skipTest("root ignores file write permissions") + toolcache = self.root / "toolcache" + toolcache.mkdir() + marker = toolcache / "nvx-host-marker" + marker.write_text("host-to-guest\n", encoding="utf-8") + marker.chmod(0o444) + self.addCleanup(marker.chmod, 0o644) + + call = f'check_shares_isolated "$share" "{toolcache.as_posix()}"' + result = self._run(call) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn( + f"NVX-UBUNTU-SANDBOX-SHARES-ISOLATED-OK rw={self.share.as_posix()} " + f"ro={toolcache.as_posix()}", + result.stdout, + ) + self.assertFalse(os.path.lexists(self.share / "nvx-cross-share-link")) + self.assertEqual(marker.read_text(encoding="utf-8"), "host-to-guest\n") + + # A write that reaches the read-only share fails the check. + marker.chmod(0o644) + result = self._run(call) + self.assertEqual(result.returncode, 1, result.stdout) + self.assertNotIn("NVX-UBUNTU-SANDBOX-SHARES-ISOLATED-OK", result.stdout) + + def test_caller_share_owns_and_populates_its_own_directories(self): + if sys.platform != "linux": + self.skipTest("ownership checks use GNU stat and Linux identities") + result = self._run('check_share "$share" caller') + + self.assertEqual(result.returncode, 0, result.stderr) + share = self.share.as_posix() + self.assertIn( + f"NVX-UBUNTU-SANDBOX-OWNER-OK target={share} " + f"identity={os.getuid()}:{os.getgid()}", + result.stdout, + ) + self.assertIn(f"NVX-UBUNTU-SANDBOX-SYMLINK-OK target={share}", result.stdout) + self.assertIn( + f"NVX-UBUNTU-SANDBOX-SHARE-OK target={share} mode=caller", result.stdout + ) + links = self.share / "nvx-caller-links" + self.assertEqual( + {name: os.readlink(links / name) for name in os.listdir(links)}, + { + "nvx-tool": "../nvx-tool", + "absolute": "/nvx/absolute/target", + "outside": "../../nvx-outside-marker", + "denied": "../nvx-denied/secret", + "denied-directory": "../nvx-denied", + }, + ) + self.assertEqual(os.listdir(self.links), []) + renamed = self.share / "nvx-guest-directory" / "nested" / "renamed" + self.assertEqual(renamed.read_text(encoding="utf-8"), "nested\n") + + def test_eperm_share_requires_reads_to_fail_with_eperm_and_writes_to_fail(self): + result = self._run('check_share "$share" eperm') + self.assertNotEqual(result.returncode, 0, result.stdout) + self.assertNotIn("NVX-UBUNTU-SANDBOX-SHARE-OK", result.stdout) + + geteuid = getattr(os, "geteuid", None) + if geteuid is not None and geteuid() == 0: + self.skipTest("root ignores directory write permissions") + # Writes need only fail, so a read-only host directory stands in for + # the host's refusal. + self.share.chmod(0o555) + self.addCleanup(self.share.chmod, 0o755) + denied = "".join( + f'{command}() {{ echo "{command}: $1: Operation not permitted" >&2; ' + "return 1; }\n" + for command in ("cat", "ls") + ) + result = self._run(denied + 'check_share "$share" eperm') + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn( + f"NVX-UBUNTU-SANDBOX-SHARE-OK target={self.share.as_posix()} mode=eperm", + result.stdout, + ) + self.assertFalse((self.share / "nvx-guest-marker").exists()) + + other_error = denied.replace("Operation not permitted", "Permission denied") + result = self._run(other_error + 'check_share "$share" eperm') + self.assertNotEqual(result.returncode, 0, result.stdout) + self.assertIn("unexpected share error", result.stderr) + + self.share.chmod(0o755) + result = self._run(denied + 'check_share "$share" eperm') + self.assertNotEqual(result.returncode, 0, result.stdout) + self.assertNotIn("NVX-UBUNTU-SANDBOX-SHARE-OK", result.stdout) + + +_SANDBOX_MOUNTINFO = ( + "20 1 0:20 / / rw,relatime - overlay overlay rw," + "lowerdir=/run/nvx/layers/distro,upperdir=/run/nvx/scratch/upper," + "workdir=/run/nvx/scratch/work,metacopy=on,xino=on\n" + "21 20 0:21 / /.nvx-agent ro,nosuid,nodev,relatime - tmpfs tmpfs ro,mode=755\n" + "22 20 0:22 / /proc rw,nosuid,nodev,noexec,relatime - proc proc rw\n" + "23 20 0:23 / /sys ro,nosuid,nodev,noexec,relatime - sysfs sysfs ro\n" + "24 20 0:24 / /dev rw,nosuid,relatime - tmpfs tmpfs rw,mode=755\n" + "25 24 0:25 / /dev/pts rw,relatime - devpts devpts rw,mode=620,ptmxmode=666\n" + "26 24 0:26 / /dev/shm rw,nosuid,nodev,relatime - tmpfs tmpfs rw\n" +) +_SANDBOX_STATUS = ( + "Name:\tsh\nUid:\t65534\t65534\t65534\t65534\n" + "Gid:\t65534\t65534\t65534\t65534\nGroups:\t\nNoNewPrivs:\t1\n" + + "".join( + f"{name}:\t0000000000000000\n" + for name in ("CapInh", "CapPrm", "CapEff", "CapBnd", "CapAmb") + ) +) + + +@unittest.skipIf(os.name == "nt", "requires a POSIX shell and process tools") +class SandboxSmokeProfileTests(unittest.TestCase): + SMOKE = Path(__file__).parents[1] / "guest" / "common" / "nvx-sandbox-smoke" + + def setUp(self): + shell = _posix_shell() + if shell is None: + self.skipTest("POSIX shell is unavailable") + self.shell = shell + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) + self.source = self.SMOKE.read_text(encoding="utf-8") + + def _fixture(self, name: str, content: str) -> str: + path = self.root / name + path.write_text(content, encoding="utf-8") + return path.as_posix() + + def _run( + self, + names: tuple[str, ...], + call: str, + replacements: dict[str, str] | None = None, + ) -> subprocess.CompletedProcess[str]: + # The globals before fail() configure the checks. + script = self.source[: self.source.index("\nfail() {\n") + 1] + "".join( + _shell_function(self.source, name) for name in ("fail", *names) + ) + for old, new in (replacements or {}).items(): + script = script.replace(old, new) + return subprocess.run( + [self.shell, "-s"], + input=f"{script}{call}\n", + text=True, + capture_output=True, + check=False, + ) + + def test_status_fields_join_their_values(self): + status = self._fixture("status", _SANDBOX_STATUS) + result = self._run( + ("status_field",), + f"status_field {status} Uid\nstatus_field {status} Groups\n" + f"status_field {status} Missing", + ) + + self.assertEqual(result.returncode, 1, result.stderr) + self.assertEqual(result.stdout, "65534 65534 65534 65534\n\n") + self.assertIn(f"{status} has no Missing field", result.stderr) + + def test_identity_and_capability_checks_read_the_process_status(self): + identity = ( + 'id() { case "$1" in -u | -g) echo 65534 ;; esac; }\nHOME=/nonexistent\n' + ) + mutations = { + "valid": ("", ""), + "saved-uid": ( + "Uid:\t65534\t65534\t65534\t65534", + "Uid:\t65534\t65534\t0\t65534", + ), + "groups": ("Groups:\t", "Groups:\t27 "), + "bounding": ( + "CapBnd:\t0000000000000000", + "CapBnd:\t000001ffffffffff", + ), + "no-new-privs": ("NoNewPrivs:\t1", "NoNewPrivs:\t0"), + } + for mutation, (old, new) in mutations.items(): + with self.subTest(mutation=mutation): + status = self._fixture("status", _SANDBOX_STATUS.replace(old, new)) + result = self._run( + ("status_field", "check_identity", "check_capabilities"), + f"{identity}check_identity\ncheck_capabilities\necho checked", + {"/proc/self/status": status}, + ) + if mutation == "valid": + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout, "checked\n") + else: + self.assertEqual(result.returncode, 1, result.stdout) + self.assertIn("NVX-UBUNTU-SANDBOX-FAIL", result.stderr) + + def test_namespace_check_rejects_the_initial_namespaces(self): + def run( + links: dict[str, str], + *, + init_status: str = _SANDBOX_STATUS, + command_line: str = "console=hvc0 nvx_hostname=nvx-lifecycle", + hostname: str = "nvx-lifecycle", + ) -> subprocess.CompletedProcess[str]: + readlink = ( + 'readlink() { case "$1" in ' + + " ".join( + f"*/{name}) echo '{link}' ;;" for name, link in links.items() + ) + + " esac; }\n" + ) + return self._run( + ("status_field", "check_namespaces"), + f"{readlink}check_namespaces\necho checked", + { + "/proc/1/status": self._fixture("init-status", init_status), + "/proc/cmdline": self._fixture("cmdline", command_line + "\n"), + "/proc/sys/kernel/hostname": self._fixture( + "hostname", hostname + "\n" + ), + }, + ) + + private = { + "mnt": "mnt:[4026532200]", + "pid": "pid:[4026532201]", + "uts": "uts:[4026532202]", + } + result = run(private) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout, "checked\n") + result = run(private, command_line="console=hvc0", hostname="nvx-sandbox") + self.assertEqual(result.returncode, 0, result.stderr) + + for name, initial in ( + ("mnt", "mnt:[4026531832]"), + ("pid", "pid:[4026531836]"), + ("uts", "uts:[4026531838]"), + ): + with self.subTest(namespace=name): + result = run({**private, name: initial}) + self.assertEqual(result.returncode, 1, result.stdout) + self.assertIn(f"shares the initial {name} namespace", result.stderr) + for link in ("pid:[]", "pid:[42a]", "mnt:[42]", "pid:42"): + with self.subTest(link=link): + result = run({**private, "pid": link}) + self.assertEqual(result.returncode, 1, result.stdout) + self.assertIn("unexpected pid namespace link", result.stderr) + result = run( + private, + init_status=_SANDBOX_STATUS.replace( + "Uid:\t65534\t65534\t65534\t65534", "Uid:\t0\t0\t0\t0" + ), + ) + self.assertEqual(result.returncode, 1, result.stdout) + self.assertIn("PID 1 of the workload PID namespace runs as 0", result.stderr) + result = run(private, hostname="nvx-sandbox") + self.assertEqual(result.returncode, 1, result.stdout) + self.assertIn("expected nvx-lifecycle", result.stderr) + + def test_mount_check_requires_private_runtime_mounts_once(self): + functions = ("mount_records", "require_mount", "check_mounts") + mountinfo = self._fixture("mountinfo", _SANDBOX_MOUNTINFO) + result = self._run( + functions, + "check_mounts\necho checked", + {"/proc/self/mountinfo": mountinfo}, + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout, "checked\n") + + proc = "22 20 0:22 / /proc rw,nosuid,nodev,noexec,relatime - proc proc rw\n" + mutations = { + "stacked": ( + proc, + proc + proc.replace("22 20 0:22", "27 22 0:27"), + "several mounts are stacked at /proc", + ), + "propagation": ( + proc, + proc.replace("relatime - proc", "relatime shared:5 - proc"), + "the mount at /proc propagates: shared:5", + ), + "writable-sysfs": ( + "/sys ro,nosuid", + "/sys rw,nosuid", + "/sys is mounted rw,nosuid,nodev,noexec,relatime, without ro", + ), + "upper": ( + "upperdir=/run/nvx/scratch/upper", + "upperdir=/run/nvx/upper", + "upper layer is not on scratch", + ), + "lower": ( + "lowerdir=/run/nvx/layers/distro", + "lowerdir=/srv/distro", + "does not use the image layers", + ), + "missing-shm": ( + "26 24 0:26 / /dev/shm rw,nosuid,nodev,relatime - tmpfs tmpfs rw\n", + "", + "nothing is mounted at /dev/shm", + ), + "host-devices": ( + "- tmpfs tmpfs rw,mode=755\n", + "- devtmpfs devtmpfs rw,mode=755\n", + "/dev is devtmpfs, expected tmpfs", + ), + } + for mutation, (old, new, message) in mutations.items(): + with self.subTest(mutation=mutation): + self.assertIn(old, _SANDBOX_MOUNTINFO) + mountinfo = self._fixture( + "mountinfo", _SANDBOX_MOUNTINFO.replace(old, new, 1) + ) + result = self._run( + functions, "check_mounts", {"/proc/self/mountinfo": mountinfo} + ) + self.assertEqual(result.returncode, 1, result.stdout) + self.assertIn(message, result.stderr) + + def test_cgroup_check_requires_the_container_cgroup(self): + for content, returncode in ( + ("0::/container\n", 0), + ("0::/agent\n", 1), + ("12:pids:/container\n", 1), + ): + with self.subTest(content=content): + cgroup = self._fixture("cgroup", content) + result = self._run( + ("check_cgroup",), "check_cgroup", {"/proc/self/cgroup": cgroup} + ) + self.assertEqual(result.returncode, returncode, result.stderr) + + def test_limits_check_validates_its_arguments(self): + for arguments, message in ( + ("abc 8", "limits requires numeric MEMORY_MAX and PIDS_MAX"), + ("33554432 ''", "limits requires numeric MEMORY_MAX and PIDS_MAX"), + ("33554432 2", "requires PIDS_MAX of at least 3"), + ): + with self.subTest(arguments=arguments): + result = self._run( + ("start_sleepers", "check_limits"), f"check_limits {arguments}" + ) + self.assertEqual(result.returncode, 1, result.stdout) + self.assertIn(message, result.stderr) + + @unittest.skipUnless(sys.platform == "linux", "inspects the Linux process table") + def test_sleeper_subshell_records_and_reaps_its_sleepers(self): + record = self.root / "sleepers" + result = self._run( + ("start_sleepers",), + f'(start_sleepers "{record.as_posix()}" 3)\necho done', + # A distinct duration identifies this test's sleepers. + {"sleep 600": "sleep 611.25"}, + ) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(record.read_text(encoding="utf-8"), "3\n") + for process in Path("/proc").iterdir(): + if process.name.isdigit(): + try: + command = (process / "cmdline").read_bytes() + except OSError: + continue + self.assertNotEqual(command, b"sleep\x00611.25\x00") + + def test_main_body_runs_the_profile_then_limits_and_shares(self): + body = self.source[self.source.index("\ncheck_identity\n") + 1 :] + stubs = "".join( + f'{name}() {{ echo "{name} $*"; }}\n' + for name in ( + "check_identity", + "check_capabilities", + "check_namespaces", + "check_mounts", + "check_devices", + "check_cgroup", + "check_image", + "check_limits", + "check_share", + "check_shares_isolated", + ) + ) + script = ( + self.source[: self.source.index("\nfail() {\n") + 1] + + _shell_function(self.source, "fail") + + stubs + + body + ) + + def run(*arguments: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [self.shell, "-s", "--", *arguments], + input=script, + text=True, + capture_output=True, + check=False, + ) + + result = run("limits", "33554432", "8", "/workspace", "rw", "/tools") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual( + result.stdout.splitlines(), + [ + "check_identity ", + "check_capabilities ", + "check_namespaces ", + "check_mounts ", + "check_devices ", + "check_cgroup ", + "check_image ", + "NVX-UBUNTU-SANDBOX-PROFILE-OK uid=65534 gid=65534", + "check_limits 33554432 8", + "check_share /workspace rw", + "check_share /tools ro", + "check_shares_isolated /workspace /tools", + "NVX-UBUNTU-SANDBOX-OK uid=65534 gid=65534", + ], + ) + result = run("limits", "33554432") + self.assertEqual(result.returncode, 1, result.stdout) + self.assertIn("limits requires MEMORY_MAX and PIDS_MAX", result.stderr) + self.assertNotIn("NVX-UBUNTU-SANDBOX-OK", result.stdout) + + +class ManagedAgentCgroupTests(unittest.TestCase): + SOURCE = Path(__file__).parents[1] / "guest" / "common" / "nvx-managed-agent.c" + + def setUp(self): + if sys.platform != "linux": + self.skipTest("the managed agent requires Linux") + compiler = shutil.which("cc") or shutil.which("gcc") + if compiler is None: + self.skipTest("C compiler is unavailable") + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + root = Path(temporary.name) + self.cgroup = root / "cgroup" + self.cgroup.mkdir() + (self.cgroup / "cgroup.procs").touch() + self.subtree_control = self.cgroup / "cgroup.subtree_control" + self.subtree_control.touch() + self.execution = self.cgroup / "nvx-exec" + self.execution.mkdir() + self.tasks = self.execution / "pids.current" + self.tasks.write_bytes(b"0\n") + (self.execution / "cgroup.kill").touch() + source = root / "cgroup-test.c" + source.write_text( + f"""#define _GNU_SOURCE +#include +#include +static pid_t harness_waitpid(pid_t pid, int *status, int options); +#define waitpid(pid, status, options) harness_waitpid(pid, status, options) +#define CGROUP_ROOT {json.dumps(str(self.cgroup))} +#define main managed_agent_main +#include {json.dumps(str(self.SOURCE))} +#undef main +#undef waitpid + +static pid_t harness_process = -1; + +/* Like the pids controller, stops charging a process once it is reaped. */ +static pid_t harness_waitpid(pid_t pid, int *status, int options) +{{ + pid_t result = waitpid(pid, status, options); + + if (result > 0 && result == harness_process) {{ + int fd = open(EXEC_CGROUP "/pids.current", O_WRONLY | O_TRUNC | O_CLOEXEC); + + if (fd < 0 || write(fd, "0\\n", 2) != 2 || close(fd) != 0) {{ + _exit(3); + }} + harness_process = -1; + }} + return result; +}} + +/* + * Calls `check` while a workload process that the cgroup charges exits, or + * once it has exited if `exited` is set, and requires `check` to reap it. + */ +static int with_exiting_process(int (*check)(void), int exited) +{{ + siginfo_t information = {{0}}; + int result; + int status; + pid_t child = fork(); + + if (child < 0) {{ + _exit(2); + }} + if (child == 0) {{ + const struct timespec delay = {{.tv_nsec = 50L * 1000L * 1000L}}; + + nanosleep(&delay, NULL); + _exit(0); + }} + harness_process = child; + if (exited && waitid(P_PID, (id_t)child, &information, WEXITED | WNOWAIT) != 0) {{ + _exit(2); + }} + result = check(); + status = errno; + if (harness_process >= 0 || waitpid(child, NULL, WNOHANG) >= 0 || errno != ECHILD) {{ + _exit(4); + }} + errno = status; + return result; +}} + +int main(int argc, char **argv) +{{ + int result; + if (argc != 2) {{ + return 2; + }} + errno = 0; + if (strcmp(argv[1], "retry") == 0) {{ + struct control_session session = {{.fd = STDOUT_FILENO}}; + struct agent_config config = {{.direct = 1}}; + struct exec_config exec_config = {{0}}; + char *command[] = {{"/bin/true", NULL}}; + result = run_exec(&session, &config, 42, 0, command, &exec_config); + return result == 0 + ? run_exec(&session, &config, 43, 0, command, &exec_config) + : result; + }} + if (strcmp(argv[1], "tasks") == 0) {{ + result = exec_cgroup_has_tasks(); + }} else if (strcmp(argv[1], "prepare") == 0) {{ + result = prepare_exec_cgroup(); + }} else if (strcmp(argv[1], "settle") == 0) {{ + result = settle_exec_cgroup(); + }} else if (strcmp(argv[1], "settle-reap") == 0) {{ + result = with_exiting_process(settle_exec_cgroup, 0); + }} else if (strcmp(argv[1], "verify-reap") == 0) {{ + result = with_exiting_process(reap_exec_cgroup, 1); + }} else {{ + return 2; + }} + printf("%d %d\\n", result, errno); + return 0; +}} +""", + encoding="utf-8", + ) + self.helper = root / "cgroup-test" + flags = [ + flag + for flag in InitramfsBuildConstants.STATIC_HELPER_CFLAGS + if flag != "-static" + ] + result = subprocess.run( + [compiler, *flags, "-o", str(self.helper), str(source)], + capture_output=True, + text=True, + timeout=30, + check=False, + ) + self.assertEqual(result.returncode, 0, result.stderr) + + def _check(self, mode: str) -> tuple[int, int]: + result = subprocess.run( + [str(self.helper), mode], + capture_output=True, + text=True, + timeout=5, + check=False, + ) + self.assertEqual(result.returncode, 0, result.stderr) + value, status = result.stdout.split() + return int(value), int(status) + + def _assert_further_exec_refused(self): + result = subprocess.run( + [str(self.helper), "retry"], + capture_output=True, + timeout=5, + check=False, + ) + self.assertEqual(result.returncode, 0, result.stderr) + frames = result.stdout + for expected_id in (42, 43): + outer = ManagedAgentStopTests.OUTER + app = ManagedAgentStopTests.APP + *_, length = outer.unpack(frames[: outer.size]) + frame = frames[outer.size : outer.size + length] + _, _, kind, _, request_id, status, payload_len = app.unpack( + frame[: app.size] + ) + payload = frame[app.size :] + self.assertEqual((kind, request_id, status), (0xFF, expected_id, 125)) + self.assertEqual(payload_len, len(payload)) + self.assertEqual(payload, b"containment-failed") + frames = frames[outer.size + length :] + self.assertEqual(frames, b"") + + def test_task_count_requires_a_canonical_decimal(self): + for contents, expected in ( + (b"0\n", 0), + (b"1\n", 1), + (b"10\n", 1), + (b"4194304\n", 1), + ): + with self.subTest(contents=contents): + self.tasks.write_bytes(contents) + self.assertEqual(self._check("tasks")[0], expected) + for contents in ( + b"", + b"\n", + b"0", + b"00\n", + b"01\n", + b"-1\n", + b"+1\n", + b" 1\n", + b"1 \n", + b"0\n0\n", + b"1\x00\n", + b"max\n", + ): + with self.subTest(contents=contents): + self.tasks.write_bytes(contents) + self.assertEqual(self._check("tasks"), (-1, errno.EPROTO)) + self.tasks.write_bytes(b"1" * 40 + b"\n") + self.assertEqual(self._check("tasks"), (-1, errno.EOVERFLOW)) + + def test_missing_and_unreadable_task_counts_are_errors(self): + self.tasks.unlink() + self.assertEqual(self._check("tasks"), (-1, errno.ENOENT)) + self._assert_further_exec_refused() + self.tasks.mkdir() + self.assertEqual(self._check("tasks"), (-1, errno.EISDIR)) + self._assert_further_exec_refused() + + def test_preparation_enables_the_pids_controller_once(self): + self.assertEqual(self._check("prepare")[0], 0) + self.assertEqual(self.subtree_control.read_bytes(), b"") + self.tasks.unlink() + self.assertEqual(self._check("prepare")[0], 0) + self.assertEqual(self.subtree_control.read_bytes(), b"+pids") + self.subtree_control.unlink() + self.assertEqual(self._check("prepare"), (-1, errno.ENOENT)) + + def test_settlement_verifies_emptiness(self): + self.assertEqual(self._check("settle")[0], 0) + self.assertEqual((self.execution / "cgroup.kill").read_bytes(), b"1") + + def test_settlement_and_verification_reap_what_the_workload_left(self): + for mode in ("settle-reap", "verify-reap"): + with self.subTest(mode=mode): + self.tasks.write_bytes(b"1\n") + self.assertEqual(self._check(mode)[0], 0) + self.assertEqual(self.tasks.read_bytes(), b"0\n") + + def test_settlement_does_not_hide_kill_or_verification_failures(self): + self.tasks.write_bytes(b"1\n") + (self.execution / "cgroup.kill").unlink() + self.assertEqual(self._check("settle"), (-1, errno.ENOENT)) + self._assert_further_exec_refused() + (self.execution / "cgroup.kill").touch() + self.tasks.unlink() + self.assertEqual(self._check("settle"), (-1, errno.ENOENT)) + self._assert_further_exec_refused() + + def test_unreaped_tasks_at_deadline_are_not_settled(self): + # cgroup.events no longer counts a killed process that awaits its + # reaper, but the pids controller still charges it. + (self.execution / "cgroup.events").write_bytes(b"populated 0\nfrozen 0\n") + self.tasks.write_bytes(b"1\n") + started = time.monotonic() + self.assertEqual(self._check("settle"), (-1, errno.ETIMEDOUT)) + self.assertGreaterEqual(time.monotonic() - started, 1.9) + self._assert_further_exec_refused() + + def test_malformed_task_count_refuses_further_exec_until_verified_empty(self): + self.tasks.write_bytes(b"max\n") + self.assertEqual(self._check("settle"), (-1, errno.EPROTO)) + self._assert_further_exec_refused() + self.tasks.write_bytes(b"0\n") + self.assertEqual(self._check("tasks")[0], 0) + self.assertEqual(self._check("settle")[0], 0) + + +class GuestExitTests(unittest.TestCase): + """nvx-exit's arguments, and its single one-byte write at port 0x604.""" + + SOURCE = Path(__file__).parents[1] / "guest" / "common" / "nvx-exit.c" + PORT = 0x604 + + def setUp(self): + if sys.platform != "linux": + self.skipTest("nvx-exit builds on Linux") + compiler = shutil.which("cc") or shutil.which("gcc") + if compiler is None: + self.skipTest("C compiler is unavailable") + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + root = Path(temporary.name) + # A regular file stands in for /dev/port, so no test writes a port. + self.device = root / "port" + flags = [ + flag + for flag in InitramfsBuildConstants.STATIC_HELPER_CFLAGS + if flag != "-static" + ] + self.binaries: dict[str, Path] = {} + for name, device in (("ok", self.device), ("missing", root / "no" / "port")): + binary = root / f"nvx-exit-{name}" + result = subprocess.run( + [ + compiler, + *flags, + f'-DNVX_EXIT_DEVICE="{device}"', + "-o", + str(binary), + str(self.SOURCE), + ], + capture_output=True, + text=True, + check=False, + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.binaries[name] = binary + + def _run(self, name: str, *arguments: str) -> subprocess.CompletedProcess[bytes]: + return subprocess.run( + [str(self.binaries[name]), *arguments], + capture_output=True, + check=False, + timeout=10, + ) + + def test_writes_the_exit_code_as_one_byte_at_the_control_port(self): + cases = ( + ((), 1), + (("",), 1), + (("0",), 0), + (("37",), 37), + (("255",), 255), + (("256",), 1), + (("-1",), 1), + (("+5",), 1), + ((" 7",), 1), + (("12a",), 1), + (("99999999999999999999",), 1), + # Out of range after any digit, so the value never wraps. + (("4294967296",), 1), + (("4294967551",), 1), + (("00000000000000000255",), 255), + # Decimal: the old script's printf read a leading zero as octal. + (("010",), 10), + (("7", "9"), 7), + ) + for arguments, expected in cases: + with self.subTest(arguments=arguments): + self.device.write_bytes(b"") + result = self._run("ok", *arguments) + self.assertEqual( + (result.returncode, result.stdout, result.stderr), (0, b"", b"") + ) + self.assertEqual( + self.device.read_bytes(), bytes(self.PORT) + bytes([expected]) + ) + + def test_fails_silently_when_the_port_cannot_be_opened(self): + result = self._run("missing", "0") + self.assertEqual( + (result.returncode, result.stdout, result.stderr), (1, b"", b"") + ) + + +class ManagedAgentStopTests(unittest.TestCase): + SOURCE = Path(__file__).parents[1] / "guest" / "common" / "nvx-managed-agent.c" + OUTER = struct.Struct("<4sHBB16sQQI") + APP = struct.Struct("<4sBBHQiI") + + def setUp(self): + if sys.platform != "linux": + self.skipTest("the managed agent requires a Linux pseudo-terminal") + compiler = shutil.which("cc") or shutil.which("gcc") + if compiler is None: + self.skipTest("C compiler is unavailable") + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.agent = Path(temporary.name) / "nvx-managed-agent" + flags = [ + flag + for flag in InitramfsBuildConstants.STATIC_HELPER_CFLAGS + if flag != "-static" + ] + result = subprocess.run( + [compiler, *flags, "-o", str(self.agent), str(self.SOURCE)], + capture_output=True, + text=True, + check=False, + ) + self.assertEqual(result.returncode, 0, result.stderr) + + def _record( + self, record_type: int, instance: bytes, sequence: int, payload: bytes = b"" + ) -> bytes: + header = self.OUTER.pack( + b"NVXS", 1, record_type, 0, instance, 1, sequence, len(payload) + ) + return header + payload + + def _read_exact(self, fd: int, length: int, deadline: float) -> bytes: + data = b"" + while len(data) < length: + remaining = deadline - time.monotonic() + self.assertGreater(remaining, 0, "managed agent did not respond") + readable, _, _ = select.select([fd], [], [], remaining) + if readable: + chunk = os.read(fd, length - len(data)) + self.assertTrue(chunk, "managed agent closed the control tty") + data += chunk + return data + + def _read_record(self, fd: int, deadline: float) -> tuple[int, int, bytes]: + header = self._read_exact(fd, self.OUTER.size, deadline) + magic, version, record_type, flags, _, _, sequence, length = self.OUTER.unpack( + header + ) + self.assertEqual((magic, version, flags), (b"NVXS", 1, 0)) + return record_type, sequence, self._read_exact(fd, length, deadline) + + @staticmethod + def _reap(process: subprocess.Popen[bytes]) -> None: + if process.poll() is None: + process.kill() + process.wait(timeout=5) + if process.stderr is not None: + process.stderr.close() + + def _open_session( + self, rootfs: str + ) -> tuple[subprocess.Popen[bytes], int, bytes, float]: + """Starts an agent and completes the attach and reset handshake.""" + if sys.platform != "linux": + self.skipTest("the managed agent requires a Linux pseudo-terminal") + import pty + + master, slave = pty.openpty() + self.addCleanup(os.close, master) + self.addCleanup(os.close, slave) + process = subprocess.Popen( + [ + str(self.agent), + os.ttyname(slave), + rootfs, + "nvx-sandbox", + "65534", + "65534", + "nobody", + "/nonexistent", + ], + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + ) + self.addCleanup(self._reap, process) + deadline = time.monotonic() + 10 + instance = bytes(range(1, 17)) + + self.assertEqual(self._read_record(master, deadline)[0], 1) + os.write(master, self._record(3, instance, 7)) + record_type, sequence, credit = self._read_record(master, deadline) + self.assertEqual((record_type, sequence, len(credit)), (4, 0, 4)) + return process, master, instance, deadline + + def _request( + self, rootfs: str, kind: int, payload: bytes = b"" + ) -> tuple[int, int, int, bytes]: + """Sends one request to a fresh agent; returns the kind, request ID, status, and payload of its answer.""" + _, master, instance, deadline = self._open_session(rootfs) + frame = self.APP.pack(b"NVXC", 1, kind, 0, 42, 0, len(payload)) + payload + os.write(master, self._record(5, instance, 8, frame)) + self.assertEqual(self._read_record(master, deadline)[0], 9) + record_type, _, answer = self._read_record(master, deadline) + self.assertEqual(record_type, 5) + _, _, answered, _, request_id, status, length = self.APP.unpack( + answer[: self.APP.size] + ) + body = answer[self.APP.size :] + self.assertEqual(length, len(body)) + return answered, request_id, status, body + + def test_sandbox_agent_returns_to_init_agent_after_stop(self): + if sys.platform != "linux": + self.skipTest("the managed agent requires a Linux pseudo-terminal") + process, master, instance, deadline = self._open_session("/run/nvx/rootfs") + stop = self.APP.pack(b"NVXC", 1, 3, 0, 42, 0, 0) + os.write(master, self._record(5, instance, 8, stop)) + self.assertEqual(self._read_record(master, deadline)[0], 9) + record_type, _, payload = self._read_record(master, deadline) + self.assertEqual(record_type, 5) + _, _, kind, _, request_id, status, length = self.APP.unpack(payload) + self.assertEqual((kind, request_id, status, length), (0x85, 42, 0, 0)) + + _, stderr = process.communicate(timeout=10) + self.assertEqual(process.returncode, 0, stderr) + + def test_agent_advertises_the_control_features_of_its_mode(self): + cancel, workload_account, exec_cgroup, environment = (1, 4, 8, 16) + exec_cwd, host_mapping_table = (32, 64) + for rootfs, expected in ( + # Only the direct agent maps host paths, gives each workload a cgroup, and + # refuses a working directory that the workload cannot enter. Bit 1, which + # announced nvx_map= kernel tokens, is retired. + ( + "-", + cancel + | workload_account + | exec_cgroup + | environment + | exec_cwd + | host_mapping_table, + ), + ("/run/nvx/rootfs", cancel | workload_account | environment), + ): + with self.subTest(rootfs=rootfs): + kind, request_id, status, body = self._request(rootfs, 5) + self.assertEqual((kind, request_id, status), (0x81, 42, 0)) + self.assertEqual(struct.unpack(" bytes: + """Encodes an extended EXEC payload that carries `cwd` after the arguments.""" + arguments = b"".join(struct.pack(" +int harness_setenv(const char *name, const char *value, int overwrite); +#define setenv(name, value, overwrite) harness_setenv(name, value, overwrite) +#define CGROUP_ROOT {json.dumps(str(cgroup))} +#define setgroups(size, list) harness_setgroups(size, list) +#define setegid(gid) harness_setegid(gid) +#define seteuid(uid) harness_seteuid(uid) +#define main managed_agent_main +#include {json.dumps(str(self.SOURCE))} +#undef main + +int harness_setenv(const char *name, const char *value, int overwrite) +{{ + if (strcmp(name, "PWD") == 0 && getenv("HARNESS_FAIL_PWD") != NULL) {{ + errno = ENOMEM; + return -1; + }} + return (setenv)(name, value, overwrite); +}} + +int harness_setgroups(size_t size, const gid_t *list) +{{ + (void)size; + (void)list; + if (getenv("HARNESS_FAIL_SETGROUPS") != NULL) {{ + errno = EPERM; + return -1; + }} + return 0; +}} + +int harness_setegid(gid_t gid) +{{ + (void)gid; + return 0; +}} + +int harness_seteuid(uid_t uid) +{{ + (void)uid; + return 0; +}} + +int main(int argc, char **argv) +{{ + if (argc >= 2 && strcmp(argv[1], "--exec-config-fd") == 0) {{ + return launch_workload(argc, argv); + }} + + struct control_session session = {{.fd = STDOUT_FILENO}}; + struct agent_config config = {{ + .direct = 1, + .uid = "65534", + .gid = "65534", + .user = "nobody", + .home = "/", + }}; + char *environment[] = {{getenv("HARNESS_ENVIRONMENT"), NULL}}; + struct exec_config exec_config = {{ + .cwd = argc == 3 ? argv[2] : NULL, + .environment = environment[0] == NULL ? NULL : environment, + .environment_count = environment[0] == NULL ? 0 : 1, + .environment_present = environment[0] != NULL, + .inherit_default_env = + environment[0] != NULL && getenv("HARNESS_LAYER_ENVIRONMENT") != NULL, + }}; + char *command[] = {{"/bin/sh", "-c", argv[1], NULL}}; + + if (argc != 2 && argc != 3) {{ + return 2; + }} + return run_exec(&session, &config, 42, 0, command, &exec_config) == 0 ? 0 : 1; +}} +""", + encoding="utf-8", + ) + flags = [ + flag + for flag in InitramfsBuildConstants.STATIC_HELPER_CFLAGS + if flag != "-static" + ] + result = subprocess.run( + [compiler, *flags, "-o", str(self.harness), str(source)], + capture_output=True, + text=True, + timeout=60, + check=False, + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.environment = { + **os.environ, + "PATH": f"{tools}{os.pathsep}{os.environ.get('PATH', '/usr/bin:/bin')}", + } + for name in ( + "HARNESS_BEFORE_LAUNCH", + "HARNESS_ENVIRONMENT", + "HARNESS_FAIL_PWD", + "HARNESS_FAIL_SETGROUPS", + "HARNESS_LAYER_ENVIRONMENT", + ): + self.environment.pop(name, None) + + def _exec( + self, + script: str, + cwd: str | None = None, + environment: dict[str, str] | None = None, + ) -> list[tuple[int, int, int, bytes]]: + """Runs one workload; returns the kind, request ID, status, and payload of each frame.""" + arguments = [str(self.harness), script] + if cwd is not None: + arguments.append(cwd) + result = subprocess.run( + arguments, + capture_output=True, + timeout=30, + env=environment or self.environment, + check=False, + ) + self.assertEqual(result.returncode, 0, result.stderr) + frames: list[tuple[int, int, int, bytes]] = [] + rest = result.stdout + while rest: + _, _, record_type, _, _, _, _, length = self.OUTER.unpack( + rest[: self.OUTER.size] + ) + payload = rest[self.OUTER.size : self.OUTER.size + length] + rest = rest[self.OUTER.size + length :] + self.assertEqual(record_type, 5) + _, _, kind, _, request_id, status, payload_len = self.APP.unpack( + payload[: self.APP.size] + ) + body = payload[self.APP.size :] + self.assertEqual(payload_len, len(body)) + frames.append((kind, request_id, status, body)) + return frames + + def _completed(self, frames: list[tuple[int, int, int, bytes]]) -> bytes: + """Returns the standard output of a workload that exited with status zero.""" + self.assertEqual(frames[-1], (0x84, 42, 0, b"exit"), frames) + self.assertTrue(all(kind == 0x82 for kind, *_ in frames[:-1]), frames) + return b"".join(body for *_, body in frames[:-1]) + + def _refused( + self, frames: list[tuple[int, int, int, bytes]], cwd: str, error: int + ) -> None: + diagnostic = ( + f"nvx-managed-agent: cannot enter working directory {cwd}: " + f"{os.strerror(error)}\n" + ).encode() + self.assertEqual( + frames, + [(0x83, 42, 0, diagnostic), (0xFF, 42, error, b"cwd-failed")], + ) + + def test_workloads_start_in_the_root_directory_by_default(self): + output = self._completed(self._exec('pwd; printf "%s\\n" "$PWD"')) + self.assertEqual(output, b"/\n/\n") + + def test_workloads_start_in_the_requested_directory(self): + for name in ("a", "b"): + directory = self.root / "work" / name + directory.mkdir(parents=True) + with self.subTest(directory=directory): + output = self._completed( + self._exec('pwd; printf "%s\\n" "$PWD"', str(directory)) + ) + self.assertEqual(output, f"{directory}\n{directory}\n".encode()) + + def test_pwd_keeps_the_requested_name_of_a_canonical_path(self): + target = self.root / "target" + target.mkdir() + link = self.root / "link" + link.symlink_to(target) + output = self._completed(self._exec("pwd; /bin/pwd -P", str(link))) + self.assertEqual(output, f"{link}\n{target}\n".encode()) + # Without a canonical name, PWD names the directory itself. + output = self._completed(self._exec('printf "%s\\n" "$PWD"', f"{link}/.")) + self.assertEqual(output, f"{target}\n".encode()) + + def test_unusable_directories_refuse_the_launch(self): + marker = self.root / "ran" + regular = self.root / "file" + regular.touch() + cases = [ + (self.root / "missing", errno.ENOENT), + (regular, errno.ENOTDIR), + ] + # Root may search any directory, so only an unprivileged user observes EACCES. + if sys.platform == "linux" and os.geteuid() != 0: + locked = self.root / "locked" + locked.mkdir(mode=0) + self.addCleanup(locked.chmod, 0o755) + cases.append((locked, errno.EACCES)) + for directory, error in cases: + with self.subTest(directory=directory): + frames = self._exec(f"touch {marker}", str(directory)) + self._refused(frames, str(directory), error) + self.assertFalse(marker.exists()) + + def test_identity_failures_refuse_the_launch(self): + environment = {**self.environment, "HARNESS_FAIL_SETGROUPS": "1"} + frames = self._exec(f"touch {self.root / 'ran'}", None, environment) + self.assertEqual(frames, [(0xFF, 42, 125, b"launch-failed")]) + self.assertFalse((self.root / "ran").exists()) + + def test_workloads_start_in_the_directory_that_was_checked(self): + checked = self.root / "checked" + retarget = self.root / "retarget" + renamed = self.root / "renamed" + checked.mkdir() + retarget.mkdir() + link = self.root / "link" + link.symlink_to(checked) + # Neither a retargeted link nor a renamed directory may fail the + # launch or move the workload to a directory that was not checked. + quote = shlex.quote + for cwd, change, expected in ( + (link, f"ln -sfn {quote(str(retarget))} {quote(str(link))}", checked), + (retarget, f"mv {quote(str(retarget))} {quote(str(renamed))}", renamed), + ): + with self.subTest(cwd=cwd): + environment = {**self.environment, "HARNESS_BEFORE_LAUNCH": change} + frames = self._exec("/bin/pwd -P", str(cwd), environment) + self.assertEqual(self._completed(frames), f"{expected}\n".encode()) + + def test_pwd_failures_refuse_the_launch(self): + marker = self.root / "ran" + environment = {**self.environment, "HARNESS_FAIL_PWD": "1"} + for cwd in (None, str(self.root)): + with self.subTest(cwd=cwd): + frames = self._exec(f"touch {marker}", cwd, environment) + self.assertEqual(frames, [(0xFF, 42, 125, b"launch-failed")]) + self.assertFalse(marker.exists()) + + def test_exact_environments_control_pwd(self): + # The agent leaves PWD to a request that replaces the environment, so + # it does not try to set PWD, and cannot fail to. + environment = { + **self.environment, + "HARNESS_ENVIRONMENT": "MODE=exact", + "HARNESS_FAIL_PWD": "1", + } + output = self._completed( + self._exec( + 'printf "%s\\n" "$MODE"; /bin/pwd -P', str(self.root), environment + ) + ) + self.assertEqual(output, f"exact\n{self.root}\n".encode()) + + def test_layered_environments_keep_the_agents_pwd(self): + # Entries layered over the default environment keep the PWD that the + # agent points at the directory, which, unlike a PWD that the shell + # computes, names a link as the request did. + target = self.root / "target" + target.mkdir() + link = self.root / "link" + link.symlink_to(target) + layered = { + **self.environment, + "HARNESS_ENVIRONMENT": "MODE=layered", + "HARNESS_LAYER_ENVIRONMENT": "1", + } + output = self._completed( + self._exec('printf "%s %s\\n" "$MODE" "$PWD"', str(link), layered) + ) + self.assertEqual(output, f"layered {link}\n".encode()) + frames = self._exec("true", str(link), {**layered, "HARNESS_FAIL_PWD": "1"}) + self.assertEqual(frames, [(0xFF, 42, 125, b"launch-failed")]) + + +class AciSandboxRunnerTests(unittest.TestCase): + def _artifacts(self, root: Path) -> dict[str, Path]: + names = { + KernelBuildConstants.BINARY_NAME: b"kernel", + AlpineBuildConstants.INITRAMFS_NAME: b"initrd", + "openvmm": b"vmm", + } + paths: dict[str, Path] = {} + for name, contents in names.items(): + paths[name] = root / name + paths[name].write_bytes(contents) + return paths + + def _environment(self, root: Path, paths: dict[str, Path]) -> dict[str, str]: + def artifact(name: str) -> Path: + return root / name + + def openvmm() -> Path: + return paths["openvmm"] + + with ( + patch.object(aci_edge_sandboxes_tests, "artifact_path", artifact), + patch.object(aci_edge_sandboxes_tests, "openvmm_binary_path", openvmm), + ): + return aci_edge_sandboxes_tests.e2e_environment( + "kvm", root / "state", root / "out" + ) + + def test_cli_registers_the_lifecycle_test(self): + args = nvx.parse_args(["test-aci-edge-sandboxes", "--backend", "mshv"]) + + self.assertEqual(args.backend, "mshv") + self.assertEqual(args.cargo, "cargo") + self.assertFalse(hasattr(args, "scratch_template")) + self.assertIs( + args.handler, aci_edge_sandboxes_tests.command_test_aci_edge_sandboxes + ) + + def test_environment_resolves_repository_artifacts(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + paths = self._artifacts(root) + environment = self._environment(root, paths) + + self.assertEqual(environment["ACI_EDGE_SANDBOXES_E2E_HYPERVISOR"], "kvm") + self.assertEqual( + Path(environment["ACI_EDGE_SANDBOXES_E2E_OPENVMM"]), + paths["openvmm"].resolve(), + ) + self.assertEqual( + Path(environment["ACI_EDGE_SANDBOXES_E2E_INITRD"]), + paths[AlpineBuildConstants.INITRAMFS_NAME].resolve(), + ) + self.assertEqual( + Path(environment["ACI_EDGE_SANDBOXES_E2E_STATE_ROOT"]), + (root / "state").resolve(), + ) + self.assertFalse( + any( + name.startswith("ACI_EDGE_SANDBOXES_E2E_DISTRO") + for name in environment + ) + ) + self.assertNotIn("ACI_EDGE_SANDBOXES_E2E_SCRATCH", environment) + + def test_environment_requires_the_alpine_initramfs(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + paths = self._artifacts(root) + paths[AlpineBuildConstants.INITRAMFS_NAME].unlink() + with self.assertRaisesRegex(common.ScriptError, "Alpine initramfs"): + self._environment(root, paths) + + def test_command_runs_only_the_ignored_lifecycle_test(self): + command = aci_edge_sandboxes_tests.e2e_command("cargo") + + self.assertEqual(command[:2], ["cargo", "test"]) + self.assertEqual( + Path(command[command.index("--manifest-path") + 1]), + BuildConstants.REPO_ROOT / "aci_edge_sandboxes" / "Cargo.toml", + ) + self.assertIn("--locked", command) + self.assertEqual(command[command.index("--test") + 1], "openvmm_e2e") + self.assertEqual(command[-2:], ["--ignored", "--nocapture"]) + + def test_handler_returns_the_test_status(self): + observed: list[str] = [] + + def fake_run( + command: list[str], *, env: dict[str, str], check: bool + ) -> subprocess.CompletedProcess[bytes]: + self.assertFalse(check) + self.assertEqual(command, aci_edge_sandboxes_tests.e2e_command("cargo")) + observed.append(env["ACI_EDGE_SANDBOXES_E2E_STATE_ROOT"]) + return subprocess.CompletedProcess(command, 3) + + def fake_environment( + backend: str, state_root: Path, output_dir: Path + ) -> dict[str, str]: + self.assertEqual(backend, "whp") + return {"ACI_EDGE_SANDBOXES_E2E_STATE_ROOT": os.fspath(state_root)} + + with tempfile.TemporaryDirectory() as temporary: + args = argparse.Namespace( + backend="whp", + output_dir=Path(temporary) / "results", + cargo="cargo", + ) + with ( + patch.object(aci_edge_sandboxes_tests, "validate_openvmm_test_backend"), + patch.object( + aci_edge_sandboxes_tests, "e2e_environment", fake_environment + ), + patch.object(aci_edge_sandboxes_tests.subprocess, "run", fake_run), + ): + self.assertEqual( + aci_edge_sandboxes_tests.command_test_aci_edge_sandboxes(args), 3 + ) + self.assertTrue((Path(temporary) / "results").is_dir()) + + self.assertEqual(len(observed), 1) + self.assertEqual(Path(observed[0]).name, "state") + + def _run_leaving( + self, records: dict[str, object] | None, *, interrupt: bool = False + ) -> tuple[Path, str]: + """Runs the handler with a test command that leaves one sandbox provisioned. + + The sandbox gets `records` as extra state files; `None` leaves only the lock + directories that deprovisioning keeps. Returns the state root that the command + used and the handler's standard error. + """ + state_roots: list[Path] = [] + + def fake_environment( + backend: str, state_root: Path, output_dir: Path + ) -> dict[str, str]: + state_roots.append(state_root) + return {} + + def fake_run( + command: list[str], *, env: dict[str, str], check: bool + ) -> subprocess.CompletedProcess[bytes]: + (state_roots[0] / "filesystem" / ".locks").mkdir(parents=True) + if records is not None: + sandbox = state_roots[0] / "network" / ("a" * 32) + sandbox.mkdir(parents=True) + (sandbox / "sandbox.json").write_text("{}", encoding="utf-8") + for name, record in records.items(): + (sandbox / name).write_text(json.dumps(record), encoding="utf-8") + if interrupt: + raise KeyboardInterrupt + return subprocess.CompletedProcess(command, 0) + + with tempfile.TemporaryDirectory() as temporary: + args = argparse.Namespace( + backend="kvm", output_dir=Path(temporary) / "results", cargo="cargo" + ) + with ( + patch.object(aci_edge_sandboxes_tests, "validate_openvmm_test_backend"), + patch.object( + aci_edge_sandboxes_tests, "e2e_environment", fake_environment + ), + patch.object(aci_edge_sandboxes_tests.subprocess, "run", fake_run), + patch("sys.stderr", io.StringIO()) as stderr, + ): + if interrupt: + with self.assertRaises(KeyboardInterrupt): + aci_edge_sandboxes_tests.command_test_aci_edge_sandboxes(args) + else: + self.assertEqual( + aci_edge_sandboxes_tests.command_test_aci_edge_sandboxes(args), + 0, + ) + state_root = state_roots[0] + if state_root.parent.exists(): + self.addCleanup(shutil.rmtree, state_root.parent, True) + return state_root, stderr.getvalue() + + def test_handler_deletes_state_once_every_sandbox_is_deprovisioned(self): + state_root, stderr = self._run_leaving(None) + + self.assertFalse(state_root.parent.exists()) + self.assertEqual(stderr, "") + + def test_handler_keeps_state_of_sandboxes_left_provisioned(self): + state_root, stderr = self._run_leaving({"runtime.json": {"pid": 4242}}) + + sandbox = state_root / "network" / ("a" * 32) + self.assertTrue((sandbox / "runtime.json").is_file()) + self.assertIn(f"kept {state_root}", stderr) + self.assertIn(f"{sandbox} (OpenVMM pid 4242)", stderr) + + def test_handler_keeps_state_when_the_test_is_interrupted(self): + state_root, stderr = self._run_leaving( + {"launch.json": {"process": {"pid": 77}}}, interrupt=True + ) + + sandbox = state_root / "network" / ("a" * 32) + self.assertTrue((sandbox / "launch.json").is_file()) + self.assertIn(f"{sandbox} (OpenVMM pid 77)", stderr) + + +def _write_managed_runtime(state: Path, pid: int, start_time: int | None) -> None: + runtime: dict[str, object] = { + "format": sandbox_lifecycle.STATE_FORMAT, + "pid": pid, + "control_endpoint": os.fspath(state / sandbox_lifecycle.CONTROL_SOCKET_NAME), + } + if start_time is not None: + runtime["start_time"] = start_time + sandbox_lifecycle._write_json(state / sandbox_lifecycle.RUNTIME_NAME, runtime) + (state / sandbox_lifecycle.CAPABILITY_NAME).write_bytes(b"x" * 32) + + +def _provision_managed_sandbox(root: Path) -> Path: + layer_path = root / "distro.erofs" + scratch_path = root / "scratch.ext4" + layer_path.write_bytes(b"layer") + scratch_path.write_bytes(b"scratch") + state = root / "state" + sandbox_lifecycle.provision( + state, + sandbox.SandboxLaunch( + layers=( + sandbox.SandboxLayer( + role="distro", + path=layer_path, + uuid="11111111-1111-1111-1111-111111111111", + ), + ), + scratch=scratch_path, + ), + hypervisor="whp", + memory_mib=256, + net=None, + network_profile=None, + network_egress=None, + network_ingress=None, + network_egress_allow=(), + network_egress_deny=(), + host_loopback=None, + network_proxy=None, + host_loopback_forward=(), + cmdline="quiet", + timeout=10, + ) + return state + + +def _stdin_bound_child() -> subprocess.Popen[bytes]: + return subprocess.Popen( + [sys.executable, "-c", "import sys; sys.stdin.read()"], + stdin=subprocess.PIPE, + ) + + +def _finish_child(child: subprocess.Popen[bytes]) -> None: + if child.stdin is not None: + child.stdin.close() + child.wait(timeout=30) + + +def _exit_unreaped(child: subprocess.Popen[bytes]) -> None: + if sys.platform != "linux": + raise unittest.SkipTest("zombie processes require a Linux host") + assert child.stdin is not None + child.stdin.close() + # WNOWAIT leaves the exited child unreaped, so it stays a zombie. + os.waitid(os.P_PID, child.pid, os.WEXITED | os.WNOWAIT) + + +def _end_child(child: subprocess.Popen[bytes]) -> None: + if child.poll() is None: + child.kill() + child.wait(timeout=30) + + +def _openvmm_mock(status: int | None) -> MagicMock: + """Mocks an OpenVMM process that exits on its own with `status`. + + With None, it runs until it is terminated and then exits with status 1. + """ + process = MagicMock() + process.pid = 123 + process.stdin = io.BytesIO() + process.poll.return_value = status + + def wait(timeout: float) -> int: + if process.terminate.called: + return 1 + if status is None: + raise subprocess.TimeoutExpired("openvmm", timeout) + return status + + process.wait.side_effect = wait + return process + + +# Stands in for OpenVMM when the managed guest exits during startup. Like OpenVMM, +# it closes the control endpoint before it publishes its outcome report through a +# staging file, and a slow flush keeps that file in place for a second. It takes +# the size of the attach request and then OpenVMM's arguments. +_EXITING_OPENVMM = r""" +import json +import os +import socket +import sys +import tempfile +import time + +attach_size = int(sys.argv[1]) +arguments = sys.argv[2:] +endpoint = arguments[arguments.index("--microvm-control-console") + 1] +endpoint = endpoint.removeprefix("listen=") +report = arguments[arguments.index("--microvm-report") + 1] +sys.stdin.buffer.read() +received = 0 +if os.name == "nt": + import _winapi + + pipe = _winapi.CreateNamedPipe( + endpoint.replace("/", "\\"), + _winapi.PIPE_ACCESS_DUPLEX, + _winapi.PIPE_WAIT, + 1, + 65536, + 65536, + 0, + _winapi.NULL, + ) + try: + _winapi.ConnectNamedPipe(pipe, False) + except OSError as error: + if error.winerror != _winapi.ERROR_PIPE_CONNECTED: + raise + while received < attach_size: + data, _ = _winapi.ReadFile(pipe, attach_size - received) + received += len(data) + _winapi.CloseHandle(pipe) +else: + listener = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + listener.bind(endpoint) + listener.listen(1) + connection, _ = listener.accept() + while received < attach_size: + chunk = connection.recv(attach_size - received) + if not chunk: + sys.exit("the control client closed before it attached") + received += len(chunk) + connection.close() + listener.close() +report_document = json.dumps( + { + "schema_version": 1, + "outcome": { + "operation": "managed", + "category": "guest-exit", + "status_code": 125, + }, + "network_policy": {"status": "not-requested"}, + "teardown": {"vm_stopped": True}, + } +).encode() +staging = tempfile.NamedTemporaryFile( + dir=os.path.dirname(report), prefix=".tmp", delete=False +) +staging.write(report_document[: len(report_document) // 2]) +staging.flush() +time.sleep(1) +staging.write(report_document[len(report_document) // 2 :]) +staging.close() +os.rename(staging.name, report) +sys.exit(125) +""" + +# Stands in for the OpenVMM of a managed sandbox that runs until it is stopped. +# Like OpenVMM, it reads its capability from standard input, serves the control +# endpoint one client at a time, rejects an attach with another capability, +# acknowledges a ping, and on a stop publishes its outcome report and exits. +_SERVING_OPENVMM = r""" +import json +import os +import secrets +import socket +import struct +import sys + +OUTER = struct.Struct("<4sHBB16sQQI") +APP = struct.Struct("<4sBBHQiI") +arguments = sys.argv[1:] +endpoint = arguments[arguments.index("--microvm-control-console") + 1] +endpoint = endpoint.removeprefix("listen=") +report = arguments[arguments.index("--microvm-report") + 1] +capability = sys.stdin.buffer.read() +instance = secrets.token_bytes(16) +if os.name == "nt": + import _winapi + + def accept(): + pipe = _winapi.CreateNamedPipe( + endpoint.replace("/", "\\"), + _winapi.PIPE_ACCESS_DUPLEX, + _winapi.PIPE_WAIT, + 1, + 65536, + 65536, + 0, + _winapi.NULL, + ) + try: + _winapi.ConnectNamedPipe(pipe, False) + except OSError as error: + if error.winerror != _winapi.ERROR_PIPE_CONNECTED: + raise + return pipe + + def receive(pipe, length): + return _winapi.ReadFile(pipe, length)[0] + + def send(pipe, data): + _winapi.WriteFile(pipe, data) + + def close(pipe): + _winapi.CloseHandle(pipe) + +else: + listener = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + listener.bind(endpoint) + listener.listen(1) + + def accept(): + return listener.accept()[0] + + def receive(connection, length): + return connection.recv(length) + + def send(connection, data): + connection.sendall(data) + + def close(connection): + connection.close() + + +def read_exact(connection, length): + data = b"" + while len(data) < length: + chunk = receive(connection, length - len(data)) + if not chunk: + raise EOFError + data += chunk + return data + + +def read_record(connection): + header = OUTER.unpack(read_exact(connection, OUTER.size)) + return header[2], read_exact(connection, header[7]) if header[7] else b"" + + +def write_record(connection, kind, sequence, payload=b""): + header = OUTER.pack(b"NVXS", 1, kind, 0, instance, 1, sequence, len(payload)) + send(connection, header + payload) + + +while True: + connection = accept() + try: + kind, payload = read_record(connection) + if kind != 2 or payload != capability: + write_record(connection, 8, 0) + continue + write_record(connection, 7, 0) + sequence = 1 + while True: + request = APP.unpack(read_record(connection)[1][: APP.size]) + stopping = request[2] == 3 + if stopping: + with open(report, "w", encoding="utf-8") as output: + json.dump( + { + "schema_version": 1, + "outcome": { + "operation": "managed", + "category": "success", + "status_code": 0, + }, + "network_policy": {"status": "not-requested"}, + "teardown": {"vm_stopped": True}, + }, + output, + ) + reply = APP.pack(b"NVXC", 1, 0x85 if stopping else 0x81, 0, request[4], 0, 0) + write_record(connection, 5, sequence, reply) + sequence += 1 + if stopping: + sys.exit(0) + except (EOFError, OSError): + pass + finally: + close(connection) +""" + + +class SandboxTests(unittest.TestCase): + def test_launch_contract_orders_roles_and_builds_agent_command_line(self): + custom = sandbox.SandboxLayer.parse( + "custom,custom.erofs,22222222-2222-2222-2222-222222222222" + ) + distro = sandbox.SandboxLayer.parse( + "distro,distro.erofs,11111111-1111-1111-1111-111111111111" + ) + launch = sandbox.SandboxLaunch( + layers=(custom, distro), + scratch=Path("scratch.ext4"), + entrypoint="/bin/workload", + args=("--serve",), + hostname="example", + workload_identity=(1000, 1001), + memory_max=268435456, + pids_max=64, + ) + + self.assertEqual( + [layer.role for layer in launch.ordered_layers()], + ["distro", "custom"], + ) + self.assertEqual( + launch.kernel_command_line("quiet"), + ( + "quiet nvx_sandbox=1 " + "nvx_layer=distro,0xd0003000,11111111-1111-1111-1111-111111111111 " + "nvx_layer=custom,0xd0005000,22222222-2222-2222-2222-222222222222 " + "nvx_scratch=0xd0006000,ext4 " + "nvx_entrypoint=/bin/workload nvx_hostname=example " + "nvx_arg=--serve nvx_memory_max=268435456 nvx_pids_max=65" + ), + ) + self.assertEqual( + launch.openvmm_arguments(), + [ + "--machine", + "microvm", + "--microvm-sandbox-block", + "distro:file:distro.erofs,ro", + "--microvm-sandbox-block", + "custom:file:custom.erofs,ro", + "--microvm-sandbox-block", + "scratch:file:scratch.ext4", + "--microvm-workload-identity", + "1000:1001", + ], + ) + + def test_launch_contract_rejects_duplicates_and_reserved_tokens(self): + distro = sandbox.SandboxLayer.parse( + "distro,distro.erofs,11111111-1111-1111-1111-111111111111" + ) + with self.assertRaisesRegex(common.ScriptError, "duplicate.*distro"): + sandbox.SandboxLaunch( + layers=(distro, distro), + scratch=Path("scratch.ext4"), + ) + + launch = sandbox.SandboxLaunch( + layers=(distro,), + scratch=Path("scratch.ext4"), + ) + with self.assertRaisesRegex(common.ScriptError, "owned"): + launch.kernel_command_line("nvx_sandbox=0") + with self.assertRaisesRegex(common.ScriptError, "owned"): + launch.kernel_command_line(r"foo=bar\ nvx_memory_max=max") + with self.assertRaisesRegex(common.ScriptError, "1024-byte"): + launch.kernel_command_line("x" * sandbox.SANDBOX_COMMAND_LINE_MAX_SIZE) + with self.assertRaisesRegex(common.ScriptError, "between 1"): + sandbox.parse_workload_identity("0:0") + + def test_layer_parser_rejects_invalid_role_and_uuid(self): + with self.assertRaisesRegex(common.ScriptError, "unsupported layer role"): + sandbox.SandboxLayer.parse( + "unknown,layer.erofs,11111111-1111-1111-1111-111111111111" + ) + with self.assertRaisesRegex(common.ScriptError, "UUID is invalid"): + sandbox.SandboxLayer.parse("distro,layer.erofs,not-a-uuid") + + def test_mount_parser_defaults_to_read_only_and_accepts_rw(self): + default = sandbox.SandboxMount.parse("/workspace,host-dir") + self.assertEqual(default.guest_target, "/workspace") + self.assertEqual(default.host_path, Path("host-dir")) + self.assertEqual(default.access, "ro") + self.assertEqual(default.denied_paths, ()) + + writable = sandbox.SandboxMount.parse( + "/opt/hostedtoolcache,host-dir,rw", ("logs", "secrets") + ) + self.assertEqual(writable.access, "rw") + self.assertEqual(writable.denied_paths, ("logs", "secrets")) + self.assertEqual( + sandbox.mounts_openvmm_arguments((writable,)), + [ + "--mount", + f"/opt/hostedtoolcache,{os.fspath(Path('host-dir'))},rw", + "--mount-deny", + "logs", + "--mount-deny", + "secrets", + ], + ) + self.assertEqual( + writable.command_line_fragment(), + " virtfs_dir=/opt/hostedtoolcache virtfs_tag=microvm virtfs_mode=rw", + ) + + def test_mount_parser_rejects_malformed_specifications(self): + for value, message in ( + ("/workspace", "GUEST_TARGET,HOST_PATH"), + ("/workspace,host,rw,extra", "GUEST_TARGET,HOST_PATH"), + ("/workspace,", "host path is empty"), + ("/workspace,host,rx", "unsupported sandbox mount mode"), + ): + with self.subTest(value=value): + with self.assertRaisesRegex(common.ScriptError, message): + sandbox.SandboxMount.parse(value) + with self.assertRaisesRegex(common.ScriptError, "commas"): + sandbox.SandboxMount(guest_target="/workspace", host_path=Path("a,b")) + with self.assertRaisesRegex(common.ScriptError, "parent component"): + sandbox.SandboxMount.parse("/workspace,link/../share") + + def test_mount_owner_defaults_to_vmm_and_forwards_caller(self): + default = sandbox.SandboxMount.parse("/workspace,host-dir,rw") + self.assertEqual(default.owner, "vmm") + self.assertNotIn("--mount-owner", sandbox.mounts_openvmm_arguments((default,))) + + caller = sandbox.SandboxMount.parse( + "/workspace,host-dir,rw", ("secrets",), "caller" + ) + self.assertEqual( + sandbox.mounts_openvmm_arguments((caller,)), + [ + "--mount", + f"/workspace,{os.fspath(Path('host-dir'))},rw", + "--mount-deny", + "secrets", + "--mount-owner", + "caller", + ], + ) + self.assertEqual(caller.absolute().owner, "caller") + self.assertEqual( + caller.command_line_fragment(), default.command_line_fragment() + ) + with self.assertRaisesRegex( + common.ScriptError, "unsupported sandbox mount owner" + ): + sandbox.SandboxMount.parse("/workspace,host-dir", (), "root") + + def test_mount_access_policy_is_validated_and_forwarded(self): + mount = sandbox.SandboxMount.parse( + "/tmp/mcp-runtime,host-dir,rw", + ("mcp-logs",), + allowed_paths=("mcp-logs/payloads",), + writable_paths=("agent", "cache"), + ) + self.assertEqual( + sandbox.mounts_openvmm_arguments((mount,)), + [ + "--mount", + f"/tmp/mcp-runtime,{os.fspath(Path('host-dir'))},rw", + "--mount-deny", + "mcp-logs", + "--mount-allow", + "mcp-logs/payloads", + "--mount-write", + "agent", + "--mount-write", + "cache", + ], + ) + absolute = mount.absolute() + self.assertEqual(absolute.allowed_paths, ("mcp-logs/payloads",)) + self.assertEqual(absolute.writable_paths, ("agent", "cache")) + # OpenVMM enforces the policy on the host, so the guest's bootstrap + # tokens are those of the plain share. + self.assertEqual( + mount.command_line_fragment(), + " virtfs_dir=/tmp/mcp-runtime virtfs_tag=microvm virtfs_mode=rw", + ) + for create_mount, message in ( + ( + lambda: sandbox.SandboxMount.parse( + "/w,host,ro", writable_paths=("out",) + ), + "/w is read-only", + ), + ( + lambda: sandbox.SandboxMount.parse( + "/w,host,rw", allowed_paths=("logs/payloads",) + ), + "which has none", + ), + ( + lambda: sandbox.SandboxMount.parse( + "/w,host,rw", ("logs",), allowed_paths=("logs/a", "logs/a") + ), + "allowed paths must be unique", + ), + ( + lambda: sandbox.SandboxMount.parse("/w,host,rw", writable_paths=("",)), + "writable paths must be nonempty", + ), + ( + lambda: sandbox.SandboxMount.parse( + "/w,host,rw", + writable_paths=tuple(f"path-{index}" for index in range(129)), + ), + "at most 128 writable paths", + ), + ): + with self.subTest(message=message): + with self.assertRaisesRegex(common.ScriptError, message): + create_mount() + + def test_caller_owner_requires_linux_and_a_directory_that_root_does_not_own(self): + with patch.object(sandbox.os, "name", "nt"): + with self.assertRaisesRegex(common.ScriptError, "requires a Linux host"): + sandbox.require_mount_owner_supported("caller") + sandbox.require_mount_owner_supported("vmm") + if sys.platform == "win32": + with tempfile.TemporaryDirectory() as temporary: + mount = sandbox.SandboxMount( + guest_target="/workspace", host_path=Path(temporary), owner="caller" + ) + with self.assertRaisesRegex( + common.ScriptError, "requires a Linux host" + ): + mount.validated() + return + + with tempfile.TemporaryDirectory() as temporary: + mount = sandbox.SandboxMount( + guest_target="/workspace", host_path=Path(temporary), owner="caller" + ) + owner = Path(temporary).stat() + if owner.st_uid != 0 and owner.st_gid != 0: + self.assertIs(mount.validated(), mount) + filesystem_root = Path("/") + if filesystem_root.stat().st_uid == 0: + root_owned = sandbox.SandboxMount( + guest_target="/workspace", host_path=filesystem_root, owner="caller" + ) + with self.assertRaisesRegex(common.ScriptError, "UID 0 or GID 0"): + root_owned.validated() + sandbox.SandboxMount( + guest_target="/workspace", host_path=filesystem_root + ).validated() + + def test_mount_absolute_path_keeps_symbolic_link_components(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + (root / "link-target" / "share").mkdir(parents=True) + try: + (root / "link").symlink_to( + root / "link-target", target_is_directory=True + ) + except OSError as error: + self.skipTest(f"symbolic links are unavailable: {error}") + previous = Path.cwd() + os.chdir(root) + try: + mount = sandbox.SandboxMount.parse("/workspace,link/share,rw") + absolute = mount.absolute() + expected = Path.cwd() / "link" / "share" + finally: + os.chdir(previous) + + self.assertEqual(absolute.host_path, expected) + self.assertNotIn("link-target", absolute.host_path.parts) + self.assertEqual(absolute.absolute(), absolute) + with self.assertRaisesRegex(common.ScriptError, "unique"): + sandbox.SandboxMount.parse("/workspace,host", ("logs", "logs")) + with self.assertRaisesRegex(common.ScriptError, "nonempty"): + sandbox.SandboxMount.parse("/workspace,host", ("",)) + with self.assertRaisesRegex(common.ScriptError, "at most 128"): + sandbox.SandboxMount.parse( + "/workspace,host", tuple(f"path-{index}" for index in range(129)) + ) + + def test_mount_target_validation_rejects_unsafe_and_reserved_targets(self): + for target in ( + "", + "/", + "workspace", + "/workspace/", + "/work//space", + "/work/./space", + "/work/../space", + "/work space", + "/work=space", + "/work\\space", + "/" + "x" * 4096, + ): + with self.subTest(target=target): + with self.assertRaisesRegex(common.ScriptError, "invalid sandbox"): + sandbox.validate_mount_target(target) + for target in ( + "/proc", + "/proc/self", + "/sys", + "/sys/fs", + "/dev", + "/dev/shm", + "/.nvx-agent", + "/.nvx-agent/tools", + "/etc", + ): + with self.subTest(target=target): + with self.assertRaisesRegex(common.ScriptError, "reserved"): + sandbox.validate_mount_target(target) + for target in ("/workspace", "/procfs", "/etc/app", "/opt/hostedtoolcache"): + with self.subTest(target=target): + self.assertEqual(sandbox.validate_mount_target(target), target) + + def test_launch_contract_attaches_mount_and_reserves_bootstrap_tokens(self): + distro = sandbox.SandboxLayer.parse( + "distro,distro.erofs,11111111-1111-1111-1111-111111111111" + ) + launch = sandbox.SandboxLaunch( + layers=(distro,), + scratch=Path("scratch.ext4"), + mounts=(sandbox.SandboxMount.parse("/workspace,share,rw", ("secrets",)),), + ) + + arguments = launch.openvmm_arguments() + self.assertEqual( + arguments[arguments.index("--mount") :], + [ + "--mount", + f"/workspace,{os.fspath(Path('share'))},rw", + "--mount-deny", + "secrets", + ], + ) + self.assertNotIn("virtfs_", launch.kernel_command_line("quiet")) + for token in ("virtfs_dir=/other", "virtfs_tag=other", "virtfs_mode=rw"): + with self.subTest(token=token): + with self.assertRaisesRegex(common.ScriptError, "--mount option"): + launch.kernel_command_line(token) + + unmounted = sandbox.SandboxLaunch(layers=(distro,), scratch=Path("s.ext4")) + base = unmounted.kernel_command_line() + fragment = sandbox.mounts_command_line_fragment(launch.mounts) + self.assertEqual( + fragment, " virtfs_dir=/workspace virtfs_tag=microvm virtfs_mode=rw" + ) + fitting = "x" * ( + sandbox.SANDBOX_COMMAND_LINE_MAX_SIZE - len(base) - len(fragment) - 2 + ) + launch.kernel_command_line(fitting) + with self.assertRaisesRegex(common.ScriptError, "1024-byte"): + launch.kernel_command_line(fitting + "x") + + def test_launch_contract_attaches_several_shares_as_an_aggregate(self): + distro = sandbox.SandboxLayer.parse( + "distro,distro.erofs,11111111-1111-1111-1111-111111111111" + ) + launch = sandbox.SandboxLaunch( + layers=(distro,), + scratch=Path("scratch.ext4"), + mounts=( + sandbox.SandboxMount.parse("/workspace,work,rw", ("secrets",)), + sandbox.SandboxMount.parse( + "/opt/hostedtoolcache,tools,ro", ("credentials",) + ), + sandbox.SandboxMount.parse("/srv/cache,cache,ro"), + ), + ) + + arguments = launch.openvmm_arguments() + workspace, toolcache, cache = ( + sandbox.aggregate_child_name(index, mount.guest_target) + for index, mount in enumerate(launch.mounts) + ) + # Several shares are the children of one aggregate, and OpenVMM + # attributes each denied path to the child that contains it, so + # relative paths are joined to their share. + self.assertNotIn("--mount", arguments) + self.assertEqual( + arguments[arguments.index("--mount-aggregate") :], + [ + "--mount-aggregate", + "/run/nvx/shares", + "--mount-child", + f"{workspace},{os.fspath(Path('work'))},rw", + "--mount-child", + f"{toolcache},{os.fspath(Path('tools'))},ro", + "--mount-child", + f"{cache},{os.fspath(Path('cache'))},ro", + "--mount-deny", + os.fspath(Path.cwd() / "work" / "secrets"), + "--mount-deny", + os.fspath(Path.cwd() / "tools" / "credentials"), + ], + ) + fragment = sandbox.mounts_command_line_fragment(launch.mounts) + self.assertEqual( + fragment, + " virtfs_dir=/run/nvx/shares virtfs_tag=microvm virtfs_mode=rw" + " virtfs_aggregate=1", + ) + # The guest binds each child at its target, in order. + command_line = launch.kernel_command_line() + self.assertNotIn("virtfs_", command_line) + self.assertTrue( + command_line.endswith( + f" nvx_share={workspace},/workspace,rw" + f" nvx_share={toolcache},/opt/hostedtoolcache,ro" + f" nvx_share={cache},/srv/cache,ro" + ), + command_line, + ) + self.assertEqual( + sandbox.mounts_command_line_fragment(launch.mounts[1:]), + " virtfs_dir=/run/nvx/shares virtfs_tag=microvm virtfs_mode=ro" + " virtfs_aggregate=1", + ) + with self.assertRaisesRegex(common.ScriptError, "owned by the sandbox"): + launch.kernel_command_line("nvx_share=0,/etc,rw") + + # Every share's tokens count toward the x86 budget. + base = sandbox.SandboxLaunch( + layers=(distro,), scratch=Path("s.ext4") + ).kernel_command_line() + shares = len(command_line) - len(base) + fitting = "x" * ( + sandbox.SANDBOX_COMMAND_LINE_MAX_SIZE + - len(base) + - shares + - len(fragment) + - 2 + ) + launch.kernel_command_line(fitting) + with self.assertRaisesRegex(common.ScriptError, "1024-byte"): + launch.kernel_command_line(fitting + "x") + + caller = tuple( + sandbox.SandboxMount( + guest_target=target, host_path=Path(path), owner="caller" + ) + for target, path in (("/workspace", "work"), ("/tools", "tools")) + ) + arguments = sandbox.mounts_openvmm_arguments(caller) + self.assertEqual(arguments.count("--mount-owner"), 1) + self.assertEqual(arguments[-2:], ["--mount-owner", "caller"]) + + def test_aggregate_child_names_pin_their_targets(self): + name = sandbox.aggregate_child_name(0, "/workspace") + self.assertEqual(name, f"0-{hashlib.sha256(b'/workspace').hexdigest()[:32]}") + # Equal targets give equal names, so a restore names its children as + # the boot did, and another target or position gives another name. + self.assertEqual(sandbox.aggregate_child_name(0, "/workspace"), name) + self.assertNotEqual(sandbox.aggregate_child_name(0, "/workspace2"), name) + self.assertNotEqual(sandbox.aggregate_child_name(1, "/workspace"), name) + # Every name follows OpenVMM's rule for the name of a child. + for index, target in ((0, "/w"), (999, "/" + "x" * 1000)): + self.assertRegex( + sandbox.aggregate_child_name(index, target), + r"\A[0-9]+-[0-9a-f]{32}\Z", + ) + self.assertLessEqual(len(sandbox.aggregate_child_name(999, "/w")), 64) + + def test_launch_bounds_shares_by_the_kernel_command_line(self): + distro = sandbox.SandboxLayer.parse( + "distro,distro.erofs,11111111-1111-1111-1111-111111111111" + ) + + def launch(count: int) -> sandbox.SandboxLaunch: + return sandbox.SandboxLaunch( + layers=(distro,), + scratch=Path("s.ext4"), + mounts=tuple( + sandbox.SandboxMount.parse(f"/share/{index:02},host{index}") + for index in range(count) + ), + ) + + # No fixed number of shares is the limit, only the command line. + launch(10).kernel_command_line() + with self.assertRaisesRegex(common.ScriptError, "fewer --mount shares"): + launch(20).kernel_command_line() + + def test_launch_rejects_invalid_share_sets(self): + distro = sandbox.SandboxLayer.parse( + "distro,distro.erofs,11111111-1111-1111-1111-111111111111" + ) + + def mount(value: str, owner: str = "vmm") -> sandbox.SandboxMount: + return sandbox.SandboxMount.parse(value, (), owner) + + for mounts, message in ( + ((mount("/workspace,a,rw"), mount("/workspace,b")), "overlap"), + ((mount("/workspace,a,rw"), mount("/workspace/cache,b")), "overlap"), + ((mount("/opt/tools/node,a"), mount("/opt,b,rw")), "overlap"), + ( + (mount("/workspace,a,rw"), mount("/tools,b", "caller")), + "same --mount-owner", + ), + ): + with self.subTest(mounts=[item.guest_target for item in mounts]): + with self.assertRaisesRegex(common.ScriptError, message): + sandbox.SandboxLaunch( + layers=(distro,), scratch=Path("s.ext4"), mounts=mounts + ) + # A shared prefix that is not a path component is not an overlap. + sandbox.SandboxLaunch( + layers=(distro,), + scratch=Path("s.ext4"), + mounts=(mount("/work,a,rw"), mount("/workspace,b")), + ) + + def test_launch_validation_rejects_overlapping_or_foreign_host_paths(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + layer = root / "distro.erofs" + scratch = root / "scratch.ext4" + work = root / "work" + tools = root / "tools" + layer.write_bytes(b"layer") + scratch.write_bytes(b"scratch") + (work / "nested").mkdir(parents=True) + (tools / "credentials").mkdir(parents=True) + + def launch(*mounts: sandbox.SandboxMount) -> sandbox.SandboxLaunch: + return sandbox.SandboxLaunch( + layers=( + sandbox.SandboxLayer( + role="distro", + path=layer, + uuid="11111111-1111-1111-1111-111111111111", + ), + ), + scratch=scratch, + mounts=mounts, + ) + + workspace = sandbox.SandboxMount.parse(f"/workspace,{work},rw") + for host_path in (work / "nested", work, root): + with self.subTest(host_path=host_path): + with self.assertRaisesRegex(common.ScriptError, "overlap"): + launch( + workspace, + sandbox.SandboxMount.parse( + f"/opt/hostedtoolcache,{host_path}" + ), + ).validated() + + # Each denied path must be inside its own share. + foreign = sandbox.SandboxMount.parse( + f"/workspace,{work},rw", (os.fspath(tools / "credentials"),) + ) + toolcache = sandbox.SandboxMount.parse(f"/opt/hostedtoolcache,{tools}") + with self.assertRaisesRegex(common.ScriptError, "not inside"): + launch(foreign, toolcache).validated() + # A denied path may name a share's root, which OpenVMM then hides + # behind its allowed paths, but no allowed or writable path may. + launch( + sandbox.SandboxMount.parse( + f"/workspace,{work},rw", + (".",), + allowed_paths=(os.fspath(work / "nested"),), + ), + toolcache, + ).validated() + for allowed_paths, writable_paths in (((".",), ()), ((), (".",))): + with self.subTest(allowed=allowed_paths, writable=writable_paths): + with self.assertRaisesRegex(common.ScriptError, "not inside"): + launch( + sandbox.SandboxMount.parse( + f"/workspace,{work},rw", + ("nested",), + allowed_paths=allowed_paths, + writable_paths=writable_paths, + ), + toolcache, + ).validated() + + validated = launch( + sandbox.SandboxMount.parse(f"/workspace,{work},rw", ("nested",)), + sandbox.SandboxMount.parse( + f"/opt/hostedtoolcache,{tools}", + (os.fspath(tools / "credentials"),), + ), + ).validated() + self.assertEqual(len(validated.mounts), 2) + + # Resolving a path doesn't see through a bind mount, so one directory + # reached through two paths, or a share inside a bind mount of the + # other share's directory, is rejected by its file identities. + sandbox.validate_mount_host_paths((workspace, toolcache)) + with patch.object(sandbox, "_path_identity", return_value=(1, 1)): + with self.assertRaisesRegex(common.ScriptError, "same directory"): + sandbox.validate_mount_host_paths((workspace, toolcache)) + alias = root / "alias" + (alias / "cache").mkdir(parents=True) + + def identity(path: Path) -> tuple[int, int]: + # `alias` is a bind mount of the workspace's directory. + if path in (work.resolve(), alias.resolve()): + return 1, 1 + return 0, hash(path) + + cache = sandbox.SandboxMount.parse( + f"/opt/hostedtoolcache,{alias / 'cache'}" + ) + with patch.object(sandbox, "_path_identity", side_effect=identity): + sandbox.validate_mount_host_paths((workspace, toolcache)) + for mounts in ((workspace, cache), (cache, workspace)): + with self.assertRaisesRegex(common.ScriptError, "inside the other"): + sandbox.validate_mount_host_paths(mounts) + + def test_launch_validation_requires_plain_mount_directory(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + layer = root / "distro.erofs" + scratch = root / "scratch.ext4" + share = root / "share" + layer.write_bytes(b"layer") + scratch.write_bytes(b"scratch") + + def launch(host_path: Path) -> sandbox.SandboxLaunch: + return sandbox.SandboxLaunch( + layers=( + sandbox.SandboxLayer( + role="distro", + path=layer, + uuid="11111111-1111-1111-1111-111111111111", + ), + ), + scratch=scratch, + mounts=( + sandbox.SandboxMount( + guest_target="/workspace", host_path=host_path + ), + ), + ) + + with self.assertRaisesRegex(common.ScriptError, "plain directory"): + launch(share).validated() + share.write_bytes(b"file") + with self.assertRaisesRegex(common.ScriptError, "plain directory"): + launch(share).validated() + share.unlink() + share.mkdir() + validated = launch(share).validated() + self.assertEqual(validated.mounts[0].host_path, share) + + def test_sandbox_command_forwards_mount_to_openvmm(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + layer = root / "distro.erofs" + scratch = root / "scratch.ext4" + share = root / "share" + layer.write_bytes(b"distro") + scratch.write_bytes(b"scratch") + share.mkdir() + args = nvx.parse_args( + [ + "sandbox", + "--layer", + f"distro,{layer},11111111-1111-1111-1111-111111111111", + "--scratch", + str(scratch), + "--mount", + f"/workspace,{share},rw", + "--mount-deny", + "secrets", + "--dry-run", + ] + ) + + def require(path: Path, _description: str) -> Path: + return path + + with ( + patch.object(nvx, "require_file", side_effect=require), + patch.object( + nvx, "_format_command", return_value="formatted" + ) as format_command, + ): + nvx.command_sandbox(args) + + command = format_command.call_args.args[0] + self.assertEqual( + command[command.index("--mount") + 1], f"/workspace,{share},rw" + ) + self.assertEqual(command[command.index("--mount-deny") + 1], "secrets") + self.assertNotIn("--mount-owner", command) + + def test_sandbox_command_shares_a_file(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + layer = root / "distro.erofs" + scratch = root / "scratch.ext4" + settings = root / "settings.json" + layer.write_bytes(b"distro") + scratch.write_bytes(b"scratch") + settings.write_bytes(b"{}") + + def sandbox_args(*mount: str) -> argparse.Namespace: + return nvx.parse_args( + [ + "sandbox", + "--layer", + f"distro,{layer},11111111-1111-1111-1111-111111111111", + "--scratch", + str(scratch), + *mount, + "--dry-run", + ] + ) + + def require(path: Path, _description: str) -> Path: + return path + + with ( + patch.object(nvx, "require_file", side_effect=require), + patch.object( + nvx, "_format_command", return_value="formatted" + ) as format_command, + ): + nvx.command_sandbox( + sandbox_args("--mount", f"/etc/app/settings.json,{settings},ro") + ) + command = format_command.call_args.args[0] + name = sandbox.aggregate_child_name(0, "/etc/app/settings.json") + # The regular file is a child of its own, without its directory, + # which OpenVMM must find to be a regular file. + self.assertNotIn("--mount", command) + self.assertEqual( + command[command.index("--mount-child") + 1], + f"{name},{settings},ro,file", + ) + self.assertIn( + f"nvx_share={name},/etc/app/settings.json,ro,file", + command[command.index("--cmdline") + 1], + ) + with self.assertRaisesRegex(common.ScriptError, "shares a file"): + nvx.command_sandbox( + sandbox_args( + "--mount", + f"/etc/app/settings.json,{settings},rw", + "--mount-write", + str(settings), + ) + ) + + def test_run_attaches_a_file_as_an_aggregate_child(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + settings = root / "settings.json" + settings.write_bytes(b"{}") + work = root / "work" + work.mkdir() + with ( + patch.object(nvx, "require_file", return_value=Path("artifact")), + patch.object( + nvx, "_format_command", return_value="formatted" + ) as format_command, + ): + nvx.command_run( + nvx.parse_args( + [ + "run", + "--mount", + f"/config/settings.json,{settings},ro", + "--dry-run", + ] + ) + ) + command = format_command.call_args.args[0] + name = sandbox.aggregate_child_name(0, "/config/settings.json") + # Even alone, a file is the child of an aggregate. + self.assertNotIn("--mount", command) + self.assertEqual( + command[command.index("--mount-aggregate") + 1], "/run/nvx/shares" + ) + self.assertEqual( + command[command.index("--mount-child") + 1], + f"{name},{settings},ro,file", + ) + self.assertEqual( + command[command.index("--cmdline") + 1], + f"nvx_share={name},/config/settings.json,ro,file", + ) + + # A directory and a file share the aggregate, and only the + # file's token is marked. + nvx.command_run( + nvx.parse_args( + [ + "run", + "--mount", + f"/work,{work},rw", + "--mount", + f"/config/settings.json,{settings},ro", + "--dry-run", + ] + ) + ) + command = format_command.call_args.args[0] + work_name, settings_name = ( + sandbox.aggregate_child_name(index, target) + for index, target in enumerate(("/work", "/config/settings.json")) + ) + self.assertEqual( + command[command.index("--cmdline") + 1], + f"nvx_share={work_name},/work,rw " + f"nvx_share={settings_name},/config/settings.json,ro,file", + ) + + for arguments, message in ( + (["--mount-deny", "secrets"], "absolute host path"), + (["--cmdline", "x" * 1024], "1024-byte"), + ): + with self.subTest(arguments=arguments): + invalid = nvx.parse_args( + [ + "run", + "--mount", + f"/config/settings.json,{settings},ro", + *arguments, + "--dry-run", + ] + ) + with self.assertRaisesRegex(common.ScriptError, message): + nvx.command_run(invalid) + invalid = nvx.parse_args( + ["run", f"--mount=/a b,{settings}", "--dry-run"] + ) + with self.assertRaisesRegex( + common.ScriptError, "invalid --mount target" + ): + nvx.command_run(invalid) + + def test_sandbox_command_forwards_mount_owner(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + layer = root / "distro.erofs" + scratch = root / "scratch.ext4" + share = root / "share" + layer.write_bytes(b"distro") + scratch.write_bytes(b"scratch") + share.mkdir() + args = nvx.parse_args( + [ + "sandbox", + "--layer", + f"distro,{layer},11111111-1111-1111-1111-111111111111", + "--scratch", + str(scratch), + "--mount", + f"/workspace,{share},rw", + "--mount-owner", + "caller", + "--dry-run", + ] + ) + + def require(path: Path, _description: str) -> Path: + return path + + with ( + patch.object(nvx, "require_file", side_effect=require), + patch.object( + nvx, "_format_command", return_value="formatted" + ) as format_command, + ): + if sys.platform == "win32": + with self.assertRaisesRegex(common.ScriptError, "Linux host"): + nvx.command_sandbox(args) + format_command.assert_not_called() + return + if share.stat().st_uid == 0: + self.skipTest("caller ownership refuses a root-owned share") + nvx.command_sandbox(args) + + command = format_command.call_args.args[0] + self.assertEqual(command[command.index("--mount-owner") + 1], "caller") + self.assertLess(command.index("--mount"), command.index("--mount-owner")) + + def test_sandbox_command_rejects_misplaced_mount_options(self): + deny_only = nvx.parse_args( + [ + "sandbox", + "--layer", + "distro,distro.erofs,11111111-1111-1111-1111-111111111111", + "--scratch", + "scratch.ext4", + "--mount-deny", + "secrets", + ] + ) + with self.assertRaisesRegex(common.ScriptError, "requires --mount"): + nvx.command_sandbox(deny_only) + owner_only = nvx.parse_args( + [ + "sandbox", + "--layer", + "distro,distro.erofs,11111111-1111-1111-1111-111111111111", + "--scratch", + "scratch.ext4", + "--mount-owner", + "caller", + ] + ) + with self.assertRaisesRegex(common.ScriptError, "--mount-owner requires"): + nvx.command_sandbox(owner_only) + for operation in ("start", "exec", "stop", "deprovision"): + with self.subTest(operation=operation): + args = nvx.parse_args( + [ + "sandbox", + operation, + "--state-dir", + "state", + "--mount", + "/workspace,share,rw", + ] + ) + with self.assertRaisesRegex(common.ScriptError, "only valid"): + nvx.command_sandbox(args) + + def test_sandbox_command_attributes_policy_paths_to_the_preceding_mount(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + layer = root / "distro.erofs" + scratch = root / "scratch.ext4" + work = root / "work" + tools = root / "tools" + layer.write_bytes(b"distro") + scratch.write_bytes(b"scratch") + work.mkdir() + tools.mkdir() + common_args = [ + "sandbox", + "--layer", + f"distro,{layer},11111111-1111-1111-1111-111111111111", + "--scratch", + str(scratch), + ] + args = nvx.parse_args( + [ + *common_args, + "--mount", + f"/workspace,{work},rw", + "--mount-deny", + "logs", + "--mount-allow", + "logs/payloads", + "--mount-write", + "out", + "--mount", + f"/opt/hostedtoolcache,{tools},ro", + "--mount-deny", + "credentials", + "--dry-run", + ] + ) + workspace, toolcache = nvx._sandbox_mounts(args) + self.assertEqual( + ( + workspace.denied_paths, + workspace.allowed_paths, + workspace.writable_paths, + ), + (("logs",), ("logs/payloads",), ("out",)), + ) + self.assertEqual( + ( + toolcache.denied_paths, + toolcache.allowed_paths, + toolcache.writable_paths, + ), + (("credentials",), (), ()), + ) + + def require(path: Path, _description: str) -> Path: + return path + + with ( + patch.object(nvx, "require_file", side_effect=require), + patch.object( + nvx, "_format_command", return_value="formatted" + ) as format_command, + ): + nvx.command_sandbox(args) + + command = format_command.call_args.args[0] + + def values(option: str) -> list[str]: + return [ + command[index + 1] + for index, value in enumerate(command) + if value == option + ] + + # With several shares, OpenVMM attributes absolute paths. + self.assertEqual( + values("--mount-allow"), [os.fspath(work / "logs" / "payloads")] + ) + self.assertEqual(values("--mount-write"), [os.fspath(work / "out")]) + + for arguments, message in ( + (["--mount-write", "out"], "--mount-write requires --mount"), + # A --mount-write after the read-only share would narrow it. + ( + [ + "--mount", + f"/workspace,{work},rw", + "--mount", + f"/opt/hostedtoolcache,{tools},ro", + "--mount-write", + "cache", + ], + "is read-only", + ), + ( + [ + "--mount-allow", + "logs", + "--mount", + f"/workspace,{work},rw", + "--mount", + f"/opt/hostedtoolcache,{tools}", + ], + "each --mount-allow must follow the --mount", + ), + # A path in one share cannot be attributed to the other. + ( + [ + "--mount", + f"/workspace,{work},rw", + "--mount-write", + os.fspath(tools / "cache"), + "--mount", + f"/opt/hostedtoolcache,{tools}", + ], + "--mount-write .* is not inside the host directory", + ), + # Only a denied path may name a share's root. + ( + [ + "--mount", + f"/workspace,{work},rw", + "--mount-deny", + "logs", + "--mount-allow", + os.fspath(work), + "--mount", + f"/opt/hostedtoolcache,{tools}", + ], + "--mount-allow .* is not inside the host directory", + ), + ( + [ + "--mount", + f"/workspace,{work},rw", + "--mount-write", + os.fspath(work), + "--mount", + f"/opt/hostedtoolcache,{tools}", + ], + "--mount-write .* is not inside the host directory", + ), + ): + with self.subTest(message=message): + with self.assertRaisesRegex(common.ScriptError, message): + nvx.command_sandbox(nvx.parse_args([*common_args, *arguments])) + + def test_sandbox_command_hides_a_share_root_behind_its_allowed_paths(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + layer = root / "distro.erofs" + scratch = root / "scratch.ext4" + work = root / "work" + tools = root / "tools" + layer.write_bytes(b"distro") + scratch.write_bytes(b"scratch") + (work / "src").mkdir(parents=True) + tools.mkdir() + args = nvx.parse_args( + [ + "sandbox", + "--layer", + f"distro,{layer},11111111-1111-1111-1111-111111111111", + "--scratch", + str(scratch), + "--mount", + f"/workspace,{work},rw", + "--mount-deny", + os.fspath(work), + "--mount-allow", + os.fspath(work / "src"), + "--mount", + f"/opt/hostedtoolcache,{tools}", + "--dry-run", + ] + ) + + def require(path: Path, _description: str) -> Path: + return path + + with ( + patch.object(nvx, "require_file", side_effect=require), + patch.object( + nvx, "_format_command", return_value="formatted" + ) as format_command, + ): + nvx.command_sandbox(args) + + command = format_command.call_args.args[0] + + def values(option: str) -> list[str]: + return [ + command[index + 1] + for index, value in enumerate(command) + if value == option + ] + + # OpenVMM hides the workspace's root behind its one allowed path. + workspace = sandbox.aggregate_child_name(0, "/workspace") + toolcache = sandbox.aggregate_child_name(1, "/opt/hostedtoolcache") + self.assertEqual( + values("--mount-child"), + [f"{workspace},{work},rw", f"{toolcache},{tools},ro"], + ) + self.assertEqual(values("--mount-deny"), [os.fspath(work)]) + self.assertEqual(values("--mount-allow"), [os.fspath(work / "src")]) + + def test_sandbox_command_attributes_mount_deny_to_the_preceding_mount(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + layer = root / "distro.erofs" + scratch = root / "scratch.ext4" + work = root / "work" + tools = root / "tools" + layer.write_bytes(b"distro") + scratch.write_bytes(b"scratch") + work.mkdir() + tools.mkdir() + common_args = [ + "sandbox", + "--layer", + f"distro,{layer},11111111-1111-1111-1111-111111111111", + "--scratch", + str(scratch), + ] + args = nvx.parse_args( + [ + *common_args, + "--mount", + f"/workspace,{work},rw", + "--mount-deny", + "secrets", + "--mount", + f"/opt/hostedtoolcache,{tools},ro", + "--mount-deny", + "credentials", + "--mount-deny", + "logs", + "--dry-run", + ] + ) + mounts = nvx._sandbox_mounts(args) + self.assertEqual( + [(mount.guest_target, mount.denied_paths) for mount in mounts], + [ + ("/workspace", ("secrets",)), + ("/opt/hostedtoolcache", ("credentials", "logs")), + ], + ) + + def require(path: Path, _description: str) -> Path: + return path + + with ( + patch.object(nvx, "require_file", side_effect=require), + patch.object( + nvx, "_format_command", return_value="formatted" + ) as format_command, + ): + nvx.command_sandbox(args) + + command = format_command.call_args.args[0] + workspace = sandbox.aggregate_child_name(0, "/workspace") + toolcache = sandbox.aggregate_child_name(1, "/opt/hostedtoolcache") + self.assertNotIn("--mount", command) + self.assertEqual( + [ + command[index + 1] + for index, value in enumerate(command) + if value in ("--mount-aggregate", "--mount-child") + ], + [ + "/run/nvx/shares", + f"{workspace},{work},rw", + f"{toolcache},{tools},ro", + ], + ) + self.assertEqual( + [ + command[index + 1] + for index, value in enumerate(command) + if value == "--mount-deny" + ], + [ + os.fspath(work / "secrets"), + os.fspath(tools / "credentials"), + os.fspath(tools / "logs"), + ], + ) + cmdline = command[command.index("--cmdline") + 1] + self.assertIn( + f"nvx_share={workspace},/workspace,rw " + f"nvx_share={toolcache},/opt/hostedtoolcache,ro", + cmdline, + ) + + # With one share, a --mount-deny may come first; with several, it + # must follow the share whose directory it hides. + single = nvx.parse_args( + [*common_args, "--mount-deny", "secrets", "--mount", f"/w,{work},rw"] + ) + self.assertEqual(nvx._sandbox_mounts(single)[0].denied_paths, ("secrets",)) + leading = nvx.parse_args( + [ + *common_args, + "--mount-deny", + "secrets", + "--mount", + f"/workspace,{work},rw", + "--mount", + f"/opt/hostedtoolcache,{tools}", + ] + ) + with self.assertRaisesRegex(common.ScriptError, "must follow the --mount"): + nvx.command_sandbox(leading) + # More shares are children of the same aggregate. + (root / "cache").mkdir() + three = nvx.parse_args( + [ + *common_args, + "--mount", + f"/a,{work}", + "--mount", + f"/b,{tools}", + "--mount", + f"/c,{root / 'cache'}", + "--dry-run", + ] + ) + with ( + patch.object(nvx, "require_file", side_effect=require), + patch.object( + nvx, "_format_command", return_value="formatted" + ) as format_command, + ): + nvx.command_sandbox(three) + command = format_command.call_args.args[0] + self.assertEqual(command.count("--mount-child"), 3) + + def test_run_attaches_several_mounts_as_an_aggregate(self): + args = nvx.parse_args( + [ + "run", + "--mount", + "/workspace,work,rw", + "--mount", + "/opt/hostedtoolcache,tools", + "--mount", + "/srv/cache,cache,ro", + "--cmdline", + "quiet", + "--dry-run", + ] + ) + with ( + patch.object(nvx, "require_file", return_value=Path("artifact")), + patch.object( + nvx, "_format_command", return_value="formatted" + ) as format_command, + ): + nvx.command_run(args) + command = format_command.call_args.args[0] + workspace, toolcache, cache = ( + sandbox.aggregate_child_name(index, target) + for index, target in enumerate( + ("/workspace", "/opt/hostedtoolcache", "/srv/cache") + ) + ) + booted = [ + "/run/nvx/shares", + f"{workspace},work,rw", + f"{toolcache},tools,ro", + f"{cache},cache,ro", + ] + self.assertNotIn("--mount", command) + self.assertEqual( + [ + command[index + 1] + for index, value in enumerate(command) + if value in ("--mount-aggregate", "--mount-child") + ], + booted, + ) + self.assertEqual( + command[command.index("--cmdline") + 1], + f"quiet nvx_share={workspace},/workspace,rw " + f"nvx_share={toolcache},/opt/hostedtoolcache,ro " + f"nvx_share={cache},/srv/cache,ro", + ) + + # One share stays a plain --mount without bind tokens. + single = nvx.parse_args( + ["run", "--mount", "/workspace,work,rw", "--dry-run"] + ) + nvx.command_run(single) + command = format_command.call_args.args[0] + self.assertEqual( + command[command.index("--mount") :], ["--mount", "/workspace,work,rw"] + ) + self.assertNotIn("--cmdline", command) + + # A restored guest keeps its binds, so only the devices are named, + # with the children's names of the boot: they pin the targets that + # the guest bound, and another target names another child. + restored = nvx.parse_args( + [ + "run", + "--restore-snapshot", + "snapshot", + "--mount", + "/workspace,work,rw", + "--mount", + "/opt/hostedtoolcache,tools", + "--mount", + "/srv/cache,cache,ro", + "--dry-run", + ] + ) + nvx.command_run(restored) + command = format_command.call_args.args[0] + self.assertEqual( + [ + command[index + 1] + for index, value in enumerate(command) + if value in ("--mount-aggregate", "--mount-child") + ], + booted, + ) + self.assertNotIn("--cmdline", command) + retargeted = nvx.parse_args( + [ + "run", + "--restore-snapshot", + "snapshot", + "--mount", + "/workspace,work,rw", + "--mount", + "/opt/tools,tools", + "--mount", + "/srv/cache,cache,ro", + "--dry-run", + ] + ) + nvx.command_run(retargeted) + command = format_command.call_args.args[0] + children = [ + command[index + 1] + for index, value in enumerate(command) + if value == "--mount-child" + ] + self.assertEqual(children[0::2], booted[1::2]) + self.assertNotEqual(children[1], booted[2]) + self.assertEqual(children[1].split(",", 1)[1], "tools,ro") + + for mounts, message in ( + (["/a,a,rx"], "unsupported --mount mode"), + (["/workspace,a", "/workspace/cache,b"], "overlap"), + (["/run/nvx,a", "/b,b"], "overlap"), + (["/a b,a", "/b,b"], "invalid --mount target"), + (["relative,a", "/b,b"], "invalid --mount target"), + ([f"/share/{index:02},host" for index in range(60)], "1024-byte"), + ): + with self.subTest(mounts=mounts[:2]): + invalid = nvx.parse_args( + ["run", *[f"--mount={mount}" for mount in mounts], "--dry-run"] + ) + with self.assertRaisesRegex(common.ScriptError, message): + nvx.command_run(invalid) + relative = nvx.parse_args( + [ + "run", + "--mount", + "/a,a", + "--mount", + "/b,b", + "--mount-deny", + "secrets", + "--dry-run", + ] + ) + with self.assertRaisesRegex(common.ScriptError, "absolute host path"): + nvx.command_run(relative) + + def test_managed_lifecycle_persists_and_replays_two_shares(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary).resolve() + layer_path = root / "distro.erofs" + scratch_path = root / "scratch.ext4" + work = root / "work" + tools = root / "tools" + layer_path.write_bytes(b"layer") + scratch_path.write_bytes(b"scratch") + work.mkdir() + tools.mkdir() + state = root / "state" + sandbox_lifecycle.provision( + state, + sandbox.SandboxLaunch( + layers=( + sandbox.SandboxLayer( + role="distro", + path=layer_path, + uuid="11111111-1111-1111-1111-111111111111", + ), + ), + scratch=scratch_path, + mounts=( + sandbox.SandboxMount.parse( + f"/workspace,{work},rw", ("secrets",) + ), + sandbox.SandboxMount.parse(f"/opt/hostedtoolcache,{tools}"), + ), + ), + hypervisor="whp", + memory_mib=256, + net=None, + network_profile=None, + network_egress=None, + network_ingress=None, + network_egress_allow=(), + network_egress_deny=(), + host_loopback=None, + network_proxy=None, + host_loopback_forward=(), + cmdline="quiet", + timeout=10, + ) + + config = json.loads( + (state / sandbox_lifecycle.CONFIG_NAME).read_text(encoding="utf-8") + ) + self.assertEqual( + config["format"], sandbox_lifecycle.MULTI_MOUNT_CONFIG_FORMAT + ) + # Older releases know only the single `mount`, and reject format 4. + self.assertNotIn("mount", config) + with self.assertRaisesRegex(common.ScriptError, "unsupported format"): + sandbox_lifecycle._read_json( + state / sandbox_lifecycle.CONFIG_NAME, + "sandbox configuration", + version=(1, 2, 3), + ) + self.assertEqual( + config["mounts"], + [ + { + "guest_target": "/workspace", + "host_path": os.fspath(work), + "access": "rw", + "denied_paths": ["secrets"], + "owner": "vmm", + }, + { + "guest_target": "/opt/hostedtoolcache", + "host_path": os.fspath(tools), + "access": "ro", + "denied_paths": [], + "owner": "vmm", + }, + ], + ) + + process = MagicMock() + process.pid = 123 + process.stdin = io.BytesIO() + + def require(path: Path, _description: str) -> Path: + return path + + with ( + patch.object(sandbox_lifecycle, "require_file", side_effect=require), + patch.object( + sandbox_lifecycle.subprocess, "Popen", return_value=process + ) as popen, + patch.object( + sandbox_lifecycle, "_process_start_time", return_value=456 + ), + patch.object( + sandbox_lifecycle.ControlSession, + "connect", + return_value=MagicMock(), + ), + ): + sandbox_lifecycle.start(state, 10) + + command = popen.call_args.args[0] + self.assertEqual( + [ + command[index + 1] + for index, value in enumerate(command) + if value in ("--mount-aggregate", "--mount-child", "--mount-deny") + ], + [ + "/run/nvx/shares", + f"{sandbox.aggregate_child_name(0, '/workspace')},{work},rw", + f"{sandbox.aggregate_child_name(1, '/opt/hostedtoolcache')}," + f"{tools},ro", + os.fspath(work / "secrets"), + ], + ) + + def test_managed_lifecycle_persists_and_replays_the_access_policy(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary).resolve() + layer_path = root / "distro.erofs" + scratch_path = root / "scratch.ext4" + work = root / "work" + layer_path.write_bytes(b"layer") + scratch_path.write_bytes(b"scratch") + work.mkdir() + state = root / "state" + sandbox_lifecycle.provision( + state, + sandbox.SandboxLaunch( + layers=( + sandbox.SandboxLayer( + role="distro", + path=layer_path, + uuid="11111111-1111-1111-1111-111111111111", + ), + ), + scratch=scratch_path, + mounts=( + sandbox.SandboxMount.parse( + f"/workspace,{work},rw", + ("logs",), + allowed_paths=("logs/payloads",), + writable_paths=("out",), + ), + ), + ), + hypervisor="whp", + memory_mib=256, + net=None, + network_profile=None, + network_egress=None, + network_ingress=None, + network_egress_allow=(), + network_egress_deny=(), + host_loopback=None, + network_proxy=None, + host_loopback_forward=(), + cmdline="quiet", + timeout=10, + ) + + config = json.loads( + (state / sandbox_lifecycle.CONFIG_NAME).read_text(encoding="utf-8") + ) + self.assertEqual(config["format"], sandbox_lifecycle.POLICY_CONFIG_FORMAT) + # Earlier releases read at most format 4, which would start the + # share without its policy, so they must reject format 5. + self.assertNotIn("mount", config) + with self.assertRaisesRegex(common.ScriptError, "unsupported format"): + sandbox_lifecycle._read_json( + state / sandbox_lifecycle.CONFIG_NAME, + "sandbox configuration", + version=(1, 2, 3, 4), + ) + self.assertEqual( + config["mounts"], + [ + { + "guest_target": "/workspace", + "host_path": os.fspath(work), + "access": "rw", + "denied_paths": ["logs"], + "allowed_paths": ["logs/payloads"], + "writable_paths": ["out"], + "owner": "vmm", + }, + ], + ) + + process = MagicMock() + process.pid = 123 + process.stdin = io.BytesIO() + + def require(path: Path, _description: str) -> Path: + return path + + with ( + patch.object(sandbox_lifecycle, "require_file", side_effect=require), + patch.object( + sandbox_lifecycle.subprocess, "Popen", return_value=process + ) as popen, + patch.object( + sandbox_lifecycle, "_process_start_time", return_value=456 + ), + patch.object( + sandbox_lifecycle.ControlSession, + "connect", + return_value=MagicMock(), + ), + ): + sandbox_lifecycle.start(state, 10) + + command = popen.call_args.args[0] + self.assertEqual( + [ + (value, command[index + 1]) + for index, value in enumerate(command) + if value.startswith("--mount") + ], + [ + ("--mount", f"/workspace,{work},rw"), + ("--mount-deny", "logs"), + ("--mount-allow", "logs/payloads"), + ("--mount-write", "out"), + ], + ) + + # Format 5 holds an access policy, and only format 5 does. + config["mounts"][0]["allowed_paths"] = [] + config["mounts"][0]["writable_paths"] = [] + with ( + patch.object(sandbox, "require_file", side_effect=require), + self.assertRaisesRegex(common.ScriptError, "does not match"), + ): + sandbox_lifecycle._deserialize_launch(config) + + def test_launch_contract_attaches_a_file_as_an_aggregate_child(self): + distro = sandbox.SandboxLayer.parse( + "distro,distro.erofs,11111111-1111-1111-1111-111111111111" + ) + settings = sandbox.SandboxMount.parse( + "/etc/app/settings.json,settings.json,ro", kind="file" + ) + launch = sandbox.SandboxLaunch( + layers=(distro,), scratch=Path("scratch.ext4"), mounts=(settings,) + ) + name = sandbox.aggregate_child_name(0, "/etc/app/settings.json") + # Even alone, a file is the child of an aggregate, because the guest + # mounts the device's root, which is a directory. + self.assertTrue(sandbox.mounts_use_aggregate(launch.mounts)) + arguments = launch.openvmm_arguments() + self.assertNotIn("--mount", arguments) + self.assertEqual( + arguments[arguments.index("--mount-aggregate") :], + [ + "--mount-aggregate", + "/run/nvx/shares", + "--mount-child", + f"{name},{os.fspath(Path('settings.json'))},ro,file", + ], + ) + self.assertEqual( + sandbox.mounts_command_line_fragment(launch.mounts), + " virtfs_dir=/run/nvx/shares virtfs_tag=microvm virtfs_mode=ro" + " virtfs_aggregate=1", + ) + self.assertTrue( + launch.kernel_command_line().endswith( + f" nvx_share={name},/etc/app/settings.json,ro,file" + ) + ) + + # Files and directories share one aggregate, and only a file's token + # is marked. + workspace = sandbox.SandboxMount.parse("/workspace,work,rw", ("secrets",)) + output = sandbox.SandboxMount.parse( + "/results/output.txt,output.txt,rw", kind="file" + ) + mixed = sandbox.SandboxLaunch( + layers=(distro,), + scratch=Path("scratch.ext4"), + mounts=(workspace, settings, output), + ) + names = [ + sandbox.aggregate_child_name(index, mount.guest_target) + for index, mount in enumerate(mixed.mounts) + ] + self.assertTrue( + mixed.kernel_command_line().endswith( + f" nvx_share={names[0]},/workspace,rw" + f" nvx_share={names[1]},/etc/app/settings.json,ro,file" + f" nvx_share={names[2]},/results/output.txt,rw,file" + ) + ) + arguments = mixed.openvmm_arguments() + self.assertEqual( + arguments[arguments.index("--mount-deny") + 1], + os.fspath(Path.cwd() / "work" / "secrets"), + ) + + # A file's mode applies to the whole file, so no policy path names + # one, and the runtime owns the machine ID's path. + for option in ("denied_paths", "allowed_paths", "writable_paths"): + policy: dict[str, Any] = {option: ("x",)} + with self.subTest(option=option): + with self.assertRaisesRegex(common.ScriptError, "shares a file"): + sandbox.SandboxMount( + guest_target="/results/output.txt", + host_path=Path("output.txt"), + access="rw", + kind="file", + **policy, + ) + with self.assertRaisesRegex( + common.ScriptError, "unsupported sandbox mount kind" + ): + sandbox.SandboxMount.parse("/a,a", kind="socket") + # The runtime replaces /etc/machine-id with a file, so nothing at or + # below it may be shared. + for target in ("/etc/machine-id", "/etc/machine-id/child"): + with self.subTest(target=target): + with self.assertRaisesRegex(common.ScriptError, "reserved"): + sandbox.SandboxMount.parse(f"{target},id", kind="file") + # Only /etc itself is reserved, not what lies below it. + sandbox.SandboxMount.parse("/etc/app,app") + + def test_launch_validation_requires_the_kind_of_each_share(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + config = root / "config" + config.mkdir() + settings = config / "settings.json" + settings.write_bytes(b"{}") + self.assertEqual(sandbox.mount_kind(settings), "file") + self.assertEqual(sandbox.mount_kind(config), "directory") + self.assertEqual(sandbox.mount_kind(root / "missing"), "directory") + share = sandbox.SandboxMount( + guest_target="/etc/app/settings.json", host_path=settings, kind="file" + ) + self.assertIs(share.validated(), share) + # A directory at the file's path is not shared in its place. + settings.unlink() + settings.mkdir() + with self.assertRaisesRegex(common.ScriptError, "not a regular file"): + share.validated() + settings.rmdir() + with self.assertRaisesRegex(common.ScriptError, "not a regular file"): + share.validated() + + # A file overlaps a shared directory that contains it, and another + # share of the same file, such as through a hard link, but not a + # file beside it. + settings.write_bytes(b"{}") + other = config / "other.json" + other.write_bytes(b"{}") + link = root / "link.json" + os.link(settings, link) + + def file_share(target: str, path: Path) -> sandbox.SandboxMount: + return sandbox.SandboxMount( + guest_target=target, host_path=path, kind="file" + ) + + sandbox.validate_mount_host_paths( + (file_share("/a.json", settings), file_share("/b.json", other)) + ) + for mounts, message in ( + ( + ( + sandbox.SandboxMount(guest_target="/config", host_path=config), + file_share("/a.json", settings), + ), + "overlap", + ), + ( + (file_share("/a.json", settings), file_share("/b.json", link)), + "same directory or file", + ), + ): + with self.subTest(message=message): + with self.assertRaisesRegex(common.ScriptError, message): + sandbox.validate_mount_host_paths(mounts) + + def test_managed_configuration_format_records_file_shares(self): + def entry(target: str, path: str, kind: str) -> dict[str, object]: + return { + "guest_target": target, + "host_path": path, + "access": "ro", + "denied_paths": [], + "owner": "vmm", + "allowed_paths": [], + "writable_paths": [], + "kind": kind, + } + + config: dict[str, object] = { + "format": sandbox_lifecycle.FILE_MOUNT_CONFIG_FORMAT, + "layers": [ + { + "role": "distro", + "path": "distro.erofs", + "uuid": "11111111-1111-1111-1111-111111111111", + } + ], + "scratch": "scratch.ext4", + "hostname": "nvx-sandbox", + "workload_uid": 65534, + "workload_gid": 65534, + "memory_max": None, + "pids_max": None, + "mounts": [entry("/etc/app/settings.json", "settings.json", "file")], + } + + def require(path: Path, _description: str) -> Path: + return path + + def unchanged(mount: sandbox.SandboxMount) -> sandbox.SandboxMount: + return mount + + with ( + patch.object(sandbox, "require_file", side_effect=require), + patch.object(sandbox.SandboxMount, "validated", unchanged), + ): + launch = sandbox_lifecycle._deserialize_launch(config) + self.assertEqual([mount.kind for mount in launch.mounts], ["file"]) + serialized = sandbox_lifecycle._serialize_launch( + launch, + hypervisor="kvm", + memory_mib=256, + net=None, + network_profile=None, + network_egress=None, + network_ingress=None, + network_egress_allow=(), + network_egress_deny=(), + host_loopback=None, + network_proxy=None, + host_loopback_forward=(), + cmdline="", + ) + self.assertEqual( + serialized["format"], sandbox_lifecycle.FILE_MOUNT_CONFIG_FORMAT + ) + self.assertEqual(serialized["mounts"][0]["kind"], "file") + # Only format 6 holds a file share, and every format-6 share + # records its kind. + config["mounts"] = [entry("/workspace", "work", "directory")] + with self.assertRaisesRegex(common.ScriptError, "does not match"): + sandbox_lifecycle._deserialize_launch(config) + config["mounts"] = [{**entry("/a.json", "a.json", "file"), "kind": None}] + with self.assertRaisesRegex(common.ScriptError, "kind"): + sandbox_lifecycle._deserialize_launch(config) + unmarked = entry("/a.json", "a.json", "file") + del unmarked["kind"] + config["mounts"] = [unmarked] + with self.assertRaisesRegex(common.ScriptError, "malformed"): + sandbox_lifecycle._deserialize_launch(config) + config["format"] = sandbox_lifecycle.POLICY_CONFIG_FORMAT + config["mounts"] = [entry("/a.json", "a.json", "file")] + with self.assertRaisesRegex(common.ScriptError, "does not match"): + sandbox_lifecycle._deserialize_launch(config) + + def test_managed_configuration_format_binds_several_mounts(self): + def entry(target: str, path: str, owner: str = "vmm") -> dict[str, object]: + return { + "guest_target": target, + "host_path": path, + "access": "rw", + "denied_paths": [], + "owner": owner, + } + + config: dict[str, object] = { + "format": sandbox_lifecycle.MULTI_MOUNT_CONFIG_FORMAT, + "layers": [ + { + "role": "distro", + "path": "distro.erofs", + "uuid": "11111111-1111-1111-1111-111111111111", + } + ], + "scratch": "scratch.ext4", + "hostname": "nvx-sandbox", + "workload_uid": 65534, + "workload_gid": 65534, + "memory_max": None, + "pids_max": None, + "mounts": [entry("/workspace", "work"), entry("/tools", "tools")], + } + + def require(path: Path, _description: str) -> Path: + return path + + def unchanged(mount: sandbox.SandboxMount) -> sandbox.SandboxMount: + return mount + + def identity(path: Path) -> tuple[int, int]: + return 0, hash(path) + + with ( + patch.object(sandbox, "require_file", side_effect=require), + patch.object(sandbox.SandboxMount, "validated", unchanged), + patch.object(sandbox, "_path_identity", side_effect=identity), + ): + launch = sandbox_lifecycle._deserialize_launch(config) + self.assertEqual( + [mount.guest_target for mount in launch.mounts], + ["/workspace", "/tools"], + ) + # Format 4 always holds several shares, and only format 4 does. + config["mounts"] = [entry("/workspace", "work")] + with self.assertRaisesRegex(common.ScriptError, "does not match"): + sandbox_lifecycle._deserialize_launch(config) + config["mounts"] = {"guest_target": "/workspace"} + with self.assertRaisesRegex(common.ScriptError, "malformed"): + sandbox_lifecycle._deserialize_launch(config) + config["mounts"] = [entry("/workspace", "work"), entry("/a", "a", "caller")] + with self.assertRaisesRegex(common.ScriptError, "same --mount-owner"): + sandbox_lifecycle._deserialize_launch(config) + config["mounts"] = [entry("/workspace", "work"), entry("/workspace", "b")] + with self.assertRaisesRegex(common.ScriptError, "overlap"): + sandbox_lifecycle._deserialize_launch(config) + config["format"] = sandbox_lifecycle.MOUNT_CONFIG_FORMAT + config["mounts"] = [entry("/workspace", "work"), entry("/tools", "tools")] + with self.assertRaisesRegex(common.ScriptError, "does not match"): + sandbox_lifecycle._deserialize_launch(config) + + def test_managed_lifecycle_persists_and_replays_mount(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary).resolve() + layer_path = root / "distro.erofs" + scratch_path = root / "scratch.ext4" + share = root / "share" + layer_path.write_bytes(b"layer") + scratch_path.write_bytes(b"scratch") + share.mkdir() + state = root / "state" + previous = Path.cwd() + os.chdir(root) + try: + sandbox_lifecycle.provision( + state, + sandbox.SandboxLaunch( + layers=( + sandbox.SandboxLayer( + role="distro", + path=layer_path, + uuid="11111111-1111-1111-1111-111111111111", + ), + ), + scratch=scratch_path, + mounts=( + sandbox.SandboxMount.parse( + "/workspace,share,rw", ("secrets",) + ), + ), + ), + hypervisor="whp", + memory_mib=256, + net=None, + network_profile=None, + network_egress=None, + network_ingress=None, + network_egress_allow=(), + network_egress_deny=(), + host_loopback=None, + network_proxy=None, + host_loopback_forward=(), + cmdline="quiet", + timeout=10, + ) + finally: + os.chdir(previous) + + config = json.loads( + (state / sandbox_lifecycle.CONFIG_NAME).read_text(encoding="utf-8") + ) + self.assertEqual(config["format"], sandbox_lifecycle.MOUNT_CONFIG_FORMAT) + with self.assertRaisesRegex(common.ScriptError, "unsupported format"): + sandbox_lifecycle._read_json( + state / sandbox_lifecycle.CONFIG_NAME, + "sandbox configuration", + version=1, + ) + self.assertEqual( + config["mount"], + { + "guest_target": "/workspace", + "host_path": os.fspath(share), + "access": "rw", + "denied_paths": ["secrets"], + "owner": "vmm", + }, + ) + + process = MagicMock() + process.pid = 123 + process.stdin = io.BytesIO() + context = MagicMock() + + def require(path: Path, _description: str) -> Path: + return path + + with ( + patch.object(sandbox_lifecycle, "require_file", side_effect=require), + patch.object( + sandbox_lifecycle.subprocess, "Popen", return_value=process + ) as popen, + patch.object( + sandbox_lifecycle, "_process_start_time", return_value=456 + ), + patch.object( + sandbox_lifecycle.ControlSession, "connect", return_value=context + ), + ): + sandbox_lifecycle.start(state, 10) + + command = popen.call_args.args[0] + self.assertEqual( + command[command.index("--mount") + 1], f"/workspace,{share},rw" + ) + self.assertEqual(command[command.index("--mount-deny") + 1], "secrets") + + def test_managed_lifecycle_accepts_configuration_without_mount(self): + config: dict[str, object] = { + "format": sandbox_lifecycle.CONFIG_FORMAT, + "layers": [ + { + "role": "distro", + "path": "distro.erofs", + "uuid": "11111111-1111-1111-1111-111111111111", + } + ], + "scratch": "scratch.ext4", + "hostname": "nvx-sandbox", + "workload_uid": 65534, + "workload_gid": 65534, + "memory_max": None, + "pids_max": None, + } + + def require(path: Path, _description: str) -> Path: + return path + + with ( + patch.object(sandbox, "require_file", side_effect=require), + ): + self.assertEqual(sandbox_lifecycle._deserialize_launch(config).mounts, ()) + config["format"] = sandbox_lifecycle.MOUNT_CONFIG_FORMAT + with self.assertRaisesRegex(common.ScriptError, "does not match"): + sandbox_lifecycle._deserialize_launch(config) + config["mount"] = {"guest_target": "/workspace"} + with self.assertRaisesRegex(common.ScriptError, "malformed"): + sandbox_lifecycle._deserialize_launch(config) + config["mount"] = { + "guest_target": "/proc", + "host_path": "share", + "access": "rw", + "denied_paths": [], + } + with self.assertRaisesRegex(common.ScriptError, "reserved"): + sandbox_lifecycle._deserialize_launch(config) + config["mount"] = { + "guest_target": "/workspace", + "host_path": "share", + "access": "rw", + "denied_paths": [], + } + config["format"] = sandbox_lifecycle.CONFIG_FORMAT + with self.assertRaisesRegex(common.ScriptError, "does not match"): + sandbox_lifecycle._deserialize_launch(config) + + def test_managed_configuration_format_binds_the_mount_owner(self): + mount: dict[str, object] = { + "guest_target": "/workspace", + "host_path": "share", + "access": "rw", + "denied_paths": [], + } + config: dict[str, object] = { + "format": sandbox_lifecycle.MOUNT_CONFIG_FORMAT, + "layers": [ + { + "role": "distro", + "path": "distro.erofs", + "uuid": "11111111-1111-1111-1111-111111111111", + } + ], + "scratch": "scratch.ext4", + "hostname": "nvx-sandbox", + "workload_uid": 65534, + "workload_gid": 65534, + "memory_max": None, + "pids_max": None, + "mount": mount, + } + + def require(path: Path, _description: str) -> Path: + return path + + def unchanged(mount: sandbox.SandboxMount) -> sandbox.SandboxMount: + return mount + + with ( + patch.object(sandbox, "require_file", side_effect=require), + patch.object(sandbox.SandboxMount, "validated", unchanged), + ): + # A share configured before ownership modes runs as the VMM. + launch = sandbox_lifecycle._deserialize_launch(config) + self.assertEqual(launch.mounts[0].owner, "vmm") + mount["owner"] = "vmm" + launch = sandbox_lifecycle._deserialize_launch(config) + self.assertEqual(launch.mounts[0].owner, "vmm") + # Only format 3, which older releases reject, carries caller ownership. + mount["owner"] = "caller" + with self.assertRaisesRegex(common.ScriptError, "does not match"): + sandbox_lifecycle._deserialize_launch(config) + config["format"] = sandbox_lifecycle.OWNER_CONFIG_FORMAT + launch = sandbox_lifecycle._deserialize_launch(config) + self.assertEqual(launch.mounts[0].owner, "caller") + mount["owner"] = "vmm" + with self.assertRaisesRegex(common.ScriptError, "does not match"): + sandbox_lifecycle._deserialize_launch(config) + mount["owner"] = "root" + with self.assertRaisesRegex( + common.ScriptError, "unsupported sandbox mount owner" + ): + sandbox_lifecycle._deserialize_launch(config) + + def test_managed_lifecycle_persists_and_replays_caller_owner(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary).resolve() + layer_path = root / "distro.erofs" + scratch_path = root / "scratch.ext4" + share = root / "share" + layer_path.write_bytes(b"layer") + scratch_path.write_bytes(b"scratch") + share.mkdir() + state = root / "state" + launch = sandbox.SandboxLaunch( + layers=( + sandbox.SandboxLayer( + role="distro", + path=layer_path, + uuid="11111111-1111-1111-1111-111111111111", + ), + ), + scratch=scratch_path, + mounts=( + sandbox.SandboxMount.parse(f"/workspace,{share},rw", (), "caller"), + ), + ) + + def provision() -> None: + sandbox_lifecycle.provision( + state, + launch, + hypervisor="whp" if sys.platform == "win32" else "kvm", + memory_mib=256, + net=None, + network_profile=None, + network_egress=None, + network_ingress=None, + network_egress_allow=(), + network_egress_deny=(), + host_loopback=None, + network_proxy=None, + host_loopback_forward=(), + cmdline="quiet", + timeout=10, + ) + + if sys.platform == "win32": + with self.assertRaisesRegex(common.ScriptError, "Linux host"): + provision() + self.assertFalse((state / sandbox_lifecycle.CONFIG_NAME).exists()) + return + if share.stat().st_uid == 0: + self.skipTest("caller ownership refuses a root-owned share") + provision() + + config = json.loads( + (state / sandbox_lifecycle.CONFIG_NAME).read_text(encoding="utf-8") + ) + self.assertEqual(config["format"], sandbox_lifecycle.OWNER_CONFIG_FORMAT) + self.assertEqual(config["mount"]["owner"], "caller") + # NVX releases that know only formats 1 and 2 refuse to start it. + with self.assertRaisesRegex(common.ScriptError, "unsupported format"): + sandbox_lifecycle._read_json( + state / sandbox_lifecycle.CONFIG_NAME, + "sandbox configuration", + version=( + sandbox_lifecycle.CONFIG_FORMAT, + sandbox_lifecycle.MOUNT_CONFIG_FORMAT, + ), + ) + + process = MagicMock() + process.pid = 123 + process.stdin = io.BytesIO() + + def require(path: Path, _description: str) -> Path: + return path + + with ( + patch.object(sandbox_lifecycle, "require_file", side_effect=require), + patch.object( + sandbox_lifecycle.subprocess, "Popen", return_value=process + ) as popen, + patch.object( + sandbox_lifecycle, "_process_start_time", return_value=456 + ), + patch.object( + sandbox_lifecycle.ControlSession, + "connect", + return_value=MagicMock(), + ), + ): + sandbox_lifecycle.start(state, 10) + + command = popen.call_args.args[0] + self.assertEqual(command[command.index("--mount-owner") + 1], "caller") + + def test_managed_lifecycle_provisions_and_deprovisions_state(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + layer_path = root / "distro.erofs" + scratch_path = root / "scratch.ext4" + layer_path.write_bytes(b"layer") + scratch_path.write_bytes(b"scratch") + launch = sandbox.SandboxLaunch( + layers=( + sandbox.SandboxLayer( + role="distro", + path=layer_path, + uuid="11111111-1111-1111-1111-111111111111", + ), + ), + scratch=scratch_path, + ) + state = root / "state" + + sandbox_lifecycle.provision( + state, + launch, + hypervisor="whp", + memory_mib=256, + net=None, + network_profile=None, + network_egress=None, + network_ingress=None, + network_egress_allow=(), + network_egress_deny=(), + host_loopback=None, + network_proxy=None, + host_loopback_forward=(), + cmdline="quiet", + timeout=10, + ) + + config = json.loads( + (state / sandbox_lifecycle.CONFIG_NAME).read_text(encoding="utf-8") + ) + self.assertEqual(config["format"], sandbox_lifecycle.CONFIG_FORMAT) + self.assertIsNone(config["mount"]) + self.assertEqual(config["workload_uid"], 65534) + self.assertEqual(config["hypervisor"], "whp") + self.assertFalse((state / sandbox_lifecycle.RUNTIME_NAME).exists()) + + config["network_egress_allow"] = None + (state / sandbox_lifecycle.CONFIG_NAME).write_text( + json.dumps(config), encoding="utf-8" + ) + + def require(path: Path, _description: str) -> Path: + return path + + with ( + patch.object( + sandbox_lifecycle, + "require_file", + side_effect=require, + ), + self.assertRaisesRegex( + common.ScriptError, "sandbox configuration is malformed" + ), + ): + sandbox_lifecycle.start(state, 10) + + with self.assertRaisesRegex(common.ScriptError, "already provisioned"): + sandbox_lifecycle.provision( + state, + launch, + hypervisor="whp", + memory_mib=256, + net=None, + network_profile=None, + network_egress=None, + network_ingress=None, + network_egress_allow=(), + network_egress_deny=(), + host_loopback=None, + network_proxy=None, + host_loopback_forward=(), + cmdline="quiet", + timeout=10, + ) + + sandbox_lifecycle.deprovision(state, 10) + self.assertFalse(state.exists()) + + def test_managed_lifecycle_start_requests_openvmm_outcome(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary).resolve() + layer_path = root / "distro.erofs" + scratch_path = root / "scratch.ext4" + layer_path.write_bytes(b"layer") + scratch_path.write_bytes(b"scratch") + state = root / "state" + sandbox_lifecycle.provision( + state, + sandbox.SandboxLaunch( + layers=( + sandbox.SandboxLayer( + role="distro", + path=layer_path, + uuid="11111111-1111-1111-1111-111111111111", + ), + ), + scratch=scratch_path, + ), + hypervisor="whp", + memory_mib=256, + net=None, + network_profile=None, + network_egress=None, + network_ingress=None, + network_egress_allow=(), + network_egress_deny=(), + host_loopback=None, + network_proxy=None, + host_loopback_forward=(), + cmdline="quiet", + timeout=10, + ) + process = MagicMock() + process.pid = 123 + process.stdin = io.BytesIO() + session = MagicMock() + context = MagicMock() + context.__enter__.return_value = session + + def require(path: Path, _description: str) -> Path: + return path + + with ( + patch.object( + sandbox_lifecycle, + "require_file", + side_effect=require, + ), + patch.object( + sandbox_lifecycle.subprocess, + "Popen", + return_value=process, + ) as popen, + patch.object( + sandbox_lifecycle, + "_process_start_time", + return_value=456, + ) as process_start_time, + patch.object( + sandbox_lifecycle.ControlSession, + "connect", + return_value=context, + ), + ): + sandbox_lifecycle.start(state, 10) + + command = popen.call_args.args[0] + self.assertEqual( + Path(command[command.index("--microvm-report") + 1]), + state / sandbox_lifecycle.OUTCOME_NAME, + ) + session.ping.assert_called_once_with(10) + process_start_time.assert_called_once_with(123) + runtime = json.loads( + (state / sandbox_lifecycle.RUNTIME_NAME).read_text(encoding="utf-8") + ) + self.assertEqual((runtime["pid"], runtime["start_time"]), (123, 456)) + + def test_versioned_json_reader_supports_custom_version_field(self): + with tempfile.TemporaryDirectory() as temporary: + path = Path(temporary) / "outcome.json" + payload = {"schema_version": sandbox_lifecycle.OUTCOME_SCHEMA_VERSION} + path.write_text(json.dumps(payload), encoding="utf-8") + + self.assertEqual( + sandbox_lifecycle._read_json( + path, + "OpenVMM outcome report", + version_field="schema_version", + version=sandbox_lifecycle.OUTCOME_SCHEMA_VERSION, + ), + payload, + ) + with self.assertRaisesRegex(common.ScriptError, "unsupported format"): + sandbox_lifecycle._read_json( + path, + "OpenVMM outcome report", + version_field="schema_version", + version=sandbox_lifecycle.OUTCOME_SCHEMA_VERSION + 1, + ) + + def test_versioned_json_reader_rejects_invalid_utf8(self): + with tempfile.TemporaryDirectory() as temporary: + path = Path(temporary) / "config.json" + path.write_bytes(b"\xff") + + with self.assertRaisesRegex(common.ScriptError, "failed to read"): + sandbox_lifecycle._read_json(path, "sandbox configuration") + + def test_managed_exec_outcome_excludes_workload_data(self): + with tempfile.TemporaryDirectory() as temporary: + path = Path(temporary) / "outcome.json" + sandbox_lifecycle.write_exec_outcome( + path, + sandbox_lifecycle.ManagedExecResult( + 124, + "timeout", + b"sensitive stdout", + b"sensitive stderr", + ), + ) + outcome = json.loads(path.read_text(encoding="utf-8")) + + self.assertEqual(outcome["schema_version"], 1) + self.assertEqual( + outcome["outcome"], + { + "operation": "exec", + "category": "timeout", + "status_code": 124, + }, + ) + self.assertEqual(len(outcome["operation_id"]), 32) + encoded = json.dumps(outcome) + self.assertNotIn("stdout", encoded) + self.assertNotIn("stderr", encoded) + self.assertNotIn("sensitive", encoded) + + def test_managed_exec_outcome_does_not_overwrite(self): + with tempfile.TemporaryDirectory() as temporary: + path = Path(temporary) / "outcome.json" + path.write_text("existing", encoding="utf-8") + with self.assertRaisesRegex(common.ScriptError, "already exists"): + sandbox_lifecycle.write_exec_outcome( + path, + sandbox_lifecycle.ManagedExecResult(0, "exit", b"", b""), + ) + + def test_managed_exec_outcome_rejects_existing_symlink(self): + with tempfile.TemporaryDirectory() as temporary: + directory = Path(temporary) + target = directory / "missing-target" + path = directory / "outcome.json" + try: + path.symlink_to(target) + except OSError as error: + self.skipTest(f"symlinks are unavailable: {error}") + with self.assertRaisesRegex(common.ScriptError, "already exists"): + sandbox_lifecycle.write_exec_outcome( + path, + sandbox_lifecycle.ManagedExecResult(0, "exit", b"", b""), + ) + + def test_managed_stop_cleans_runtime_state_if_report_is_invalid(self): + with tempfile.TemporaryDirectory() as temporary: + state = Path(temporary) + sandbox_lifecycle._write_json( + state / sandbox_lifecycle.RUNTIME_NAME, + { + "format": sandbox_lifecycle.STATE_FORMAT, + "pid": 123, + "control_endpoint": "control.sock", + }, + ) + (state / sandbox_lifecycle.CAPABILITY_NAME).write_bytes(b"x" * 32) + (state / sandbox_lifecycle.CONTROL_SOCKET_NAME).touch() + (state / sandbox_lifecycle.OUTCOME_NAME).write_text( + "invalid", + encoding="utf-8", + ) + context = MagicMock() + with ( + patch.object( + sandbox_lifecycle, + "_process_running", + side_effect=(True, False), + ), + patch.object( + sandbox_lifecycle.ControlSession, + "connect", + return_value=context, + ), + self.assertRaisesRegex(common.ScriptError, "outcome report"), + ): + sandbox_lifecycle.stop(state, 10) + + self.assertFalse((state / sandbox_lifecycle.RUNTIME_NAME).exists()) + self.assertFalse((state / sandbox_lifecycle.CAPABILITY_NAME).exists()) + self.assertFalse((state / sandbox_lifecycle.CONTROL_SOCKET_NAME).exists()) + self.assertTrue((state / sandbox_lifecycle.OUTCOME_NAME).exists()) + + def test_process_identity_parses_linux_process_status(self): + def status(state: bytes, start_time: bytes = b"98765") -> bytes: + fields = b" ".join([state, *(b"0" for _ in range(18)), start_time, b"0"]) + # The command name contains a space and parentheses. + return b"42 (sh) x (y) " + fields + b"\n" + + self.assertEqual(sandbox_lifecycle._linux_start_time(status(b"S")), 98765) + self.assertEqual(sandbox_lifecycle._linux_start_time(status(b"D")), 98765) + for state in (b"Z", b"X"): + self.assertIsNone(sandbox_lifecycle._linux_start_time(status(state))) + for malformed in ( + b"42 (sh", + b"42 (sh) S 1 2\n", + status(b"S", b"-1"), + status(b"S", b"tick"), + ): + with self.assertRaisesRegex(ValueError, "unexpected format"): + sandbox_lifecycle._linux_start_time(malformed) + + def test_process_identity_requires_the_recorded_start_time(self): + if os.name != "nt" and sys.platform != "linux": + self.skipTest("process identity requires a Linux or Windows host") + pid = os.getpid() + start_time = sandbox_lifecycle._process_start_time(pid) + assert isinstance(start_time, int) + + self.assertTrue(sandbox_lifecycle._process_running(pid, start_time)) + self.assertFalse(sandbox_lifecycle._process_running(pid, start_time + 1)) + # Records from earlier NVX versions identify OpenVMM by process ID alone. + self.assertTrue(sandbox_lifecycle._process_running(pid, None)) + + def test_runtime_records_report_whether_their_process_runs(self): + if os.name != "nt" and sys.platform != "linux": + self.skipTest("process identity requires a Linux or Windows host") + pid = os.getpid() + start_time = sandbox_lifecycle._process_start_time(pid) + assert isinstance(start_time, int) + + self.assertTrue( + sandbox_lifecycle.runtime_process_running( + {"pid": pid, "start_time": start_time} + ) + ) + self.assertFalse( + sandbox_lifecycle.runtime_process_running( + {"pid": pid, "start_time": start_time + 1} + ) + ) + with self.assertRaisesRegex(common.ScriptError, "invalid process ID"): + sandbox_lifecycle.runtime_process_running({"pid": "openvmm"}) + with self.assertRaisesRegex(common.ScriptError, "invalid process start"): + sandbox_lifecycle.runtime_process_running({"pid": pid, "start_time": -1}) + + def test_process_identity_counts_an_exited_process_as_stopped(self): + if os.name != "nt" and sys.platform != "linux": + self.skipTest("process identity requires a Linux or Windows host") + child = _stdin_bound_child() + try: + start_time = sandbox_lifecycle._process_start_time(child.pid) + assert isinstance(start_time, int) + self.assertTrue(sandbox_lifecycle._process_running(child.pid, start_time)) + finally: + _finish_child(child) + + # On Windows, the handle that Popen holds keeps the exited process's ID. + self.assertFalse(sandbox_lifecycle._process_running(child.pid, start_time)) + self.assertFalse(sandbox_lifecycle._process_running(child.pid, None)) + + def test_process_identity_counts_a_zombie_as_stopped(self): + if sys.platform != "linux": + self.skipTest("zombie processes require a Linux host") + child = _stdin_bound_child() + try: + start_time = sandbox_lifecycle._process_start_time(child.pid) + assert isinstance(start_time, int) + _exit_unreaped(child) + stat = Path(f"/proc/{child.pid}/stat").read_bytes() + self.assertEqual(stat[stat.rfind(b")") + 2 :][:1], b"Z") + + self.assertIsNone(sandbox_lifecycle._process_start_time(child.pid)) + self.assertFalse(sandbox_lifecycle._process_running(child.pid, start_time)) + self.assertFalse(sandbox_lifecycle._process_running(child.pid, None)) + finally: + _finish_child(child) + + def test_managed_stop_completes_while_openvmm_remains_unreaped(self): + if sys.platform != "linux": + self.skipTest("zombie processes require a Linux host") + child = _stdin_bound_child() + try: + with tempfile.TemporaryDirectory() as temporary: + state = Path(temporary) + _write_managed_runtime( + state, + child.pid, + sandbox_lifecycle._process_start_time(child.pid), + ) + outcome = { + "schema_version": sandbox_lifecycle.OUTCOME_SCHEMA_VERSION, + "outcome": {"category": "stopped"}, + "network_policy": {}, + "teardown": {}, + } + (state / sandbox_lifecycle.OUTCOME_NAME).write_text( + json.dumps(outcome), encoding="utf-8" + ) + session = MagicMock() + context = MagicMock() + context.__enter__.return_value = session + + def acknowledge_stop(_timeout: float) -> None: + # OpenVMM exits after the guest acknowledges STOP, but its + # parent, such as a container init without --init, never + # reaps it. + _exit_unreaped(child) + + session.stop.side_effect = acknowledge_stop + with patch.object( + sandbox_lifecycle.ControlSession, + "connect", + return_value=context, + ): + self.assertEqual(sandbox_lifecycle.stop(state, 10), outcome) + + session.stop.assert_called_once_with(10) + self.assertFalse((state / sandbox_lifecycle.RUNTIME_NAME).exists()) + self.assertFalse((state / sandbox_lifecycle.CAPABILITY_NAME).exists()) + finally: + _finish_child(child) + + def test_managed_lifecycle_treats_a_reused_process_id_as_stopped(self): + if os.name != "nt" and sys.platform != "linux": + self.skipTest("process identity requires a Linux or Windows host") + with tempfile.TemporaryDirectory() as temporary: + state = Path(temporary) / "state" + state.mkdir() + start_time = sandbox_lifecycle._process_start_time(os.getpid()) + assert isinstance(start_time, int) + # OpenVMM is gone, and this test process now has its process ID. + _write_managed_runtime(state, os.getpid(), start_time + 1) + (state / sandbox_lifecycle.CONFIG_NAME).write_text("{}", encoding="utf-8") + + with patch.object(sandbox_lifecycle.ControlSession, "connect") as connect: + with self.assertRaisesRegex( + common.ScriptError, "runtime state is stale" + ): + sandbox_lifecycle.stop(state, 10) + with self.assertRaisesRegex( + common.ScriptError, "runtime state is stale" + ): + sandbox_lifecycle.exec_workload( + state, ("/bin/true",), timeout_ms=0, response_timeout=10 + ) + connect.assert_not_called() + + sandbox_lifecycle.deprovision(state, 10) + self.assertFalse(state.exists()) + + def test_managed_lifecycle_identifies_legacy_records_by_process_id(self): + if os.name != "nt" and sys.platform != "linux": + self.skipTest("process identity requires a Linux or Windows host") + with tempfile.TemporaryDirectory() as temporary: + state = Path(temporary) + _write_managed_runtime(state, os.getpid(), None) + + with self.assertRaisesRegex(common.ScriptError, "must be stopped"): + sandbox_lifecycle.deprovision(state, 10) + self.assertTrue((state / sandbox_lifecycle.RUNTIME_NAME).exists()) + + def test_managed_lifecycle_rejects_invalid_process_start_times(self): + with tempfile.TemporaryDirectory() as temporary: + state = Path(temporary) + for start_time in ("1", True, -1, 1.5): + with self.subTest(start_time=start_time): + sandbox_lifecycle._write_json( + state / sandbox_lifecycle.RUNTIME_NAME, + { + "format": sandbox_lifecycle.STATE_FORMAT, + "pid": os.getpid(), + "start_time": start_time, + "control_endpoint": "control.sock", + }, + ) + for operation in ( + lambda: sandbox_lifecycle.stop(state, 10), + lambda: sandbox_lifecycle.deprovision(state, 10), + ): + with self.assertRaisesRegex( + common.ScriptError, "invalid process start time" + ): + operation() + + def test_managed_start_fails_closed_without_openvmm_identity(self): + def require(path: Path, _description: str) -> Path: + return path + + # A side effect that yields None reports that no live process has the ID. + for identity, exit_status, message in ( + ((None,), 1, "OpenVMM exited during startup"), + ((None,), None, "cannot identify the OpenVMM process$"), + (OSError("denied"), None, "cannot identify the OpenVMM process: denied"), + ): + with ( + self.subTest(message=message), + tempfile.TemporaryDirectory() as temporary, + ): + state = _provision_managed_sandbox(Path(temporary).resolve()) + process = MagicMock() + process.pid = 123 + process.stdin = io.BytesIO() + process.poll.return_value = exit_status + with ( + patch.object( + sandbox_lifecycle, "require_file", side_effect=require + ), + patch.object( + sandbox_lifecycle.subprocess, "Popen", return_value=process + ), + patch.object( + sandbox_lifecycle, "_process_start_time", side_effect=identity + ), + patch.object( + sandbox_lifecycle.ControlSession, "connect" + ) as connect, + self.assertRaisesRegex(common.ScriptError, message), + ): + sandbox_lifecycle.start(state, 10) + + connect.assert_not_called() + self.assertEqual(process.terminate.called, exit_status is None) + self.assertFalse((state / sandbox_lifecycle.RUNTIME_NAME).exists()) + self.assertFalse((state / sandbox_lifecycle.CAPABILITY_NAME).exists()) + self.assertTrue((state / sandbox_lifecycle.CONFIG_NAME).exists()) + + def test_managed_start_lets_exiting_openvmm_publish_its_outcome(self): + # A guest that exits during startup, such as one that refuses the workload + # identity, makes OpenVMM close the control endpoint before it publishes + # its outcome report. Terminating OpenVMM then left the report's staging + # file, which deprovision refuses to remove (issue #438). + if os.name != "nt" and sys.platform != "linux": + self.skipTest("managed sandboxes require a Linux or Windows host") + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + state = _provision_managed_sandbox(Path(temporary.name).resolve()) + popen = subprocess.Popen + launched: list[subprocess.Popen[bytes]] = [] + + def launch(command: list[str], **options: Any) -> subprocess.Popen[bytes]: + attach_size = control_session.OUTER_HEADER.size + 32 + # With unknown options, the type checker cannot tell that start + # requests binary streams. + process = cast( + "subprocess.Popen[bytes]", + popen( + [ + sys.executable, + "-c", + _EXITING_OPENVMM, + str(attach_size), + *command[1:], + ], + **options, + ), + ) + self.addCleanup(_end_child, process) + launched.append(process) + return process + + def require(path: Path, _description: str) -> Path: + return path + + with ( + patch.object(sandbox_lifecycle, "require_file", side_effect=require), + patch.object(sandbox_lifecycle.subprocess, "Popen", side_effect=launch), + self.assertRaisesRegex( + common.ScriptError, + "^OpenVMM exited during startup: guest-exit status 125$", + ), + ): + sandbox_lifecycle.start(state, 30) + + log = (state / sandbox_lifecycle.LOG_NAME).read_text( + encoding="utf-8", errors="replace" + ) + # OpenVMM exited on its own, after it published the report. + self.assertEqual([process.returncode for process in launched], [125], log) + self.assertEqual( + sorted(path.name for path in state.iterdir()), + [ + sandbox_lifecycle.CONFIG_NAME, + sandbox_lifecycle.LOCK_NAME, + sandbox_lifecycle.LOG_NAME, + sandbox_lifecycle.OUTCOME_NAME, + ], + log, + ) + sandbox_lifecycle.deprovision(state, 10) + self.assertFalse(state.exists()) + + def test_managed_start_waits_only_for_openvmm_that_closed_its_endpoint(self): + def require(path: Path, _description: str) -> Path: + return path + + closed = control_session.ControlEndpointClosed() + # Each case holds the control failure, the status with which OpenVMM exits + # on its own or None if it runs until terminated, the waits for it, and the + # error that start raises. + for failure, status, waits, raised, message in ( + # OpenVMM closed the endpoint and exits within the grace period. + ( + closed, + 7, + [call(timeout=10)], + common.ScriptError, + "^OpenVMM exited during startup with status 7$", + ), + # It closed the endpoint but outlives the grace period. + ( + closed, + None, + [call(timeout=10), call(timeout=5)], + control_session.ControlEndpointClosed, + "^managed control endpoint closed$", + ), + # It still serves the endpoint, so it gets no grace period. + ( + TimeoutError("managed control response timed out"), + None, + [call(timeout=5)], + TimeoutError, + "^managed control response timed out$", + ), + # A session reset also comes from an OpenVMM that serves the endpoint. + ( + ConnectionError("managed control session was reset"), + None, + [call(timeout=5)], + ConnectionError, + "^managed control session was reset$", + ), + # An interrupt propagates even if OpenVMM exited. + (KeyboardInterrupt(), 0, [], KeyboardInterrupt, ""), + ): + with ( + self.subTest(failure=failure, status=status), + tempfile.TemporaryDirectory() as temporary, + ): + state = _provision_managed_sandbox(Path(temporary).resolve()) + process = _openvmm_mock(status) + session = MagicMock() + session.ping.side_effect = failure + context = MagicMock() + context.__enter__.return_value = session + context.__exit__.return_value = False + with ( + patch.object( + sandbox_lifecycle, "require_file", side_effect=require + ), + patch.object( + sandbox_lifecycle.subprocess, "Popen", return_value=process + ), + patch.object( + sandbox_lifecycle, "_process_start_time", return_value=456 + ), + patch.object( + sandbox_lifecycle.ControlSession, + "connect", + return_value=context, + ), + self.assertRaisesRegex(raised, message), + ): + sandbox_lifecycle.start(state, 10) + + self.assertEqual(process.wait.call_args_list, waits) + self.assertEqual(process.terminate.called, status is None) + self.assertEqual( + sorted(path.name for path in state.iterdir()), + [ + sandbox_lifecycle.CONFIG_NAME, + sandbox_lifecycle.LOCK_NAME, + sandbox_lifecycle.LOG_NAME, + ], + ) + + def test_managed_start_spawns_openvmm_with_a_prepared_capability_pipe(self): + # OpenVMM reads its capability as soon as it starts, so start hands it a + # pipe that already holds the capability and the end of file (#440). + def require(path: Path, _description: str) -> Path: + return path + + capability = bytes(range(1, 33)) + prepared: list[int] = [] + spawned: list[object] = [] + + def prepare(value: bytes) -> int: + self.assertEqual(value, capability) + descriptor = control_session.capability_pipe(value) + prepared.append(descriptor) + return descriptor + + process = MagicMock() + process.pid = 123 + process.stdin = None + + def launch(_command: list[str], *, stdin: object, **_options: object) -> Any: + # The pipe stays open until OpenVMM has started with it. + self.assertIsInstance(stdin, int) + os.fstat(cast(int, stdin)) + spawned.append(stdin) + return process + + session = MagicMock() + context = MagicMock() + context.__enter__.return_value = session + context.__exit__.return_value = False + with tempfile.TemporaryDirectory() as temporary: + state = _provision_managed_sandbox(Path(temporary).resolve()) + with ( + patch.object(sandbox_lifecycle, "require_file", side_effect=require), + patch.object( + sandbox_lifecycle.secrets, "token_bytes", return_value=capability + ), + patch.object(sandbox_lifecycle, "capability_pipe", side_effect=prepare), + patch.object(sandbox_lifecycle.subprocess, "Popen", side_effect=launch), + patch.object( + sandbox_lifecycle, "_process_start_time", return_value=456 + ), + patch.object( + sandbox_lifecycle.ControlSession, "connect", return_value=context + ) as connect, + ): + sandbox_lifecycle.start(state, 10) + + self.assertEqual( + (state / sandbox_lifecycle.CAPABILITY_NAME).read_bytes(), capability + ) + self.assertEqual(len(prepared), 1) + self.assertEqual(spawned, prepared) + # start closes its copy of the pipe once OpenVMM has it. + with self.assertRaises(OSError): + os.fstat(prepared[0]) + self.assertEqual(connect.call_args.args[1], capability) + session.ping.assert_called_once_with(10) + + def test_managed_start_launches_one_openvmm_for_overlapping_starts(self): + # Overlapping starts of one sandbox each launched OpenVMM. The runtime + # record and capability could then name different VMs, so that stop + # ended one VM or neither, and a failed start removed the other's files + # (#464). + if os.name != "nt" and sys.platform != "linux": + self.skipTest("managed sandboxes require a Linux or Windows host") + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + state = _provision_managed_sandbox(Path(temporary.name).resolve()) + popen = subprocess.Popen + launched: list[subprocess.Popen[bytes]] = [] + attempts: list[None] = [] + attempts_lock = threading.Lock() + overtaken = threading.Event() + + def launch(command: list[str], **options: Any) -> subprocess.Popen[bytes]: + with attempts_lock: + first = not attempts + attempts.append(None) + if first: + # Gives the other start time to launch its own OpenVMM before + # this one records its own, as it could without the lock. + overtaken.wait(0.5) + else: + overtaken.set() + # With unknown options, the type checker cannot tell that start + # requests binary streams. + process = cast( + "subprocess.Popen[bytes]", + popen( + [sys.executable, "-c", _SERVING_OPENVMM, *command[1:]], **options + ), + ) + self.addCleanup(_end_child, process) + launched.append(process) + return process + + def require(path: Path, _description: str) -> Path: + return path + + barrier = threading.Barrier(2) + results: list[str] = [] + + def contend() -> None: + barrier.wait() + try: + sandbox_lifecycle.start(state, 30) + results.append("started") + except common.ScriptError as error: + results.append(str(error)) + + with ( + patch.object(sandbox_lifecycle, "require_file", side_effect=require), + patch.object(sandbox_lifecycle.subprocess, "Popen", side_effect=launch), + ): + contenders = [ + threading.Thread(target=contend, daemon=True) for _ in range(2) + ] + for contender in contenders: + contender.start() + for contender in contenders: + contender.join(120) + self.assertFalse(any(contender.is_alive() for contender in contenders)) + + self.assertEqual(len(launched), 1) + self.assertEqual( + sorted(results), + ["sandbox is already running or has stale runtime state", "started"], + ) + runtime = json.loads( + (state / sandbox_lifecycle.RUNTIME_NAME).read_text(encoding="utf-8") + ) + self.assertEqual(runtime["pid"], launched[0].pid) + + # OpenVMM does not inherit the lock, so the next transition can take it + # while the VM runs. + relocked = threading.Event() + + def relock() -> None: + with sandbox_lifecycle._LifecycleLock(state, 30): + relocked.set() + + threading.Thread(target=relock, daemon=True).start() + self.assertTrue(relocked.wait(30)) + + # The capability on record authenticates stop to the only VM, which ends. + outcome = sandbox_lifecycle.stop(state, 30) + self.assertEqual(outcome["outcome"]["category"], "success") + self.assertEqual(launched[0].wait(timeout=30), 0) + sandbox_lifecycle.deprovision(state, 10) + self.assertFalse(state.exists()) + + def test_managed_start_refuses_runtime_files_without_a_record(self): + # A start that was killed can leave its capability or control socket, but + # no runtime record, while its OpenVMM still runs. Start and deprovision + # refuse them rather than launch a second VM for the sandbox or remove + # the files of a VM that may still run. + def require(path: Path, _description: str) -> Path: + return path + + for name in ( + sandbox_lifecycle.CAPABILITY_NAME, + sandbox_lifecycle.CONTROL_SOCKET_NAME, + ): + with ( + self.subTest(name=name), + tempfile.TemporaryDirectory() as temporary, + ): + state = _provision_managed_sandbox(Path(temporary).resolve()) + leftover = state / name + leftover.write_bytes(b"leftover") + with ( + patch.object( + sandbox_lifecycle, "require_file", side_effect=require + ), + patch.object(sandbox_lifecycle.subprocess, "Popen") as popen, + self.assertRaisesRegex( + common.ScriptError, + "^sandbox is already running or has stale runtime state$", + ), + ): + sandbox_lifecycle.start(state, 10) + + popen.assert_not_called() + self.assertEqual(leftover.read_bytes(), b"leftover") + self.assertFalse((state / sandbox_lifecycle.RUNTIME_NAME).exists()) + + with self.assertRaisesRegex( + common.ScriptError, + "^sandbox has runtime files but no runtime record, so its " + "OpenVMM process may still run", + ): + sandbox_lifecycle.deprovision(state, 10) + self.assertEqual(leftover.read_bytes(), b"leftover") + self.assertEqual( + sorted(path.name for path in state.iterdir()), + sorted( + ( + sandbox_lifecycle.CONFIG_NAME, + sandbox_lifecycle.LOCK_NAME, + name, + ) + ), + ) + + # A start creates its capability only if no other holds one. + with tempfile.TemporaryDirectory() as temporary: + capability = Path(temporary) / sandbox_lifecycle.CAPABILITY_NAME + capability.write_bytes(b"leftover") + with self.assertRaisesRegex( + common.ScriptError, + "^sandbox is already running or has stale runtime state$", + ): + sandbox_lifecycle._create_capability(capability, bytes(range(1, 33))) + self.assertEqual(capability.read_bytes(), b"leftover") + + def test_managed_cleanups_remove_the_runtime_record_last(self): + # An interrupted cleanup then leaves a record that shows OpenVMM gone, + # which deprovision accepts, rather than runtime files that it refuses. + unlink = Path.unlink + removed: list[str] = [] + expected = [ + sandbox_lifecycle.CONTROL_SOCKET_NAME, + sandbox_lifecycle.CAPABILITY_NAME, + sandbox_lifecycle.RUNTIME_NAME, + ] + + def tracked_unlink(path: Path, missing_ok: bool = False) -> None: + if path.name in sandbox_lifecycle.RUNTIME_FILE_NAMES: + removed.append(path.name) + unlink(path, missing_ok=missing_ok) + + with tempfile.TemporaryDirectory() as temporary: + state = Path(temporary) + _write_managed_runtime(state, 123, 456) + (state / sandbox_lifecycle.OUTCOME_NAME).write_text( + json.dumps( + { + "schema_version": sandbox_lifecycle.OUTCOME_SCHEMA_VERSION, + "outcome": {"category": "success"}, + "network_policy": {}, + "teardown": {}, + } + ), + encoding="utf-8", + ) + with ( + patch.object( + sandbox_lifecycle, "_process_running", side_effect=(True, False) + ), + patch.object( + sandbox_lifecycle.ControlSession, + "connect", + return_value=MagicMock(), + ), + patch.object(Path, "unlink", autospec=True, side_effect=tracked_unlink), + ): + sandbox_lifecycle.stop(state, 10) + self.assertEqual(removed, expected) + + def require(path: Path, _description: str) -> Path: + return path + + removed.clear() + session = MagicMock() + session.ping.side_effect = TimeoutError("managed control response timed out") + context = MagicMock() + context.__enter__.return_value = session + context.__exit__.return_value = False + with tempfile.TemporaryDirectory() as temporary: + state = _provision_managed_sandbox(Path(temporary).resolve()) + with ( + patch.object(sandbox_lifecycle, "require_file", side_effect=require), + patch.object( + sandbox_lifecycle.subprocess, + "Popen", + return_value=_openvmm_mock(None), + ), + patch.object( + sandbox_lifecycle, "_process_start_time", return_value=456 + ), + patch.object( + sandbox_lifecycle.ControlSession, "connect", return_value=context + ), + patch.object(Path, "unlink", autospec=True, side_effect=tracked_unlink), + self.assertRaisesRegex(TimeoutError, "response timed out"), + ): + sandbox_lifecycle.start(state, 10) + self.assertEqual(removed, expected) + + def test_managed_start_keeps_runtime_files_while_openvmm_may_run(self): + # A failed start that cannot end its OpenVMM process leaves the runtime + # files in place rather than make the sandbox look stopped, so start and + # deprovision refuse it while OpenVMM may still run. + def require(path: Path, _description: str) -> Path: + return path + + process = _openvmm_mock(None) + process.wait.side_effect = subprocess.TimeoutExpired("openvmm", 5) + session = MagicMock() + session.ping.side_effect = TimeoutError("managed control response timed out") + context = MagicMock() + context.__enter__.return_value = session + context.__exit__.return_value = False + with tempfile.TemporaryDirectory() as temporary: + state = _provision_managed_sandbox(Path(temporary).resolve()) + with ( + patch.object(sandbox_lifecycle, "require_file", side_effect=require), + patch.object( + sandbox_lifecycle.subprocess, "Popen", return_value=process + ), + patch.object( + sandbox_lifecycle, "_process_start_time", return_value=456 + ), + patch.object( + sandbox_lifecycle.ControlSession, "connect", return_value=context + ), + self.assertRaises(subprocess.TimeoutExpired), + ): + sandbox_lifecycle.start(state, 10) + + process.kill.assert_called_once_with() + self.assertTrue((state / sandbox_lifecycle.RUNTIME_NAME).is_file()) + self.assertTrue((state / sandbox_lifecycle.CAPABILITY_NAME).is_file()) + with self.assertRaisesRegex( + common.ScriptError, + "^sandbox is already running or has stale runtime state$", + ): + sandbox_lifecycle.start(state, 10) + with ( + patch.object(sandbox_lifecycle, "_process_running", return_value=True), + self.assertRaisesRegex( + common.ScriptError, "^sandbox must be stopped before deprovision$" + ), + ): + sandbox_lifecycle.deprovision(state, 10) + + def test_lifecycle_lock_excludes_other_holders(self): + with tempfile.TemporaryDirectory() as temporary: + state = _provision_managed_sandbox(Path(temporary).resolve()) + acquired = threading.Event() + + def wait_for_lock() -> None: + with sandbox_lifecycle._LifecycleLock(state, 30): + acquired.set() + + waiter = threading.Thread(target=wait_for_lock, daemon=True) + with sandbox_lifecycle._LifecycleLock(state, 30): + waiter.start() + self.assertFalse(acquired.wait(0.5)) + self.assertTrue(acquired.wait(30)) + waiter.join(30) + self.assertFalse(waiter.is_alive()) + self.assertTrue((state / sandbox_lifecycle.LOCK_NAME).is_file()) + + def test_lifecycle_lock_waits_at_most_the_timeout(self): + # A transition queued behind another that does not end fails once its + # own timeout passes, and changes nothing. + with tempfile.TemporaryDirectory() as temporary: + state = _provision_managed_sandbox(Path(temporary).resolve()) + config = (state / sandbox_lifecycle.CONFIG_NAME).read_bytes() + with ( + sandbox_lifecycle._LifecycleLock(state, 30), + patch.object(sandbox_lifecycle.subprocess, "Popen") as popen, + ): + for operation in ( + lambda: sandbox_lifecycle.start(state, 0.2), + lambda: sandbox_lifecycle.stop(state, 0.2), + lambda: sandbox_lifecycle.deprovision(state, 0.2), + ): + began = time.monotonic() + with self.assertRaisesRegex( + TimeoutError, + "^another lifecycle transition of the sandbox is still " + "in progress$", + ): + operation() + self.assertLess(time.monotonic() - began, 10) + popen.assert_not_called() + self.assertEqual( + sorted(path.name for path in state.iterdir()), + [sandbox_lifecycle.CONFIG_NAME, sandbox_lifecycle.LOCK_NAME], + ) + self.assertEqual( + (state / sandbox_lifecycle.CONFIG_NAME).read_bytes(), config + ) + + def test_lifecycle_lock_waiter_fails_after_deprovision(self): + # A transition that waits while deprovision removes the sandbox fails as + # for an unprovisioned sandbox, without opening the lock file again. + if os.name != "nt" and sys.platform != "linux": + self.skipTest("process identity requires a Linux or Windows host") + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + state = _provision_managed_sandbox(Path(temporary.name).resolve()) + # A stale record makes deprovision check its process while it holds the + # lock. + _write_managed_runtime(state, os.getpid(), None) + try_lock_file = sandbox_lifecycle._try_lock_file + open_lock_file = sandbox_lifecycle._open_lock_file + waiting = threading.Event() + waiter_opens: list[Path] = [] + errors: list[BaseException] = [] + + def waiter_thread() -> bool: + return threading.current_thread() is not threading.main_thread() + + def try_lock(descriptor: int) -> bool: + if waiter_thread(): + waiting.set() + return try_lock_file(descriptor) + + def open_lock(path: Path) -> int: + if waiter_thread(): + waiter_opens.append(path) + return open_lock_file(path) + + def stop() -> None: + try: + sandbox_lifecycle.stop(state, 10) + except BaseException as error: # noqa: BLE001 + errors.append(error) + + waiter = threading.Thread(target=stop, daemon=True) + + def process_running(_pid: int, _start_time: int | None) -> bool: + waiter.start() + self.assertTrue(waiting.wait(30)) + return False + + with ( + patch.object(sandbox_lifecycle, "_try_lock_file", side_effect=try_lock), + patch.object(sandbox_lifecycle, "_open_lock_file", side_effect=open_lock), + patch.object( + sandbox_lifecycle, "_process_running", side_effect=process_running + ), + ): + sandbox_lifecycle.deprovision(state, 10) + waiter.join(30) + + self.assertFalse(waiter.is_alive()) + self.assertEqual(len(errors), 1) + self.assertIsInstance(errors[0], common.ScriptError) + self.assertEqual(str(errors[0]), "sandbox is not provisioned") + self.assertEqual(waiter_opens, [state / sandbox_lifecycle.LOCK_NAME]) + self.assertFalse(state.exists()) + + def test_managed_deprovision_releases_its_lock_if_the_lock_file_lingers(self): + # NFS, and Windows file systems without POSIX deletion such as ReFS, keep + # a removed file in its directory while it is open, so the directory + # empties only once deprovision releases the lock. + with tempfile.TemporaryDirectory() as temporary: + state = _provision_managed_sandbox(Path(temporary).resolve()) + rmdir = Path.rmdir + release = sandbox_lifecycle._LifecycleLock.release + events: list[str] = [] + + def lingering_rmdir(path: Path) -> None: + events.append("rmdir") + if events.count("rmdir") == 1: + raise OSError(errno.ENOTEMPTY, "Directory not empty", str(path)) + rmdir(path) + + def tracked_release(lock: sandbox_lifecycle._LifecycleLock) -> None: + events.append("release") + release(lock) + + with ( + patch.object(Path, "rmdir", autospec=True, side_effect=lingering_rmdir), + patch.object( + sandbox_lifecycle._LifecycleLock, + "release", + autospec=True, + side_effect=tracked_release, + ), + ): + sandbox_lifecycle.deprovision(state, 10) + + self.assertEqual(events, ["rmdir", "release", "rmdir", "release"]) + self.assertFalse(state.exists()) + + def test_managed_deprovision_retries_only_while_lock_files_linger(self): + # Deprovision retries removing the emptied directory only while it is not + # empty, at once if the last lingering lock file is gone by then. Any + # other failure, such as a parent that denies the removal, it reports at + # once rather than after its timeout. + def check( + failure: int, failures: int | None, attempts: int, pauses: int + ) -> None: + with tempfile.TemporaryDirectory() as temporary: + state = _provision_managed_sandbox(Path(temporary).resolve()) + rmdir = Path.rmdir + removals: list[Path] = [] + + def failing_rmdir(path: Path) -> None: + if path == state: + removals.append(path) + if failures is None or len(removals) <= failures: + raise OSError(failure, os.strerror(failure), str(path)) + rmdir(path) + + with ( + patch.object( + Path, "rmdir", autospec=True, side_effect=failing_rmdir + ), + patch.object( + sandbox_lifecycle, "_pause", wraps=sandbox_lifecycle._pause + ) as pause, + ): + if failures is None: + with self.assertRaises(PermissionError): + sandbox_lifecycle.deprovision(state, 1) + else: + sandbox_lifecycle.deprovision(state, 30) + + self.assertEqual(len(removals), attempts) + self.assertEqual(pause.call_count, pauses) + if failures is None: + self.assertEqual(list(state.iterdir()), []) + else: + self.assertFalse(state.exists()) + + for name, failure, failures, attempts, pauses in ( + ("last lock file gone", errno.ENOTEMPTY, 2, 3, 0), + ("hidden lock file", errno.ENOTEMPTY, 4, 5, 1), + ("removal denied", errno.EACCES, None, 1, 0), + ): + with self.subTest(name): + check(failure, failures, attempts, pauses) + + def test_managed_deprovision_outlasts_waiters_that_keep_the_lock_file(self): + # On NFS and on Windows file systems without POSIX deletion, each waiter's + # handle keeps the removed lock file in the directory until the waiter + # finds the sandbox gone. Deprovision then retries until they have + # closed it, and no waiter creates a new lock file. + if os.name != "nt" and sys.platform != "linux": + self.skipTest("process identity requires a Linux or Windows host") + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + state = _provision_managed_sandbox(Path(temporary.name).resolve()) + _write_managed_runtime(state, os.getpid(), None) + rmdir = Path.rmdir + try_lock_file = sandbox_lifecycle._try_lock_file + polling: set[int] = set() + polling_lock = threading.Lock() + both_polling = threading.Event() + errors: list[BaseException] = [] + + def try_lock(descriptor: int) -> bool: + if threading.current_thread() is not threading.main_thread(): + with polling_lock: + polling.add(threading.get_ident()) + if len(polling) == 2: + both_polling.set() + return try_lock_file(descriptor) + + def stop() -> None: + try: + sandbox_lifecycle.stop(state, 30) + except BaseException as error: # noqa: BLE001 + errors.append(error) + + waiters = [threading.Thread(target=stop, daemon=True) for _ in range(2)] + + def process_running(_pid: int, _start_time: int | None) -> bool: + for waiter in waiters: + waiter.start() + self.assertTrue(both_polling.wait(30)) + return False + + def lingering_rmdir(path: Path) -> None: + if path == state and any(waiter.is_alive() for waiter in waiters): + raise OSError(errno.ENOTEMPTY, "Directory not empty", str(path)) + rmdir(path) + + with ( + patch.object(sandbox_lifecycle, "_try_lock_file", side_effect=try_lock), + patch.object( + sandbox_lifecycle, "_process_running", side_effect=process_running + ), + patch.object(Path, "rmdir", autospec=True, side_effect=lingering_rmdir), + ): + sandbox_lifecycle.deprovision(state, 30) + for waiter in waiters: + waiter.join(30) + + self.assertFalse(any(waiter.is_alive() for waiter in waiters)) + self.assertEqual( + [str(error) for error in errors], ["sandbox is not provisioned"] * 2 + ) + self.assertFalse(state.exists()) + + def test_managed_deprovision_outlasts_a_transition_that_starts_meanwhile(self): + # A transition that starts after deprovision removed the lock file holds + # a new one in the directory until it finds the sandbox gone, which must + # not make deprovision fail. + with tempfile.TemporaryDirectory() as temporary: + state = _provision_managed_sandbox(Path(temporary).resolve()) + rmdir = Path.rmdir + arrived = threading.Event() + leave = threading.Event() + attempts: list[bool] = [] + + def arrive() -> None: + with sandbox_lifecycle._LifecycleLock(state, 30): + arrived.set() + leave.wait(30) + + arrival = threading.Thread(target=arrive, daemon=True) + + def contended_rmdir(path: Path) -> None: + if path != state: + rmdir(path) + return + if not attempts: + attempts.append(False) + arrival.start() + self.assertTrue(arrived.wait(30)) + raise OSError(errno.ENOTEMPTY, "Directory not empty", str(path)) + try: + rmdir(path) + except OSError: + attempts.append(False) + leave.set() + raise + attempts.append(True) + + with patch.object( + Path, "rmdir", autospec=True, side_effect=contended_rmdir + ): + sandbox_lifecycle.deprovision(state, 30) + arrival.join(30) + + self.assertFalse(arrival.is_alive()) + self.assertEqual(attempts[0], False) + self.assertIn(False, attempts[1:]) + self.assertEqual(attempts[-1], True) + self.assertFalse(state.exists()) + + def test_provision_waiting_on_deprovision_provisions_a_new_directory(self): + # A provision that waits while deprovision removes the sandbox creates + # nothing in the directory being removed. Once deprovision has removed + # it, the provision creates the directory again, and both succeed. + if os.name != "nt" and sys.platform != "linux": + self.skipTest("process identity requires a Linux or Windows host") + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + root = Path(temporary.name).resolve() + state = _provision_managed_sandbox(root) + _write_managed_runtime(state, os.getpid(), None) + rmdir = Path.rmdir + try_lock_file = sandbox_lifecycle._try_lock_file + polling = threading.Event() + removals: list[bool] = [] + errors: list[BaseException] = [] + + def try_lock(descriptor: int) -> bool: + if threading.current_thread() is not threading.main_thread(): + polling.set() + return try_lock_file(descriptor) + + def provision() -> None: + try: + _provision_managed_sandbox(root) + except BaseException as error: # noqa: BLE001 + errors.append(error) + + waiter = threading.Thread(target=provision, daemon=True) + + def process_running(_pid: int, _start_time: int | None) -> bool: + waiter.start() + self.assertTrue(polling.wait(30)) + return False + + def lingering_rmdir(path: Path) -> None: + if path == state and not removals: + # As on NFS, the waiter's handle keeps the removed lock file. + removals.append(False) + raise OSError(errno.ENOTEMPTY, "Directory not empty", str(path)) + rmdir(path) + if path == state: + removals.append(True) + + with ( + patch.object(sandbox_lifecycle, "_try_lock_file", side_effect=try_lock), + patch.object( + sandbox_lifecycle, "_process_running", side_effect=process_running + ), + patch.object(Path, "rmdir", autospec=True, side_effect=lingering_rmdir), + ): + sandbox_lifecycle.deprovision(state, 30) + waiter.join(30) + + self.assertFalse(waiter.is_alive()) + self.assertEqual(errors, []) + self.assertEqual(removals[0], False) + self.assertIn(True, removals[1:]) + self.assertEqual( + sorted(path.name for path in state.iterdir()), + [sandbox_lifecycle.CONFIG_NAME, sandbox_lifecycle.LOCK_NAME], + ) + + def test_provision_waiting_on_deprovision_retries_once_deprovision_ends(self): + # Deprovision clears the mark of the removed lock file once it has removed + # the directory, or failed to remove it or the lock file, so a provision + # that waited for it retries at once. It then finds the sandbox that + # another provision created in between, or provisions the directory that + # deprovision left. + if os.name != "nt" and sys.platform != "linux": + self.skipTest("process identity requires a Linux or Windows host") + + def check(ending: str) -> None: + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + root = Path(temporary.name).resolve() + state = _provision_managed_sandbox(root) + _write_managed_runtime(state, os.getpid(), None) + rmdir = Path.rmdir + unlink = Path.unlink + try_lock_file = sandbox_lifecycle._try_lock_file + polling = threading.Event() + denials: list[Path] = [] + errors: list[BaseException] = [] + + def deny(path: Path) -> None: + denials.append(path) + raise PermissionError( + errno.EACCES, os.strerror(errno.EACCES), str(path) + ) + + def try_lock(descriptor: int) -> bool: + if threading.current_thread() is not threading.main_thread(): + polling.set() + return try_lock_file(descriptor) + + def provision() -> None: + try: + _provision_managed_sandbox(root) + except BaseException as error: # noqa: BLE001 + errors.append(error) + + waiter = threading.Thread(target=provision, daemon=True) + + def process_running(_pid: int, _start_time: int | None) -> bool: + waiter.start() + self.assertTrue(polling.wait(30)) + return False + + def ending_rmdir(path: Path) -> None: + if path == state and ending == "removal denied": + deny(path) + rmdir(path) + if path == state and ending == "replaced": + # Another provision recreates the directory before the waiter + # could find it gone. + _provision_managed_sandbox(root) + + def ending_unlink(path: Path, missing_ok: bool = False) -> None: + # Only the first removal of the lock file fails, so the release + # that follows removes it. + lock = path == state / sandbox_lifecycle.LOCK_NAME + if lock and ending == "unlink denied" and not denials: + deny(path) + unlink(path, missing_ok=missing_ok) + + with ( + patch.object(sandbox_lifecycle, "_try_lock_file", side_effect=try_lock), + patch.object( + sandbox_lifecycle, "_process_running", side_effect=process_running + ), + patch.object(Path, "rmdir", autospec=True, side_effect=ending_rmdir), + patch.object(Path, "unlink", autospec=True, side_effect=ending_unlink), + ): + if ending == "replaced": + sandbox_lifecycle.deprovision(state, 30) + else: + with self.assertRaises(PermissionError): + sandbox_lifecycle.deprovision(state, 30) + waiter.join(30) + + self.assertFalse(waiter.is_alive()) + self.assertEqual(len(denials), 0 if ending == "replaced" else 1) + self.assertEqual( + [str(error) for error in errors], + ["sandbox is already provisioned"] if ending == "replaced" else [], + ) + self.assertEqual( + sorted(path.name for path in state.iterdir()), + [sandbox_lifecycle.CONFIG_NAME, sandbox_lifecycle.LOCK_NAME], + ) + + for ending in ("replaced", "removal denied", "unlink denied"): + with self.subTest(ending): + check(ending) + + def test_provision_handoff_completes_or_ends_the_removal(self): + # While only lingering lock files keep the directory that deprovision + # removes, a provision waits. It stops once the path names no directory or + # another one, or the directory holds anything else, and once nothing + # keeps the directory, it removes it itself, in case deprovision has given + # up. A removal that fails for another reason leaves it to provision the + # directory in place. + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary).resolve() + state = root / "state" + state.mkdir() + lingering = state / sandbox_lifecycle.LOCK_NAME + lingering.write_bytes(b"") + removing = os.lstat(state) + with self.assertRaisesRegex( + TimeoutError, + "^another lifecycle transition of the sandbox is still in progress$", + ): + sandbox_lifecycle._wait_for_handoff( + state, removing, time.monotonic() + 0.1 + ) + + def wait() -> None: + sandbox_lifecycle._wait_for_handoff( + state, removing, time.monotonic() + 30 + ) + + with patch.object(sandbox_lifecycle, "_pause", return_value=False) as pause: + for name in (sandbox_lifecycle.CONFIG_NAME, "foreign"): + (state / name).write_text("{}", encoding="utf-8") + wait() + (state / name).unlink() + # The old directory keeps its identity under another name, so the + # new one cannot reuse it. + state.rename(root / "old") + state.mkdir() + wait() + self.assertTrue(state.is_dir()) + state.rmdir() + (root / "old").rename(state) + lingering.unlink() + with patch.object( + Path, + "rmdir", + autospec=True, + side_effect=PermissionError(errno.EACCES, "denied"), + ): + wait() + self.assertTrue(state.is_dir()) + wait() + self.assertFalse(state.exists()) + wait() + pause.assert_not_called() + + def test_provision_completes_a_removal_that_deprovision_gave_up(self): + # Once a lingering lock file makes deprovision release the lock before the + # directory is gone, a provision that waits for the removal completes it + # itself if deprovision gives up, and then provisions the path again. + if os.name != "nt" and sys.platform != "linux": + self.skipTest("process identity requires a Linux or Windows host") + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + root = Path(temporary.name).resolve() + state = _provision_managed_sandbox(root) + _write_managed_runtime(state, os.getpid(), None) + rmdir = Path.rmdir + try_lock_file = sandbox_lifecycle._try_lock_file + wait_for_handoff = sandbox_lifecycle._wait_for_handoff + polling = threading.Event() + deprovisioned = threading.Event() + attempts: list[Path] = [] + handoffs: list[Path] = [] + errors: list[BaseException] = [] + + def try_lock(descriptor: int) -> bool: + if threading.current_thread() is not threading.main_thread(): + polling.set() + return try_lock_file(descriptor) + + def provision() -> None: + try: + _provision_managed_sandbox(root) + except BaseException as error: # noqa: BLE001 + errors.append(error) + + waiter = threading.Thread(target=provision, daemon=True) + + def process_running(_pid: int, _start_time: int | None) -> bool: + waiter.start() + self.assertTrue(polling.wait(30)) + return False + + def abandoned_rmdir(path: Path) -> None: + # A lock file lingers at deprovision's first attempt, and its retry is + # denied, so that it gives up. + if path == state and threading.current_thread() is threading.main_thread(): + attempts.append(path) + if len(attempts) == 1: + raise OSError(errno.ENOTEMPTY, os.strerror(errno.ENOTEMPTY)) + raise PermissionError(errno.EACCES, os.strerror(errno.EACCES)) + rmdir(path) + + def handoff(state_dir: Path, removing: os.stat_result, deadline: float) -> None: + handoffs.append(state_dir) + if not deprovisioned.wait(30): + raise AssertionError("deprovision did not give up") + wait_for_handoff(state_dir, removing, deadline) + + with ( + patch.object(sandbox_lifecycle, "_try_lock_file", side_effect=try_lock), + patch.object( + sandbox_lifecycle, "_process_running", side_effect=process_running + ), + patch.object(Path, "rmdir", autospec=True, side_effect=abandoned_rmdir), + patch.object(sandbox_lifecycle, "_wait_for_handoff", side_effect=handoff), + ): + with self.assertRaises(PermissionError): + sandbox_lifecycle.deprovision(state, 30) + deprovisioned.set() + waiter.join(30) + + self.assertFalse(waiter.is_alive()) + self.assertEqual(errors, []) + self.assertEqual(attempts, [state, state]) + self.assertEqual(handoffs, [state]) + self.assertEqual( + sorted(path.name for path in state.iterdir()), + [sandbox_lifecycle.CONFIG_NAME, sandbox_lifecycle.LOCK_NAME], + ) + + def test_managed_deprovision_accepts_a_removal_that_a_provision_completed(self): + # A provision that waits for deprovision to remove the directory may + # remove it first and provision the path again. Deprovision then succeeds + # rather than report the new sandbox as provisioned again, which it tells + # apart even where a new directory could reuse the old inode number. + if os.name != "nt" and sys.platform != "linux": + self.skipTest("process identity requires a Linux or Windows host") + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + root = Path(temporary.name).resolve() + state = _provision_managed_sandbox(root) + _write_managed_runtime(state, os.getpid(), None) + rmdir = Path.rmdir + try_lock_file = sandbox_lifecycle._try_lock_file + polling = threading.Event() + provisioned = threading.Event() + attempts: list[Path] = [] + errors: list[BaseException] = [] + + def try_lock(descriptor: int) -> bool: + if threading.current_thread() is not threading.main_thread(): + polling.set() + return try_lock_file(descriptor) + + def provision() -> None: + try: + _provision_managed_sandbox(root) + except BaseException as error: # noqa: BLE001 + errors.append(error) + finally: + provisioned.set() + + waiter = threading.Thread(target=provision, daemon=True) + + def process_running(_pid: int, _start_time: int | None) -> bool: + waiter.start() + self.assertTrue(polling.wait(30)) + return False + + def contended_rmdir(path: Path) -> None: + if path == state and threading.current_thread() is threading.main_thread(): + attempts.append(path) + if len(attempts) == 1: + raise OSError(errno.ENOTEMPTY, os.strerror(errno.ENOTEMPTY)) + # Deprovision retries only once the waiting provision has removed + # the directory and provisioned the path again. + self.assertTrue(provisioned.wait(30)) + rmdir(path) + + with ( + patch.object(sandbox_lifecycle, "_try_lock_file", side_effect=try_lock), + patch.object( + sandbox_lifecycle, "_process_running", side_effect=process_running + ), + patch.object(Path, "rmdir", autospec=True, side_effect=contended_rmdir), + ): + sandbox_lifecycle.deprovision(state, 30) + waiter.join(30) + + self.assertFalse(waiter.is_alive()) + self.assertEqual(errors, []) + self.assertEqual(attempts, [state, state]) + self.assertEqual( + sorted(path.name for path in state.iterdir()), + [sandbox_lifecycle.CONFIG_NAME, sandbox_lifecycle.LOCK_NAME], + ) + + def test_lifecycle_lock_file_transfers_complete_despite_short_counts(self): + # The tombstone must be complete before deprovision removes the lock file, + # and a waiter must read all of it, even if os.write() or os.read() + # transfers only part of it at a time. + with tempfile.TemporaryDirectory() as temporary: + descriptor = sandbox_lifecycle._open_lock_file( + Path(temporary) / sandbox_lifecycle.LOCK_NAME + ) + try: + self._check_lock_file_transfers(descriptor) + finally: + os.close(descriptor) + + def _check_lock_file_transfers(self, descriptor: int) -> None: + write = os.write + read = os.read + tombstone = sandbox_lifecycle.LOCK_TOMBSTONE + + def short_write(fd: int, data: memoryview) -> int: + return write(fd, bytes(data[:3])) + + def short_read(fd: int, size: int) -> bytes: + return read(fd, min(size, 3)) + + with ( + patch.object( + sandbox_lifecycle.os, "write", side_effect=short_write + ) as writes, + patch.object(sandbox_lifecycle.os, "read", side_effect=short_read), + ): + sandbox_lifecycle._write_lock_file(descriptor, tombstone) + self.assertEqual( + sandbox_lifecycle._read_lock_file(descriptor, len(tombstone)), + tombstone, + ) + self.assertEqual(writes.call_count, -(-len(tombstone) // 3)) + self.assertEqual(os.lseek(descriptor, 0, os.SEEK_CUR), 0) + with ( + patch.object(sandbox_lifecycle.os, "write", return_value=0), + self.assertRaisesRegex(OSError, "made no progress"), + ): + sandbox_lifecycle._write_lock_file(descriptor, tombstone) + + def test_managed_deprovision_reports_a_provision_that_starts_meanwhile(self): + # A provision that starts after deprovision removed the lock file, and + # before it removed the directory, provisions the directory again first. + # Deprovision then reports that rather than remove the new sandbox. + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary).resolve() + state = _provision_managed_sandbox(root) + rmdir = Path.rmdir + provisioned: list[Path] = [] + + def contended_rmdir(path: Path) -> None: + if path == state and not provisioned: + arrival = threading.Thread( + target=lambda: provisioned.append( + _provision_managed_sandbox(root) + ), + daemon=True, + ) + arrival.start() + arrival.join(30) + rmdir(path) + + with ( + patch.object(Path, "rmdir", autospec=True, side_effect=contended_rmdir), + self.assertRaisesRegex( + common.ScriptError, + "^sandbox was provisioned again while deprovision removed it$", + ), + ): + sandbox_lifecycle.deprovision(state, 10) + + self.assertEqual(provisioned, [state]) + self.assertEqual( + sorted(path.name for path in state.iterdir()), + [sandbox_lifecycle.CONFIG_NAME, sandbox_lifecycle.LOCK_NAME], + ) + + def test_lifecycle_deadlines_reject_retries_after_they_pass(self): + # Each wait sleeps at most until its deadline, and no retry starts after + # it, so neither a directory that deprovision removes only then nor a lock + # file that another transition replaces only then counts. + self.assertFalse(sandbox_lifecycle._pause(time.monotonic() - 1)) + with tempfile.TemporaryDirectory() as temporary: + directory = Path(temporary) / "going" + directory.mkdir() + # A lock file lingers in the directory until the deadline. + lingering = directory / sandbox_lifecycle.LOCK_NAME + lingering.write_bytes(b"") + deadline = 3 / 64 + clock = [0.0] + sleeps: list[float] = [] + + def sleep(seconds: float) -> None: + sleeps.append(seconds) + clock[0] += seconds + if clock[0] >= deadline and directory.exists(): + lingering.unlink() + directory.rmdir() + + with ( + patch.object(sandbox_lifecycle, "LOCK_RETRY_INTERVAL", 1 / 32), + patch.object( + sandbox_lifecycle.time, "monotonic", side_effect=lambda: clock[0] + ), + patch.object(sandbox_lifecycle.time, "sleep", side_effect=sleep), + self.assertRaisesRegex( + TimeoutError, + "^another lifecycle transition of the sandbox is still in " + "progress$", + ), + ): + sandbox_lifecycle._wait_for_handoff( + directory, os.lstat(directory), deadline + ) + + self.assertEqual(sleeps, [1 / 32, 1 / 64]) + self.assertFalse(directory.exists()) + + with tempfile.TemporaryDirectory() as temporary: + state = _provision_managed_sandbox(Path(temporary).resolve()) + names_open_file = sandbox_lifecycle._names_open_file + clock = [0.0] + checks: list[Path] = [] + + def replaced_late(path: Path, descriptor: int) -> bool: + checks.append(path) + if len(checks) > 1: + return names_open_file(path, descriptor) + clock[0] = 1.0 + return False + + with ( + patch.object( + sandbox_lifecycle.time, "monotonic", side_effect=lambda: clock[0] + ), + patch.object( + sandbox_lifecycle, "_names_open_file", side_effect=replaced_late + ), + self.assertRaisesRegex( + TimeoutError, + "^another lifecycle transition of the sandbox is still in " + "progress$", + ), + sandbox_lifecycle._LifecycleLock(state, 1), + ): + self.fail("the lock was acquired after its deadline") + + self.assertEqual(checks, [state / sandbox_lifecycle.LOCK_NAME]) + self.assertEqual( + sorted(path.name for path in state.iterdir()), + [sandbox_lifecycle.CONFIG_NAME, sandbox_lifecycle.LOCK_NAME], + ) + + def test_provision_recreates_a_state_directory_that_deprovision_removed(self): + # A deprovision can remove the state directory after provision prepared + # it and before provision opens the lock file in it. The provision then + # creates the directory again instead of failing. + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary).resolve() + state = root / "state" + open_lock_file = sandbox_lifecycle._open_lock_file + opened: list[Path] = [] + + def open_after_deprovision(path: Path) -> int: + if not opened: + shutil.rmtree(state) + opened.append(path) + return open_lock_file(path) + + with patch.object( + sandbox_lifecycle, "_open_lock_file", side_effect=open_after_deprovision + ): + self.assertEqual(_provision_managed_sandbox(root), state) + + self.assertEqual(len(opened), 2) + self.assertEqual( + sorted(path.name for path in state.iterdir()), + [sandbox_lifecycle.CONFIG_NAME, sandbox_lifecycle.LOCK_NAME], + ) + + def test_provision_waits_out_a_lock_file_pending_deletion(self): + # Without POSIX deletion semantics, Windows denies opening a lock file + # that deprovision removed until its last handle closes, so a provision + # retries. Elsewhere, such a denial is a real error. + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary).resolve() + open_lock_path = sandbox_lifecycle._open_lock_path + attempts: list[Path] = [] + + def pending_deletion(path: Path) -> int: + attempts.append(path) + if len(attempts) == 1: + raise PermissionError(errno.EACCES, "Access is denied", str(path)) + return open_lock_path(path) + + with patch.object( + sandbox_lifecycle, "_open_lock_path", side_effect=pending_deletion + ): + if os.name == "nt": + _provision_managed_sandbox(root) + self.assertEqual(len(attempts), 2) + self.assertTrue( + (root / "state" / sandbox_lifecycle.CONFIG_NAME).is_file() + ) + else: + with self.assertRaises(PermissionError): + _provision_managed_sandbox(root) + self.assertEqual(len(attempts), 1) + + def test_lifecycle_lock_refuses_a_linked_lock_file(self): + # Deprovision writes to the lock file, so a link to a file elsewhere must + # not stand in for it. + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary).resolve() + state = _provision_managed_sandbox(root) + target = root / "precious" + target.write_bytes(b"precious data") + lock = state / sandbox_lifecycle.LOCK_NAME + links: dict[str, Callable[[], None]] = { + "hard link": lambda: os.link(target, lock), + "symbolic link": lambda: os.symlink(target, lock), + } + for kind, link in links.items(): + with self.subTest(kind=kind): + lock.unlink(missing_ok=True) + try: + link() + except OSError as error: + self.skipTest(f"cannot create a {kind}: {error}") + with self.assertRaisesRegex( + common.ScriptError, + "^sandbox lifecycle lock is not a standalone regular file", + ): + sandbox_lifecycle.deprovision(state, 10) + self.assertEqual(target.read_bytes(), b"precious data") + self.assertTrue((state / sandbox_lifecycle.CONFIG_NAME).is_file()) + + def test_lifecycle_lock_rejects_an_unbounded_timeout(self): + with tempfile.TemporaryDirectory() as temporary: + state = _provision_managed_sandbox(Path(temporary).resolve()) + for timeout in (0.0, -1.0, float("nan"), float("inf")): + with ( + self.subTest(timeout=timeout), + self.assertRaisesRegex( + ValueError, + "^lifecycle timeout must be finite and greater than 0$", + ), + ): + sandbox_lifecycle.stop(state, timeout) + + def test_lifecycle_lock_file_stays_only_in_a_provisioned_sandbox(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary).resolve() + unprovisioned = root / "unprovisioned" + unprovisioned.mkdir() + with self.assertRaisesRegex(common.ScriptError, "sandbox configuration"): + sandbox_lifecycle.start(unprovisioned, 10) + with self.assertRaisesRegex(common.ScriptError, "sandbox is not running"): + sandbox_lifecycle.stop(unprovisioned, 10) + self.assertEqual(list(unprovisioned.iterdir()), []) + + state = _provision_managed_sandbox(root) + self.assertEqual( + sorted(path.name for path in state.iterdir()), + [sandbox_lifecycle.CONFIG_NAME, sandbox_lifecycle.LOCK_NAME], + ) + # A sandbox that an earlier NVX version provisioned gains a lock file. + (state / sandbox_lifecycle.LOCK_NAME).unlink() + with self.assertRaisesRegex(common.ScriptError, "sandbox is not running"): + sandbox_lifecycle.stop(state, 10) + self.assertTrue((state / sandbox_lifecycle.LOCK_NAME).is_file()) + + # Deprovision removes every NVX file, the lock file included, even when + # a file of another owner keeps the directory. + foreign = state / "operator-note.txt" + foreign.write_text("operator data\n", encoding="utf-8") + with self.assertRaisesRegex(common.ScriptError, "operator-note.txt"): + sandbox_lifecycle.deprovision(state, 10) + self.assertEqual(list(state.iterdir()), [foreign]) + foreign.unlink() + sandbox_lifecycle.deprovision(state, 10) + self.assertFalse(state.exists()) + + def test_launch_contract_rejects_disk_option_delimiters(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + layer_path = root / "layer;create=1G" + scratch = root / "scratch.ext4" + layer_path.touch() + scratch.touch() + layer = sandbox.SandboxLayer( + role="distro", + path=layer_path, + uuid="11111111-1111-1111-1111-111111111111", + ) + + with self.assertRaisesRegex(common.ScriptError, "semicolons"): + sandbox.SandboxLaunch( + layers=(layer,), + scratch=scratch, + ).validated() + + +class BenchmarkTests(unittest.TestCase): + def test_kvm_worker_result_decoding(self): + completed = subprocess.CompletedProcess( + ["worker"], + 0, + stdout='noise\nOPENVMM_KVM_E2E_RESULT={"p50_ms": 1.5}\n', + stderr="", + ) + with patch.object(benchmark.subprocess, "run", return_value=completed): + result = benchmark._run_kvm_worker(["worker"], "e2e") + + self.assertEqual(result, {"p50_ms": 1.5}) + + def test_kvm_workers_forward_translated_scratch_directory(self): + with tempfile.TemporaryDirectory() as temporary: + scratch = Path(temporary).resolve() + args = argparse.Namespace( + scratch_dir=scratch, + warmups=1, + runs=2, + memory_mib=128, + processors=4, + host_cpu_reserve=1, + cpus="0-3", + timeout=1.0, + teardown_mode="guest-exit", + net=None, + network_profile=None, + keep_kvm_stage=True, + ) + workers = ( + (benchmark.benchmark_kvm, ""), + (benchmark.benchmark_e2e_kvm, "e2e"), + (benchmark.benchmark_snapshot_restore_kvm, "restore"), + (benchmark.benchmark_snapshot_kvm, "snapshot"), + ) + + def translate(path: Path) -> str: + if path == benchmark.NVX_SCRIPT: + return "/workspace/scripts/nvx.py" + self.assertEqual(path, scratch) + return "/mnt/data/nvx-benchmark-scratch" + + with ( + patch.object(benchmark, "stage_kvm"), + patch.object( + benchmark, + "windows_to_wsl", + side_effect=translate, + ) as translate_path, + patch.object( + benchmark, + "_run_kvm_worker", + return_value={}, + ) as run_worker, + ): + for worker, result_kind in workers: + with self.subTest(result_kind=result_kind or "boot"): + translate_path.reset_mock() + run_worker.reset_mock() + worker( + args, + Path("openvmm"), + Path("kernel"), + Path("initrd"), + ) + + command = run_worker.call_args.args[0] + scratch_index = command.index("--scratch-dir") + self.assertEqual( + command[scratch_index : scratch_index + 2], + [ + "--scratch-dir", + "/mnt/data/nvx-benchmark-scratch", + ], + ) + self.assertEqual( + translate_path.call_args_list, + [call(benchmark.NVX_SCRIPT), call(scratch)], + ) + self.assertEqual( + run_worker.call_args.args[1], + result_kind, + ) + + def test_kvm_worker_defaults_to_translated_system_temporary_directory(self): + with tempfile.TemporaryDirectory() as temporary: + system_temporary = Path(temporary).resolve() + args = argparse.Namespace(scratch_dir=None) + with ( + patch.object( + benchmark.tempfile, + "gettempdir", + return_value=str(system_temporary), + ), + patch.object( + benchmark, + "windows_to_wsl", + return_value="/mnt/c/system-temp", + ) as translate, + ): + self.assertEqual( + benchmark._kvm_worker_scratch_arguments(args), + ["--scratch-dir", "/mnt/c/system-temp"], + ) + + translate.assert_called_once_with(system_temporary) + + def test_require_file_preserves_resolved_path_error(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + missing = root / "nested" / ".." / "missing" + resolved_missing = missing.resolve() + + with self.assertRaises(FileNotFoundError) as raised: + benchmark.require_file(missing, "benchmark artifact") + + self.assertEqual( + str(raised.exception), + f"benchmark artifact not found: {resolved_missing}", + ) + + def test_snapshot_profile_records_are_parsed_and_summarized(self): + record = benchmark.parse_snapshot_profile_line( + b"OPENVMM_SNAPSHOT_PROFILE_V1 operation=restore " + b"phase=artifact_open exclusive=1 duration_ns=1500000 " + b"process_elapsed_ns=2000000 pid=42 logical_bytes=1024 " + b"allocated_bytes=512\r" + ) + + self.assertIsNotNone(record) + assert record is not None + self.assertEqual(record["operation"], "restore") + self.assertEqual(record["duration_ns"], 1_500_000) + self.assertTrue(record["exclusive"]) + summary = benchmark.summarize_lifecycle_profiles( + [{"records": [record, {**record, "duration_ns": 2_500_000}]}] + ) + metric = summary["phases"]["restore.artifact_open"] + self.assertEqual(metric["samples_ms"], [1.5, 2.5]) + self.assertEqual(metric["p50_ms"], 2.0) + self.assertEqual(metric["p95_ms"], 2.5) + self.assertTrue(metric["exclusive"]) + + def test_snapshot_profile_rejects_incomplete_records(self): + with self.assertRaisesRegex(ValueError, "missing"): + benchmark.parse_snapshot_profile_line( + b"OPENVMM_SNAPSHOT_PROFILE_V1 operation=restore phase=open" + ) + + def test_snapshot_profile_records_console_input_dispatch(self): + collector = benchmark.SnapshotProfileCollector(42, 100) + + with patch.object(benchmark, "process_resource_counters", return_value={}): + collector.feed( + b"OPENVMM_SNAPSHOT_PROFILE_V1 operation=capture " + b"phase=input_gate exclusive=1 duration_ns=20 " + b"process_elapsed_ns=120 pid=42\n" + b"OPENVMM_SNAPSHOT_PROFILE_V1 operation=capture " + b"phase=publication_commit exclusive=1 duration_ns=10 " + b"process_elapsed_ns=310 pid=42\n" + ) + sample = collector.finish_capture(200, 260, 300, 400, 450, 4096) + + records = cast(list[dict[str, object]], sample["records"]) + by_phase = {str(record["phase"]): record for record in records} + generation = by_phase["snapshot_generation"] + self.assertEqual(generation["duration_ns"], 210) + self.assertEqual(generation["source"], "openvmm_clock") + self.assertEqual(sample["generation_duration_ns"], 210) + dispatch = by_phase["console_input_dispatch"] + self.assertEqual(dispatch["operation"], "capture") + self.assertEqual(dispatch["phase"], "console_input_dispatch") + self.assertEqual(dispatch["duration_ns"], 60) + self.assertEqual(dispatch["observer_elapsed_ns"], 160) + self.assertEqual(dispatch["logical_bytes"], 4096) + self.assertTrue(dispatch["exclusive"]) + round_trip = by_phase["console_command_round_trip"] + self.assertEqual(round_trip["phase"], "console_command_round_trip") + self.assertEqual(round_trip["duration_ns"], 100) + guest_to_publication = by_phase["guest_dispatch_to_publication"] + self.assertEqual(guest_to_publication["phase"], "guest_dispatch_to_publication") + self.assertEqual(guest_to_publication["duration_ns"], 100) + self.assertEqual(by_phase["request_to_publication"]["duration_ns"], 200) + + def test_snapshot_generation_requires_valid_process_clock_boundaries(self): + gate: dict[str, object] = { + "operation": "capture", + "phase": "input_gate", + "duration_ns": 20, + "process_elapsed_ns": 120, + "pid": 42, + } + publication: dict[str, object] = { + "operation": "capture", + "phase": "publication_commit", + "duration_ns": 10, + "process_elapsed_ns": 310, + "pid": 42, + } + self.assertEqual( + benchmark.snapshot_generation_duration_ns([gate, publication], 42), 210 + ) + for records, error in ( + ([publication], "exactly one 'input_gate'"), + ([gate], "exactly one 'publication_commit'"), + ([gate, gate, publication], "exactly one 'input_gate'"), + ([gate, publication, publication], "exactly one 'publication_commit'"), + ([gate, {**publication, "pid": 43}], "different process"), + ([{**gate, "duration_ns": -1}, publication], "not monotonic"), + ([{**gate, "process_elapsed_ns": 10}, publication], "not monotonic"), + ([gate, {**publication, "process_elapsed_ns": 125}], "not monotonic"), + ): + with self.subTest(records=records): + with self.assertRaisesRegex(ValueError, error): + benchmark.snapshot_generation_duration_ns(records, 42) + + def test_capture_excludes_polling_input_and_matches_profile_generation(self): + for delay_ms in (0, 500, 2000): + for profiled in (False, True): + with self.subTest(delay_ms=delay_ms, profiled=profiled): + self.check_capture_timing(delay_ms, profiled) + + def check_capture_timing(self, delay_ms: int, profiled: bool) -> None: + clock_ns = 0 + writes: list[bytes] = [] + + class FakeProcess: + pid = 42 + returncode = 0 + + def poll(self) -> int: + return 0 + + def wait(self) -> int: + nonlocal clock_ns + clock_ns += 1_000_000 + return 0 + + class FakeInteraction: + process = FakeProcess() + + def read_output(self, _chunks: object) -> None: + pass + + def read_stderr(self, _chunks: object) -> None: + pass + + def write_input(self, data: bytes) -> None: + nonlocal clock_ns + writes.append(data) + if data == b"nvx-snapshot\n": + clock_ns += 20_000_000 + + def close(self) -> None: + pass + + chunks = iter( + ( + ("console", benchmark.BOOT_MARKER + b"\n", 0, False), + ("console", benchmark.SMP_PROBE_COMPLETION_MARKER + b"\n", 1, False), + ( + "console", + benchmark.SNAPSHOT_GUEST_DISPATCH_MARKER + b"\n", + delay_ms, + False, + ), + ( + "stderr", + b"OPENVMM_SNAPSHOT_PROFILE_V1 operation=capture " + b"phase=input_gate exclusive=1 duration_ns=500000 " + b"process_elapsed_ns=500000000 pid=42\n" + b"OPENVMM_SNAPSHOT_PROFILE_V1 operation=capture " + b"phase=publication_commit exclusive=1 duration_ns=100000 " + b"process_elapsed_ns=502500000 pid=42\n", + 3, + True, + ), + ("console", None, 0, False), + ("stderr", None, 0, False), + ) + ) + profiles: list[dict[str, object]] = [] + with tempfile.TemporaryDirectory() as temporary: + snapshot = Path(temporary) / "snapshot" + + def read_chunk(*, timeout: float) -> tuple[str, bytes | None]: + nonlocal clock_ns + del timeout + stream, chunk, advance_ms, publish = next(chunks) + clock_ns += advance_ms * 1_000_000 + if publish: + snapshot.mkdir() + return stream, chunk + + with ( + patch.object( + benchmark, "InteractiveProcess", return_value=FakeInteraction() + ) as interaction, + patch.object(benchmark.threading, "Thread"), + patch.object(benchmark.queue, "Queue") as queues, + patch.object( + benchmark.time, "perf_counter_ns", side_effect=lambda: clock_ns + ), + patch.object(benchmark, "_try_peak_rss", return_value=1024), + patch.object( + benchmark, "process_resource_counters", return_value={} + ) as counters, + ): + queues.return_value.get.side_effect = read_chunk + result = benchmark.capture_snapshot( + ["openvmm"], + snapshot, + processors=1, + timeout=5, + snapshot_profile=profiled, + profile_sink=profiles, + ) + + self.assertEqual(result, (3.0, delay_ms + 23.0, 1.0, 1024)) + self.assertEqual(interaction.call_args.args[0], ["openvmm"]) + self.assertEqual( + interaction.call_args.args[1][benchmark.SNAPSHOT_PROFILE_ENV], "1" + ) + self.assertIs(interaction.call_args.kwargs["separate_stderr"], True) + self.assertEqual(writes[1:], [b"nvx-snapshot\n"]) + self.assertIn( + b"echo OPENVMM-SNAPSHOT-RESTORE-OK\nnvx-exit 0\n", + writes[0], + ) + if profiled: + phases = benchmark.summarize_lifecycle_profiles(profiles)["phases"] + self.assertEqual(phases["capture.snapshot_generation"]["p50_ms"], result[0]) + self.assertEqual( + phases["capture.console_command_round_trip"]["p50_ms"], + delay_ms + 20.0, + ) + else: + counters.assert_not_called() + self.assertEqual(profiles, []) + + # OpenVMM's capture profile records from a Linux/MSHV acceptance run, in + # which the records reached the shared terminal in the middle of the + # guest's dispatch marker line. + INCIDENT_PID = 621736 + INCIDENT_CAPTURE_PROFILE = ( + ("input_gate", 1, 263787, 1422094434, ""), + ("vp_stop_at_io_boundary", 1, 120994, 1422235827, ""), + ("quiesce", 1, 393680, 1422716902, ""), + ("save_state", 1, 647567, 1423376968, ""), + ("mapped_memory_flush", 1, 2700, 1423391067, ""), + ("memory_handle_flush", 1, 1400, 1423507161, " logical_bytes=134217728"), + ( + "publication_state", + 1, + 21499, + 1423577158, + " logical_bytes=2455 allocated_bytes=4096", + ), + ( + "publication_manifest", + 1, + 8899, + 1423643854, + " logical_bytes=1454 allocated_bytes=4096", + ), + ( + "publication_memory", + 1, + 39798, + 1423694652, + " logical_bytes=134217728 allocated_bytes=65011712", + ), + ("publication_staging_sync", 1, 3000, 1423709951, ""), + ("publication_commit", 1, 12700, 1423734150, ""), + ("publication_parent_sync", 1, 2700, 1423748449, ""), + ("publication", 0, 235088, 1423760848, " logical_bytes=134217728"), + ) + + @classmethod + def incident_capture_records(cls) -> bytes: + return b"".join( + ( + f"OPENVMM_SNAPSHOT_PROFILE_V1 operation=capture phase={phase} " + f"exclusive={exclusive} duration_ns={duration} " + f"process_elapsed_ns={elapsed} pid={cls.INCIDENT_PID}{extra}\n" + ).encode() + for phase, exclusive, duration, elapsed, extra in ( + cls.INCIDENT_CAPTURE_PROFILE + ) + ) + + def test_capture_observes_dispatch_marker_split_by_profile_records(self): + records = self.incident_capture_records() + committed = b"INFO microVM snapshot committed; terminating source process\n" + snapshot_requested = threading.Event() + console_split = threading.Event() + stderr_written = threading.Event() + + class FakeProcess: + pid = BenchmarkTests.INCIDENT_PID + returncode = 0 + + def poll(self) -> None: + return None + + def wait(self) -> int: + return 0 + + class FakeInteraction: + def __init__(self, snapshot: Path) -> None: + self.process = FakeProcess() + self.snapshot = snapshot + self.writes: list[bytes] = [] + + def read_output(self, chunks: queue.Queue[bytes | None]) -> None: + chunks.put(benchmark.BOOT_MARKER + b"\r\n") + chunks.put(benchmark.SMP_PROBE_COMPLETION_MARKER + b"\r\n") + if snapshot_requested.wait(5): + # The console relay delivers the dispatch line in two + # writes, around OpenVMM's capture records on stderr. + chunks.put(b"nvx-snapshot\r\nNVX-S") + console_split.set() + if stderr_written.wait(5): + chunks.put(b"NAPSHOT-DISPATCHED\r\n") + chunks.put(None) + + def read_stderr(self, chunks: queue.Queue[bytes | None]) -> None: + if console_split.wait(5): + chunks.put(records) + self.snapshot.mkdir() + chunks.put(committed) + stderr_written.set() + chunks.put(None) + + def write_input(self, data: bytes) -> None: + self.writes.append(data) + if data == b"nvx-snapshot\n": + snapshot_requested.set() + + def close(self) -> None: + pass + + profiles: list[dict[str, object]] = [] + with tempfile.TemporaryDirectory() as temporary: + snapshot = Path(temporary) / "snapshot" + log_path = Path(temporary) / "capture.log" + interaction = FakeInteraction(snapshot) + with ( + patch.object( + benchmark, "InteractiveProcess", return_value=interaction + ) as spawn, + patch.object(benchmark, "_try_peak_rss", return_value=1024), + patch.object(benchmark, "process_resource_counters", return_value={}), + patch.object(benchmark, "terminate"), + ): + result = benchmark.capture_snapshot( + ["openvmm"], + snapshot, + processors=1, + timeout=10, + snapshot_profile=True, + profile_sink=profiles, + log_path=log_path, + ) + log = log_path.read_bytes() + + self.assertIs(spawn.call_args.kwargs["separate_stderr"], True) + self.assertEqual(result[0], 1_903_503 / 1_000_000) + self.assertEqual(result[3], 1024) + self.assertEqual(interaction.writes[1:], [b"nvx-snapshot\n"]) + sample = cast(list[dict[str, object]], profiles[0]["records"]) + self.assertEqual( + [record["phase"] for record in sample if record["source"] == "openvmm"], + [phase for phase, *_ in self.INCIDENT_CAPTURE_PROFILE], + ) + self.assertTrue( + benchmark.contains_output_line( + log, benchmark.SNAPSHOT_GUEST_DISPATCH_MARKER + ) + ) + for line in [*records.splitlines(), committed.removesuffix(b"\n")]: + self.assertTrue(benchmark.contains_output_line(log, line), line) + + def test_capture_samples_storage_from_the_request_through_source_exit(self): + records = self.incident_capture_records() + snapshot_requested = threading.Event() + + class FakeProcess: + pid = BenchmarkTests.INCIDENT_PID + returncode = 0 + + def poll(self) -> None: + return None + + def wait(self) -> int: + return 0 + + class FakeInteraction: + def __init__(self, snapshot: Path) -> None: + self.process = FakeProcess() + self.snapshot = snapshot + self.writes: list[bytes] = [] + + def read_output(self, chunks: queue.Queue[bytes | None]) -> None: + chunks.put(benchmark.BOOT_MARKER + b"\r\n") + chunks.put(benchmark.SMP_PROBE_COMPLETION_MARKER + b"\r\n") + if snapshot_requested.wait(5): + chunks.put( + b"nvx-snapshot\r\n" + + benchmark.SNAPSHOT_GUEST_DISPATCH_MARKER + + b"\r\n" + ) + chunks.put(None) + + def read_stderr(self, chunks: queue.Queue[bytes | None]) -> None: + if snapshot_requested.wait(5): + chunks.put(records) + self.snapshot.mkdir() + chunks.put(None) + + def write_input(self, data: bytes) -> None: + self.writes.append(data) + if data == b"nvx-snapshot\n": + snapshot_requested.set() + + def close(self) -> None: + pass + + class FakeTelemetry: + def __init__(self, interaction: FakeInteraction) -> None: + self.interaction = interaction + self.events: list[str] = [] + self.windows: dict[str, tuple[int, int]] = {} + + def start(self) -> None: + self.events.append(f"start after {len(self.interaction.writes)}") + + def stop(self) -> None: + self.events.append("stop") + + def result(self, windows: dict[str, tuple[int, int]]) -> dict[str, object]: + self.events.append("result") + self.windows = windows + return {"phases": {}} + + for retained in (True, False): + with ( + self.subTest(retained=retained), + tempfile.TemporaryDirectory() as temporary, + ): + snapshot_requested.clear() + snapshot = Path(temporary) / "snapshot" + interaction = FakeInteraction(snapshot) + telemetry = FakeTelemetry(interaction) + profiles: list[dict[str, object]] = [] + with ( + patch.object( + benchmark, "InteractiveProcess", return_value=interaction + ), + patch.object(benchmark, "_try_peak_rss", return_value=1024), + patch.object( + benchmark, "process_resource_counters", return_value={} + ), + patch.object(benchmark, "terminate"), + patch.object( + benchmark.host_telemetry, + "create_storage_telemetry", + return_value=telemetry, + ) as create, + ): + benchmark.capture_snapshot( + ["openvmm"], + snapshot, + processors=1, + timeout=10, + snapshot_profile=True, + profile_sink=profiles if retained else None, + ) + + if not retained: + # Warmups keep no profile, so they sample nothing. + create.assert_not_called() + continue + volumes, origin = create.call_args.args + self.assertEqual(volumes["scratch"], Path(temporary)) + self.assertEqual(volumes["workspace"], BuildConstants.REPO_ROOT) + self.assertIsInstance(origin, int) + # Sampling starts after the SMP probe script, before the request. + self.assertEqual( + telemetry.events, ["start after 1", "stop", "result", "stop"] + ) + self.assertEqual(interaction.writes[1:], [b"nvx-snapshot\n"]) + flush_start, flush_end = telemetry.windows[ + "capture.mapped_memory_flush" + ] + self.assertEqual(flush_end - flush_start, 2700) + generation_start, generation_end = telemetry.windows[ + "capture.snapshot_generation" + ] + self.assertEqual(generation_end - generation_start, 1_903_503) + self.assertEqual(profiles[0]["storage_telemetry"], {"phases": {}}) + + def test_profile_phase_windows_use_the_tightest_clock_offset(self): + records: list[dict[str, object]] = [ + { + "operation": "capture", + "phase": "save_state", + "duration_ns": 1_000, + "process_elapsed_ns": 5_000, + "observer_elapsed_ns": 20_500, + "source": "openvmm", + }, + { + "operation": "capture", + "phase": "mapped_memory_flush", + "duration_ns": 900_000, + "process_elapsed_ns": 905_000, + "observer_elapsed_ns": 920_000, + "source": "openvmm", + }, + { + "operation": "capture", + "phase": "snapshot_generation", + "duration_ns": 950_000, + "process_elapsed_ns": 955_000, + "source": "openvmm_clock", + }, + { + "operation": "capture", + "phase": "source_teardown", + "duration_ns": 9_000, + "observer_elapsed_ns": 990_000, + "source": "benchmark_observer", + }, + ] + + # The save_state record arrived 0.5 us late, so the flush record's + # 15 us offset bounds OpenVMM's clock. + self.assertEqual( + benchmark.snapshot_profile_phase_windows(records), + { + "capture.save_state": (19_000, 20_000), + "capture.mapped_memory_flush": (20_000, 920_000), + "capture.snapshot_generation": (20_000, 970_000), + "capture.source_teardown": (981_000, 990_000), + }, + ) + + def test_capture_telemetry_line_describes_the_ram_flush_window(self): + records = [ + { + "operation": "capture", + "phase": "mapped_memory_flush", + "duration_ns": 1_165_314_800, + } + ] + sample: dict[str, object] = { + "records": records, + "storage_telemetry": { + "volumes": {"F:": ["scratch"]}, + "phases": { + "capture.mapped_memory_flush": { + "volumes": { + "F:": { + "write_mib_per_second": 128.44, + "average_write_latency_ms": 31.2, + "average_queue_length": 3.94, + "busy_percent": 99.4, + } + }, + "cpu": {"busy_percent": 6.2}, + } + }, + }, + } + + self.assertEqual( + benchmark.describe_capture_telemetry(sample), + "mapped_memory_flush 1165.315 ms: scratch F: write 128.4 MiB/s at " + "31.2 ms, queue 3.9, 99% busy; CPU 6% busy", + ) + self.assertIsNone(benchmark.describe_capture_telemetry({"records": records})) + + def test_result_documents_record_host_provenance(self): + args = nvx.parse_args(["benchmark"]) + with ( + patch.object(benchmark, "_git_revision", return_value="revision"), + patch.object( + benchmark, + "benchmark_host_provenance", + return_value={"runner_name": "azure-windows-3"}, + ) as host, + ): + document = benchmark.result_document(args, None, None) + + host.assert_called_once_with(args) + self.assertEqual(document["host"], {"runner_name": "azure-windows-3"}) + + def test_host_provenance_names_workspace_temporary_and_scratch_volumes(self): + with tempfile.TemporaryDirectory() as temporary: + scratch = Path(temporary).resolve() / "scratch" + scratch.mkdir() + args = nvx.parse_args(["benchmark", "--scratch-dir", str(scratch)]) + system_temporary = Path(tempfile.gettempdir()).resolve() + with ( + benchmark.benchmark_scratch_directory(args), + patch.object( + benchmark.host_telemetry, + "host_provenance", + return_value={"schema_version": 1}, + ) as provenance, + ): + host = benchmark.benchmark_host_provenance(args) + + self.assertEqual(host, {"schema_version": 1}) + provenance.assert_called_once_with( + { + "workspace": BuildConstants.REPO_ROOT, + "temporary": system_temporary, + "scratch": scratch, + } + ) + + def test_device_io_resume_ignores_host_provenance(self): + args = nvx.parse_args( + [ + "benchmark", + "--suite", + "device-io", + "--backend", + "whp", + "--platform", + "windows-whp-baremetal", + ] + ) + benchmark.apply_benchmark_suite_defaults(args) + hosts = iter(({"machine_name": "first"}, {"machine_name": "second"})) + + def host_provenance(_: argparse.Namespace) -> dict[str, str]: + return next(hosts, {"machine_name": "third"}) + + with ( + tempfile.TemporaryDirectory() as temporary, + patch.object( + benchmark, "benchmark_host_provenance", side_effect=host_provenance + ), + patch.object(benchmark, "sha256_file", return_value="0" * 64), + patch.object( + benchmark, + "_device_io_helper_provenance", + return_value={"source": "helper"}, + ), + patch.object(benchmark, "_git_revision", return_value="revision"), + patch.object(benchmark, "_git_status", return_value=[]), + ): + output = Path(temporary) + + def write() -> Path: + return benchmark.write_benchmark_metadata( + args, + output, + Path("openvmm.exe"), + Path("vmlinux"), + Path("initramfs.cpio.gz"), + "whp", + ) + + path = write() + written = path.read_text(encoding="utf-8") + self.assertEqual(json.loads(written)["host"], {"machine_name": "first"}) + self.assertEqual(write(), path) + self.assertEqual(path.read_text(encoding="utf-8"), written) + args.device_io_port += 1 + with self.assertRaisesRegex( + ValueError, "resume controls or provenance do not match" + ): + write() + + def test_measure_once_observes_restore_marker_split_by_profile_record(self): + record = ( + b"OPENVMM_SNAPSHOT_PROFILE_V1 operation=restore " + b"phase=guest_repair_gate exclusive=0 duration_ns=1500 " + b"process_elapsed_ns=90000000 pid=4321\n" + ) + console_split = threading.Event() + stderr_written = threading.Event() + + class FakeInteraction: + def __init__(self) -> None: + self.process = MagicMock(pid=4321) + self.process.poll.return_value = None + + def read_output(self, chunks: queue.Queue[bytes | None]) -> None: + chunks.put(b"OPENVMM-SNAP") + console_split.set() + if stderr_written.wait(5): + chunks.put(b"SHOT-RESTORE-OK\r\n") + chunks.put(None) + + def read_stderr(self, chunks: queue.Queue[bytes | None]) -> None: + if console_split.wait(5): + chunks.put(record) + stderr_written.set() + chunks.put(None) + + def write_input(self, data: bytes) -> None: + raise AssertionError(f"unexpected input: {data!r}") + + def close(self) -> None: + pass + + profiles: list[dict[str, object]] = [] + with tempfile.TemporaryDirectory() as temporary: + log_path = Path(temporary) / "restore.log" + with ( + patch.object( + benchmark, "InteractiveProcess", return_value=FakeInteraction() + ) as spawn, + patch.object(benchmark, "live_peak_rss_bytes", return_value=1024), + patch.object(benchmark, "wait_for_process_exit", return_value=0), + patch.object(benchmark, "process_resource_counters", return_value={}), + ): + result = benchmark.measure_once( + ["openvmm"], + environment={}, + timeout=10, + marker=benchmark.RESTORE_MARKER, + marker_must_be_line=True, + guest_exit_prequeued=True, + snapshot_profile=True, + profile_sink=profiles, + log_path=log_path, + ) + log = log_path.read_bytes() + + self.assertIs(spawn.call_args.kwargs["separate_stderr"], True) + self.assertEqual(result[1], 1024) + records = cast(list[dict[str, object]], profiles[0]["records"]) + self.assertIn("guest_repair_gate", [item["phase"] for item in records]) + self.assertTrue(benchmark.contains_output_line(log, benchmark.RESTORE_MARKER)) + self.assertTrue(benchmark.contains_output_line(log, record.removesuffix(b"\n"))) + + def test_measure_once_keeps_profile_records_observed_after_the_marker(self): + records = ( + b"OPENVMM_SNAPSHOT_PROFILE_V1 operation=startup " + b"phase=worker_launch exclusive=0 duration_ns=2000000 " + b"process_elapsed_ns=5000000 pid=4321\n" + b"OPENVMM_SNAPSHOT_PROFILE_V1 operation=restore " + b"phase=device_start exclusive=1 duration_ns=300000 " + b"process_elapsed_ns=80000000 pid=4321\n" + b"OPENVMM_SNAPSHOT_PROFILE_V1 operation=restore " + b"phase=guest_repair_gate exclusive=0 duration_ns=1500 " + b"process_elapsed_ns=90000000 pid=4321\n" + ) + torn_down = threading.Event() + + class FakeInteraction: + def __init__(self) -> None: + self.process = MagicMock(pid=4321) + self.process.poll.return_value = None + + def read_output(self, chunks: queue.Queue[bytes | None]) -> None: + chunks.put(benchmark.RESTORE_MARKER + b"\r\n") + chunks.put(None) + + def read_stderr(self, chunks: queue.Queue[bytes | None]) -> None: + # The stderr reader falls behind: OpenVMM's records arrive + # only once the guest marker has been handled. + if torn_down.wait(5): + chunks.put(records[:100]) + chunks.put(records[100:]) + chunks.put(None) + + def write_input(self, data: bytes) -> None: + raise AssertionError(f"unexpected input: {data!r}") + + def close(self) -> None: + pass + + def counters(_pid: int) -> dict[str, int]: + return {} if torn_down.is_set() else {"rss_bytes": 7} + + def wait_for_exit(_process: object, _timeout: float) -> int: + torn_down.set() + return 0 + + profiles: list[dict[str, object]] = [] + with ( + patch.object( + benchmark, "InteractiveProcess", return_value=FakeInteraction() + ), + patch.object(benchmark, "live_peak_rss_bytes", return_value=1024), + patch.object(benchmark, "wait_for_process_exit", side_effect=wait_for_exit), + patch.object(benchmark, "process_resource_counters", side_effect=counters), + ): + benchmark.measure_once( + ["openvmm"], + environment={}, + timeout=10, + marker=benchmark.RESTORE_MARKER, + marker_must_be_line=True, + guest_exit_prequeued=True, + snapshot_profile=True, + profile_sink=profiles, + ) + + by_phase = { + str(record["phase"]): record + for record in cast(list[dict[str, object]], profiles[0]["records"]) + } + self.assertLessEqual( + {"process_startup", "worker_launch", "device_start", "guest_repair_gate"}, + set(by_phase), + ) + self.assertEqual(by_phase["process_startup"]["duration_ns"], 3_000_000) + # The device_start observation followed the marker, so it cannot bound + # the interval to readiness. + self.assertNotIn("resume_to_readiness", by_phase) + readiness = by_phase["process_launch_to_readiness"] + self.assertEqual(readiness["host_counters"], {"rss_bytes": 7}) + + def test_measure_once_reads_output_to_its_end_before_finishing(self): + record = ( + b"OPENVMM_SNAPSHOT_PROFILE_V1 operation=restore " + b"phase=guest_repair_gate exclusive=0 duration_ns=1500 " + b"process_elapsed_ns=90000000 pid=4321\n" + ) + split = record.index(b"duration_ns=") + len(b"duration_ns=") + + class FakeInteraction: + def __init__(self, exited: threading.Event) -> None: + self.process = MagicMock(pid=4321) + self.process.poll.return_value = None + self.exited = exited + + def read_output(self, chunks: queue.Queue[bytes | None]) -> None: + chunks.put(benchmark.RESTORE_MARKER + b"\r\n") + chunks.put(None) + + def read_stderr(self, chunks: queue.Queue[bytes | None]) -> None: + if self.exited.wait(5): + chunks.put(record[:split]) + # The rest of the record outlasts the bounded drain that + # error paths use. + time.sleep(0.2) + chunks.put(record[split:]) + chunks.put(None) + + def write_input(self, data: bytes) -> None: + raise AssertionError(f"unexpected input: {data!r}") + + def close(self) -> None: + pass + + for profiled in (False, True): + with self.subTest(profiled=profiled): + exited = threading.Event() + + def wait_for_exit( + _process: object, _timeout: float, exited: threading.Event = exited + ) -> int: + exited.set() + return 0 + + profiles: list[dict[str, object]] = [] + with tempfile.TemporaryDirectory() as temporary: + log_path = Path(temporary) / "restore.log" + with ( + patch.object( + benchmark, + "InteractiveProcess", + return_value=FakeInteraction(exited), + ), + patch.object(benchmark, "OUTPUT_DRAIN_TIMEOUT_SECONDS", 0.05), + patch.object( + benchmark, "live_peak_rss_bytes", return_value=1024 + ), + patch.object( + benchmark, + "wait_for_process_exit", + side_effect=wait_for_exit, + ), + patch.object( + benchmark, "process_resource_counters", return_value={} + ), + ): + benchmark.measure_once( + ["openvmm"], + environment={}, + timeout=10, + marker=benchmark.RESTORE_MARKER, + marker_must_be_line=True, + guest_exit_prequeued=True, + snapshot_profile=profiled, + profile_sink=profiles if profiled else None, + log_path=log_path, + ) + log = log_path.read_bytes() + + self.assertTrue( + benchmark.contains_output_line(log, record.removesuffix(b"\n")) + ) + if profiled: + records = cast(list[dict[str, object]], profiles[0]["records"]) + repair = next( + item for item in records if item["phase"] == "guest_repair_gate" + ) + self.assertEqual(repair["duration_ns"], 1500) + self.assertEqual(repair["process_elapsed_ns"], 90_000_000) + else: + self.assertEqual(profiles, []) + + def test_measure_once_reports_output_that_does_not_reach_eof(self): + release = threading.Event() + self.addCleanup(release.set) + + class FakeInteraction: + def __init__(self) -> None: + self.process = MagicMock(pid=4321) + self.process.poll.return_value = None + + def read_output(self, chunks: queue.Queue[bytes | None]) -> None: + chunks.put(benchmark.RESTORE_MARKER + b"\r\n") + chunks.put(None) + + def read_stderr(self, chunks: queue.Queue[bytes | None]) -> None: + chunks.put( + b"OPENVMM_SNAPSHOT_PROFILE_V1 operation=restore " + b"phase=guest_repair_gate exclusive=0 duration_ns=" + ) + release.wait(10) + chunks.put(None) + + def write_input(self, data: bytes) -> None: + raise AssertionError(f"unexpected input: {data!r}") + + def close(self) -> None: + pass + + profiles: list[dict[str, object]] = [] + with ( + patch.object( + benchmark, "InteractiveProcess", return_value=FakeInteraction() + ), + patch.object(benchmark, "OUTPUT_DRAIN_TIMEOUT_SECONDS", 0.05), + patch.object(benchmark, "live_peak_rss_bytes", return_value=1024), + patch.object(benchmark, "wait_for_process_exit", return_value=0), + patch.object(benchmark, "process_resource_counters", return_value={}), + patch.object(benchmark, "terminate") as terminate, + self.assertRaisesRegex( + RuntimeError, r"did not reach EOF within 0\.5s" + ) as raised, + ): + benchmark.measure_once( + ["openvmm"], + environment={}, + timeout=0.5, + marker=benchmark.RESTORE_MARKER, + marker_must_be_line=True, + guest_exit_prequeued=True, + snapshot_profile=True, + profile_sink=profiles, + ) + + self.assertEqual(profiles, []) + terminate.assert_called_once() + self.assertIn("phase=guest_repair_gate", str(raised.exception)) + + def test_measure_once_returns_when_a_descendant_keeps_the_output_open(self): + # The descendant inherits the console output and stderr and outlives + # the launched process, so neither stream reaches EOF. + child = ( + "import subprocess, sys\n" + "holder = subprocess.Popen(\n" + " [sys.executable, '-c', 'import time; time.sleep(60)'],\n" + " stdin=subprocess.DEVNULL,\n" + ")\n" + "with open(sys.argv[1], 'w') as stream:\n" + " stream.write(str(holder.pid))\n" + "print('NVX-OUTPUT-HOLDER-READY', flush=True)\n" + ) + with tempfile.TemporaryDirectory() as temporary: + holder_pid = Path(temporary) / "holder.pid" + + def kill_holder() -> None: + try: + os.kill(int(holder_pid.read_text()), signal.SIGTERM) + except (OSError, ValueError): + pass + + started = time.monotonic() + try: + with ( + patch.object(benchmark, "OUTPUT_DRAIN_TIMEOUT_SECONDS", 0.05), + self.assertRaisesRegex( + RuntimeError, r"did not reach EOF within 3s of its exit" + ), + ): + benchmark.measure_once( + [sys.executable, "-I", "-c", child, str(holder_pid)], + environment=dict(os.environ), + timeout=3, + marker=b"NVX-OUTPUT-HOLDER-READY", + marker_must_be_line=True, + guest_exit_prequeued=True, + log_path=Path(temporary) / "launch.log", + ) + elapsed = time.monotonic() - started + finally: + kill_holder() + + # Closing a pipe that a reader is blocked on would wait for the + # descendant to exit. + self.assertLess(elapsed, 20) + + def test_output_reader_reads_and_stops_above_fd_setsize(self): + if sys.platform != "linux": + self.skipTest("select() limits descriptors to FD_SETSIZE on POSIX") + import fcntl + import resource + + floor = 1100 + limit = floor + 64 + soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) + if soft < limit: + if hard != resource.RLIM_INFINITY and hard < limit: + self.skipTest(f"the RLIMIT_NOFILE hard limit {hard} is too low") + resource.setrlimit(resource.RLIMIT_NOFILE, (limit, hard)) + self.addCleanup(resource.setrlimit, resource.RLIMIT_NOFILE, (soft, hard)) + read_fd, write_fd = os.pipe() + self.addCleanup(os.close, write_fd) + # F_DUPFD takes the lowest free descriptor at or above the floor, so it + # cannot replace a descriptor that something else owns. + high = fcntl.fcntl(read_fd, fcntl.F_DUPFD, floor) + os.close(read_fd) + os.set_inheritable(high, False) + self.addCleanup(os.close, high) + self.assertGreaterEqual(high, floor) + + reader = benchmark._OutputReader(high) + chunks: queue.Queue[bytes | None] = queue.Queue() + thread = threading.Thread(target=reader.run, args=(chunks,), daemon=True) + thread.start() + os.write(write_fd, b"console output") + self.assertEqual(chunks.get(timeout=5), b"console output") + # The write end stays open, so only stop() can end the read. + self.assertTrue(reader.stop(5)) + self.assertIsNone(chunks.get(timeout=5)) + thread.join(5) + self.assertFalse(thread.is_alive()) + + def test_guest_failure_report_includes_stderr_written_after_the_marker(self): + terminated = threading.Event() + + class FakeInteraction: + def __init__(self) -> None: + self.process = MagicMock(pid=4321) + self.process.poll.return_value = None + + def read_output(self, chunks: queue.Queue[bytes | None]) -> None: + chunks.put(b"NVX-RESTORE-PROCESSORS-FAIL unstable-tsc\r\n") + chunks.put(None) + + def read_stderr(self, chunks: queue.Queue[bytes | None]) -> None: + if terminated.wait(5): + chunks.put(b"WARN host diagnostics after the guest failure\n") + chunks.put(None) + + def write_input(self, data: bytes) -> None: + raise AssertionError(f"unexpected input: {data!r}") + + def close(self) -> None: + pass + + def terminate_openvmm(_process: object) -> None: + terminated.set() + + with ( + patch.object( + benchmark, "InteractiveProcess", return_value=FakeInteraction() + ), + patch.object( + benchmark, "terminate", side_effect=terminate_openvmm + ) as terminate, + self.assertRaises(benchmark.GuestFailureReported) as raised, + ): + benchmark.measure_once( + ["openvmm"], + environment={}, + timeout=10, + marker=b"NVX-RESTORE-PROCESSORS-OK count=8", + marker_must_be_line=True, + guest_exit_prequeued=True, + failure_marker=b"NVX-RESTORE-PROCESSORS-FAIL", + ) + + terminate.assert_called_once() + self.assertEqual( + raised.exception.line, "NVX-RESTORE-PROCESSORS-FAIL unstable-tsc" + ) + self.assertIn("host diagnostics after the guest failure", str(raised.exception)) + + def test_separated_output_keeps_process_output_lines_whole(self): + record = ( + b"OPENVMM_SNAPSHOT_PROFILE_V1 operation=capture phase=input_gate " + b"exclusive=1 duration_ns=263787 process_elapsed_ns=1422094434 pid=1\n" + ) + child = ( + "import os, time; " + "os.write(1, b'NVX-S'); time.sleep(0.05); " + f"os.write(2, {record!r}); time.sleep(0.05); " + "os.write(1, b'NAPSHOT-DISPATCHED\\n~ # ')" + ) + interaction = benchmark.InteractiveProcess( + [sys.executable, "-I", "-c", child], + dict(os.environ), + separate_stderr=True, + ) + stderr = bytearray() + try: + output = benchmark.SeparatedOutput(interaction) + deadline = time.monotonic() + 30 + while not output.closed: + stream, chunk = output.get(max(0.0, deadline - time.monotonic())) + if stream == "stderr" and chunk is not None: + stderr.extend(chunk) + self.assertEqual(interaction.process.wait(timeout=30), 0) + finally: + benchmark.terminate(interaction.process) + interaction.close() + + marker = benchmark.SNAPSHOT_GUEST_DISPATCH_MARKER + self.assertTrue(benchmark.contains_output_line(output.console, marker)) + self.assertNotIn(b"OPENVMM_SNAPSHOT_PROFILE_V1", output.console) + self.assertIn(record, bytes(stderr)) + contents = output.contents() + self.assertTrue(benchmark.contains_output_line(contents, marker)) + self.assertTrue( + benchmark.contains_output_line(contents, record.removesuffix(b"\n")) + ) + self.assertTrue(contents.endswith(b"~ # ")) + + def test_interactive_process_gives_stderr_its_own_pipe_on_request(self): + for platform in ("linux", "win32"): + for separate_stderr in (False, True): + with self.subTest(platform=platform, separate=separate_stderr): + terminal: list[int] = [] + + def open_terminal( + terminal: list[int] = terminal, + ) -> tuple[int, int]: + terminal.extend(os.pipe()) + return terminal[0], terminal[1] + + process = MagicMock(pid=4321) + if not separate_stderr: + process.stderr = None + with ( + patch.object(benchmark.sys, "platform", platform), + patch.object( + benchmark.os, + "openpty", + side_effect=open_terminal, + create=True, + ), + patch.object( + benchmark.subprocess, "Popen", return_value=process + ) as popen, + ): + interaction = benchmark.InteractiveProcess( + ["openvmm"], {}, separate_stderr=separate_stderr + ) + try: + stderr = popen.call_args.kwargs["stderr"] + if separate_stderr: + self.assertEqual(stderr, subprocess.PIPE) + elif platform == "linux": + self.assertEqual(stderr, terminal[1]) + else: + self.assertEqual(stderr, subprocess.STDOUT) + if not separate_stderr: + chunks: queue.Queue[bytes | None] = queue.Queue() + with self.assertRaisesRegex( + RuntimeError, "shares the console stream" + ): + interaction.read_stderr(chunks) + self.assertIsNone(chunks.get_nowait()) + finally: + interaction.close() + if separate_stderr: + process.stderr.close.assert_called_once_with() + + def test_warm_snapshot_cache_reads_every_artifact(self): + with tempfile.TemporaryDirectory() as temporary: + snapshot = Path(temporary) + for index, name in enumerate(benchmark.SNAPSHOT_FILENAMES): + (snapshot / name).write_bytes(bytes([index]) * 32) + + benchmark.warm_snapshot_artifacts(snapshot) + + def test_snapshot_artifact_validation_preserves_error_prefix(self): + with tempfile.TemporaryDirectory() as temporary: + snapshot = Path(temporary) + for prefix in ("snapshot", "console/active snapshot"): + with self.subTest(prefix=prefix): + with self.assertRaises(RuntimeError) as context: + benchmark._require_snapshot_artifacts(snapshot, prefix) + + self.assertEqual( + str(context.exception), + f"{prefix} did not publish {snapshot / 'manifest.bin'}", + ) + + def test_measure_once_does_not_resend_prequeued_guest_exit(self): + class FakeProcess: + pid = 123 + returncode = 0 + + def poll(self): + return None + + def terminate(self): + raise AssertionError("guest-exit teardown terminated the host process") + + class FakeInteraction: + def __init__(self): + self.process = FakeProcess() + self.writes: list[bytes] = [] + + def read_output(self, chunks: queue.Queue[bytes | None]): + chunks.put(b"/ # echo OPENVMM-SNAPSHOT-RESTORE-OK\r\n") + chunks.put(b"OPENVMM-SNAPSHOT-RESTORE-OK\r\n") + chunks.put(None) + + def read_stderr(self, chunks: queue.Queue[bytes | None]): + chunks.put(None) + + def write_input(self, data: bytes): + self.writes.append(data) + + def close(self): + pass + + interaction = FakeInteraction() + with ( + patch.object(benchmark, "InteractiveProcess", return_value=interaction), + patch.object(benchmark, "live_peak_rss_bytes", return_value=1024), + patch.object(benchmark, "wait_for_process_exit", return_value=0), + patch.object( + benchmark.time, + "perf_counter_ns", + side_effect=[0, 10_000_000, 17_000_000], + ), + ): + result = benchmark.measure_once( + ["openvmm"], + environment={}, + timeout=1, + marker=benchmark.RESTORE_MARKER, + marker_must_be_line=True, + guest_exit_prequeued=True, + ) + + self.assertEqual(result, (10.0, 1024, 7.0, 17.0)) + self.assertEqual(interaction.writes, []) + + def test_measure_once_reports_missing_rss_when_openvmm_exits_first(self): + class FakeProcess: + pid = 123 + + def __init__(self, status: int): + self.status = status + self.exited = False + self.returncode: int | None = None + + def poll(self): + if self.exited: + self.returncode = self.status + return self.returncode + + def wait(self, timeout: float | None = None) -> int: + del timeout + return self.status + + def terminate(self): + raise AssertionError("unexpected process termination") + + class FakeInteraction: + def __init__(self, status: int): + self.process = FakeProcess(status) + + def read_output(self, chunks: queue.Queue[bytes | None]): + self.process.exited = True + chunks.put(benchmark.RESTORE_MARKER + b"\n") + chunks.put(None) + + def read_stderr(self, chunks: queue.Queue[bytes | None]): + chunks.put(None) + + def write_input(self, data: bytes): + raise AssertionError(f"unexpected input: {data!r}") + + def close(self): + pass + + for status in (0, 1): + with self.subTest(status=status): + interaction = FakeInteraction(status) + + def linux_peak_rss_bytes( + pid: int, interaction: FakeInteraction = interaction + ) -> int | None: + self.assertEqual(pid, 123) + # A zombie's /proc status no longer reports VmHWM. + return None if interaction.process.exited else 1024 + + with ( + patch.object( + benchmark, "InteractiveProcess", return_value=interaction + ), + patch.object( + benchmark, + "_linux_live_peak_rss_bytes", + side_effect=linux_peak_rss_bytes, + ), + patch.object(benchmark, "windows_peak_rss_bytes", return_value=1), + patch.object( + benchmark, "wait_for_process_exit", return_value=status + ) as wait, + patch.object( + benchmark.time, + "perf_counter_ns", + side_effect=[0, 10_000_000, 17_000_000], + ), + ): + if status == 0: + result = benchmark.measure_once( + ["openvmm"], + environment={}, + timeout=1, + marker=benchmark.RESTORE_MARKER, + marker_must_be_line=True, + guest_exit_prequeued=True, + ) + self.assertEqual(result, (10.0, None, 7.0, 17.0)) + else: + with self.assertRaisesRegex( + RuntimeError, "status 1 during teardown" + ): + benchmark.measure_once( + ["openvmm"], + environment={}, + timeout=1, + marker=benchmark.RESTORE_MARKER, + marker_must_be_line=True, + guest_exit_prequeued=True, + ) + + wait.assert_called_once_with( + interaction.process, benchmark.TEARDOWN_TIMEOUT_SECONDS + ) + + def test_completed_output_line_with_prefix_requires_a_whole_line(self): + prefix = b"NVX-RESTORE-PROCESSORS-FAIL" + self.assertIsNone( + benchmark.completed_output_line_with_prefix( + b"NVX-RESTORE-PROCESSORS-FAIL unst", prefix + ) + ) + self.assertIsNone( + benchmark.completed_output_line_with_prefix( + b'> echo "NVX-RESTORE-PROCESSORS-FAIL count=$n"\r\n', prefix + ) + ) + self.assertEqual( + benchmark.completed_output_line_with_prefix( + b"guest\r\nNVX-RESTORE-PROCESSORS-FAIL unstable-tsc\r\nmore", prefix + ), + b"NVX-RESTORE-PROCESSORS-FAIL unstable-tsc", + ) + + def test_measure_once_stops_at_a_completed_guest_failure_marker(self): + class FakeInteraction: + def __init__(self): + self.process = MagicMock(pid=123) + self.process.poll.return_value = None + + def read_output(self, chunks: queue.Queue[bytes | None]): + chunks.put(b'> echo "NVX-RESTORE-PROCESSORS-FAIL count=$n"\r\n') + chunks.put(b"Measured 6 cycles TSC warp between CPUs\r\n") + chunks.put(b"NVX-RESTORE-PROCESSORS-FAIL unst") + chunks.put(b"able-tsc\r\n") + chunks.put(None) + + def read_stderr(self, chunks: queue.Queue[bytes | None]): + chunks.put(None) + + def write_input(self, data: bytes): + raise AssertionError(f"unexpected input: {data!r}") + + def close(self): + pass + + interaction = FakeInteraction() + with tempfile.TemporaryDirectory() as temporary: + log_path = Path(temporary) / "restore.log" + with ( + patch.object(benchmark, "InteractiveProcess", return_value=interaction), + patch.object(benchmark, "terminate") as terminate, + patch.object(benchmark, "wait_for_process_exit") as wait, + ): + with self.assertRaises(benchmark.GuestFailureReported) as raised: + benchmark.measure_once( + ["openvmm"], + environment={}, + timeout=60, + marker=b"NVX-RESTORE-PROCESSORS-OK count=8", + marker_must_be_line=True, + guest_exit_prequeued=True, + log_path=log_path, + failure_marker=b"NVX-RESTORE-PROCESSORS-FAIL", + ) + log = log_path.read_bytes() + + self.assertEqual( + raised.exception.line, "NVX-RESTORE-PROCESSORS-FAIL unstable-tsc" + ) + self.assertIn("TSC warp", raised.exception.output_tail) + self.assertIn("--- OpenVMM output ---", str(raised.exception)) + terminate.assert_called_once_with(interaction.process) + wait.assert_not_called() + self.assertTrue(log.endswith(b"NVX-RESTORE-PROCESSORS-FAIL unstable-tsc\r\n")) + + def test_guest_runner_preserves_primary_and_close_failures_with_bounded_tail(self): + class FakeProcess: + pid = 123 + + def poll(self): + return 7 + + def wait(self): + return 7 + + class FakeInteraction: + process = FakeProcess() + containment = None + + def read_output(self, chunks: queue.Queue[bytes | None]): + chunks.put(benchmark.BOOT_MARKER + b"\n" + b"x" * 5000 + b"\nDONE\n") + chunks.put(None) + + def write_input(self, _data: bytes): + pass + + def close(self): + raise OSError("close failed") + + with tempfile.TemporaryDirectory() as temporary: + log_path = Path(temporary) / "guest.log" + with ( + patch.object( + benchmark, "InteractiveProcess", return_value=FakeInteraction() + ), + patch.object(benchmark, "_try_peak_rss", return_value=1024), + patch.object(benchmark, "terminate") as terminate, + ): + with self.assertRaisesRegex( + RuntimeError, "OpenVMM exited with status 7" + ) as raised: + benchmark.run_guest_script( + ["openvmm"], + "guest command\n", + b"DONE", + timeout=1, + log_path=log_path, + ) + + error = raised.exception + cleanup_errors = cast( + tuple[BaseException, ...], + error.cleanup_errors, # type: ignore[attr-defined] + ) + self.assertEqual([str(item) for item in cleanup_errors], ["close failed"]) + self.assertIn("--- OpenVMM output ---", str(error)) + self.assertIn("--- cleanup failures ---", str(error)) + self.assertNotIn(benchmark.BOOT_MARKER.decode(), str(error)) + self.assertGreater(len(log_path.read_bytes()), 4096) + self.assertTrue(log_path.read_bytes().endswith(b"\nDONE\n")) + terminate.assert_called_once() + + def test_guest_runner_propagates_success_only_close_failure(self): + class FakeProcess: + pid = 123 + + def poll(self): + return 0 + + def wait(self): + return 0 + + class FakeInteraction: + process = FakeProcess() + containment = None + + def read_output(self, chunks: queue.Queue[bytes | None]): + chunks.put(benchmark.BOOT_MARKER + b"\nDONE\n") + chunks.put(None) + + def write_input(self, _data: bytes): + pass + + def close(self): + raise OSError("close after success") + + with ( + patch.object( + benchmark, "InteractiveProcess", return_value=FakeInteraction() + ), + patch.object(benchmark, "_try_peak_rss", return_value=1024), + ): + with self.assertRaisesRegex(OSError, "close after success"): + benchmark.run_guest_script( + ["openvmm"], + "guest command\n", + b"DONE", + timeout=1, + ) + + def test_live_peak_rss_samples_linux_process_without_reaping(self): + process = MagicMock(pid=123) + with ( + patch.object(benchmark.os, "name", "posix"), + patch.object(benchmark.sys, "platform", "linux"), + patch.object( + benchmark, "_linux_live_peak_rss_bytes", side_effect=[4096, None, 0] + ) as read, + ): + self.assertEqual(benchmark.live_peak_rss_bytes(process), 4096) + self.assertIsNone(benchmark.live_peak_rss_bytes(process)) + with self.assertRaisesRegex(RuntimeError, "peak RSS 0 bytes"): + benchmark.live_peak_rss_bytes(process) + + self.assertEqual(read.call_args_list, [call(123)] * 3) + process.poll.assert_not_called() + + def test_live_peak_rss_accepts_only_running_windows_samples(self): + running = MagicMock(pid=7) + running.poll.return_value = None + exited = MagicMock(pid=8) + exited.poll.return_value = 0 + racing = MagicMock(pid=9) + racing.poll.return_value = None + + def read_counters(pid: int) -> int: + if pid == racing.pid: + racing.poll.return_value = 0 + return 8192 + + with ( + patch.object(benchmark.os, "name", "nt"), + patch.object( + benchmark, "windows_peak_rss_bytes", side_effect=read_counters + ) as read, + ): + self.assertEqual(benchmark.live_peak_rss_bytes(running), 8192) + # A retained handle still reports counters, including teardown. + self.assertIsNone(benchmark.live_peak_rss_bytes(exited)) + self.assertIsNone(benchmark.live_peak_rss_bytes(racing)) + + self.assertEqual(read.call_args_list, [call(7), call(8), call(9)]) + with ( + patch.object(benchmark.os, "name", "nt"), + patch.object( + benchmark, "windows_peak_rss_bytes", side_effect=OSError("closed") + ), + ): + self.assertIsNone(benchmark.live_peak_rss_bytes(exited)) + with self.assertRaisesRegex(OSError, "closed"): + benchmark.live_peak_rss_bytes(running) + + def test_linux_live_peak_rss_requires_process_address_space(self): + running = "Name:\topenvmm\nState:\tS (sleeping)\nVmHWM:\t 59008 kB\n" + zombie = "Name:\topenvmm\nState:\tZ (zombie)\nThreads:\t1\n" + with patch.object( + Path, + "read_text", + autospec=True, + side_effect=[running, zombie, FileNotFoundError(), ProcessLookupError()], + ) as read: + self.assertEqual(benchmark._linux_live_peak_rss_bytes(123), 59008 * 1024) + for _ in range(3): + self.assertIsNone(benchmark._linux_live_peak_rss_bytes(123)) + + self.assertEqual(read.call_args.args[0], Path("/proc/123/status")) + with ( + patch.object(Path, "read_text", side_effect=PermissionError()), + self.assertRaises(PermissionError), + ): + benchmark._linux_live_peak_rss_bytes(123) + + def test_builds_isolated_workload_command(self): + command = benchmark.workload_boot_command( + Path("openvmm"), + "kvm", + Path("vmlinux"), + Path("initramfs.cpio.gz"), + 128, + "quiet loglevel=0 nokaslr", + processors=8, + command_prefix=("taskset", "-c", "2-3"), + network="10.0.0.2/24", + mount="/mnt/host,C:/work,rw", + ) + + self.assertEqual(command[:4], ["taskset", "-c", "2-3", "openvmm"]) + self.assertEqual( + command[4:10], + [ + "--single-process", + "--machine", + "microvm", + "--processors", + "8", + "--hypervisor", + ], + ) + self.assertIn("quiet loglevel=0 nokaslr", command) + self.assertEqual( + command[-6:], + [ + "--net", + "10.0.0.2/24", + "--network-profile", + "portable", + "--mount", + "/mnt/host,C:/work,rw", + ], + ) + + def test_benchmark_network_requires_explicit_profile(self): + args = nvx.parse_args( + [ + "benchmark", + "--net", + "10.0.0.2/24", + "--network-profile", + "portable", + ] + ) + self.assertEqual(args.network_profile, "portable") + + args.network_profile = None + with self.assertRaisesRegex(ValueError, "--net and --network-profile"): + benchmark.run(args) + + def test_benchmark_scratch_directory_routes_temporary_files(self): + with tempfile.TemporaryDirectory() as temporary: + scratch = Path(temporary).resolve() / "scratch" + scratch.mkdir() + args = nvx.parse_args(["benchmark", "--scratch-dir", str(scratch)]) + previous = tempfile.tempdir + + with ( + benchmark.benchmark_scratch_directory(args), + patch.object(benchmark, "_git_revision", return_value="revision"), + ): + self.assertEqual(Path(tempfile.gettempdir()), scratch) + with tempfile.TemporaryDirectory(prefix="openvmm-e2e-") as snapshot: + self.assertEqual(Path(snapshot).parent, scratch) + document = benchmark.result_document(args, None, None) + + self.assertEqual(tempfile.tempdir, previous) + self.assertEqual(args.scratch_dir, scratch) + self.assertEqual( + document["controls"]["scratch_directory"], + str(scratch), + ) + + def test_benchmark_scratch_directory_defaults_to_system_temporary(self): + args = nvx.parse_args(["benchmark"]) + previous = tempfile.tempdir + + with ( + benchmark.benchmark_scratch_directory(args), + patch.object(benchmark, "_git_revision", return_value="revision"), + ): + self.assertEqual(tempfile.tempdir, previous) + document = benchmark.result_document(args, None, None) + + self.assertIsNone(args.scratch_dir) + self.assertEqual( + document["controls"]["scratch_directory"], + str(Path(tempfile.gettempdir()).resolve()), + ) + + def test_benchmark_scratch_directory_must_exist(self): + with tempfile.TemporaryDirectory() as temporary: + missing = Path(temporary) / "missing" + args = nvx.parse_args(["benchmark", "--scratch-dir", str(missing)]) + previous = tempfile.tempdir + + with self.assertRaisesRegex(ValueError, "scratch directory does not exist"): + benchmark.run(args) + + self.assertEqual(tempfile.tempdir, previous) + + def test_network_snapshot_restore_selects_portable_profile(self): + command = benchmark.snapshot_restore_command( + Path("openvmm"), + "mshv", + Path("snapshot"), + processors=4, + network_profile="portable", + ) + + self.assertEqual( + command[2:8], + ["--machine", "microvm", "--processors", "4", "--hypervisor", "mshv"], + ) + + self.assertEqual( + command[-2:], + [ + "--network-profile", + "portable", + ], + ) + + def test_snapshot_restore_command_separates_capacity_from_online_target(self): + command = benchmark.snapshot_restore_command( + Path("openvmm"), + "kvm", + Path("snapshot"), + processors=8, + restore_processors=4, + ) + + self.assertEqual(command[command.index("--processors") + 1], "8") + self.assertEqual(command[command.index("--restore-processors") + 1], "4") + # Restore packet v4 gives every restore fresh entropy. + self.assertNotIn("--restore-entropy", command) + + def test_snapshot_restore_command_sets_memory_target_separately(self): + command = benchmark.snapshot_restore_command( + Path("openvmm"), + "whp", + Path("snapshot"), + restore_memory_mib=2048, + ) + + self.assertNotIn("--memory", command) + self.assertEqual(command[command.index("--restore-memory") + 1], "2048M") + + def test_memory_online_latency_parser_validates_added_bytes(self): + with tempfile.TemporaryDirectory() as temporary: + log = Path(temporary) / "restore.log" + log.write_bytes( + b"NVX-MEMORY-ONLINE-OK: added_bytes=536870912 " + b"memtotal_kib=1000000 elapsed_us=12345\n" + ) + self.assertEqual( + benchmark._memory_online_elapsed_ms(log, 536870912), + 12.345, + ) + with self.assertRaisesRegex(RuntimeError, "expected 1"): + benchmark._memory_online_elapsed_ms(log, 1) + + def test_benchmark_snapshot_restore_propagates_processors(self): + args = argparse.Namespace( + processors=4, + network_profile=None, + warmups=1, + runs=1, + timeout=1.0, + teardown_mode="guest-exit", + ) + result = cast(benchmark.BenchmarkResult, {"p50_ms": 10.0}) + with patch.object(benchmark, "benchmark", return_value=result) as run: + self.assertIs( + benchmark.benchmark_snapshot_restore( + args, + Path("openvmm"), + "kvm", + ["unused-cold-command"], + snapshot_path=Path("snapshot"), + ), + result, + ) + + command = run.call_args.args[0] + self.assertEqual( + command[2:8], + ["--machine", "microvm", "--processors", "4", "--hypervisor", "kvm"], + ) + self.assertTrue(run.call_args.kwargs["marker_must_be_line"]) + self.assertTrue(run.call_args.kwargs["guest_exit_prequeued"]) + + def test_parses_dd_rates_and_network_gateway(self): + output = """ +67108864 bytes copied, 0.25 s, 268.4 MB/s +67108864 bytes copied, 0.04 s, 1.6 GB/s +""" + + self.assertEqual(benchmark.parse_dd_rate(output, 1), 268.4) + self.assertEqual(benchmark.parse_dd_rate(output, 2), 1600.0) + self.assertEqual(benchmark.network_gateway("10.0.0.2/24"), "10.0.0.1") + + def test_benchmark_command_lines_carry_no_clock_tuning(self): + # Under time ABI v1 the hypervisor identity gives Linux its TSC and + # LAPIC rates and trust, so the lifecycle line tunes no clock and the + # cold-start clocksource variant selects tsc on every backend. + for token in ( + "tsc=", + "no_timer_check", + "clocksource=", + "tsc_early_khz=", + "lapic_timer_hz=", + ): + self.assertNotIn(token, benchmark.BASE_TUNING) + args = argparse.Namespace( + runs=1, + warmups=0, + memory_mib=128, + processors=1, + timeout=10.0, + teardown_mode="guest-exit", + ) + for backend in ("kvm", "mshv", "whp"): + commands: list[list[str]] = [] + + def fake_benchmark( + command: list[str], + commands: list[list[str]] = commands, + **_kwargs: object, + ): + commands.append(command) + return {"samples_ms": [100.0]} + + with ( + self.subTest(backend=backend), + patch.object(benchmark, "benchmark", side_effect=fake_benchmark), + contextlib.redirect_stdout(io.StringIO()) as stdout, + ): + benchmark.benchmark_cold_start_workload( + args, Path("openvmm"), Path("vmlinux"), Path("initrd"), backend + ) + cmdlines = [command[command.index("--cmdline") + 1] for command in commands] + self.assertIn("quiet loglevel=0 clocksource=tsc", cmdlines) + self.assertFalse(any("kvm-clock" in cmdline for cmdline in cmdlines)) + self.assertIn(" clocksource=tsc :", stdout.getvalue()) + + def test_smp_probe_uses_explicit_topology_and_worker_rendezvous(self): + script = benchmark.smp_probe_script(4) + + self.assertIn("getconf _NPROCESSORS_ONLN", script) + self.assertIn("physical_package_id", script) + self.assertIn("die_id", script) + self.assertIn("thread_siblings_list", script) + self.assertIn('taskset -c "$cpu"', script) + self.assertIn("for pid in $worker_pids", script) + self.assertIn("loc_before", script) + self.assertIn("SMP-LAPIC-FAIL", script) + self.assertIn("SMP-IPI-FAIL", script) + self.assertIn("read_loc_counter", script) + self.assertNotIn("while :; do", script) + self.assertIn("timer_attempts=10000", script) + self.assertIn('[ "$after" -gt "$before" ]', script) + self.assertIn('[ "$after" -ge "$current" ]', script) + self.assertNotIn("sleep 0.1", script) + self.assertIn("apic_ids=0,1,2,3 bsp=0 workers=$workers", script) + self.assertIn("NVX-SMP-PROBE-OK", script) + self.assertTrue(script.endswith("nvx-exit 0\n")) + network_script = benchmark.smp_probe_script( + 4, + network_gateway="10.0.0.1", + ioapic_irq=10, + ) + self.assertIn('ping -c 2 -W 1 "10.0.0.1"', network_script) + self.assertIn("SMP-IOAPIC-FAIL", network_script) + self.assertIn("SMP-IOAPIC-OK", network_script) + with self.assertRaises(ValueError): + benchmark.smp_probe_script(3) + with self.assertRaises(ValueError): + benchmark.smp_probe_script(4, network_gateway="10.0.0.1") + + def test_snapshot_request_triggers_waiting_capture_controller(self): + script = benchmark.snapshot_request_script( + 4, + teardown_mode="guest-exit", + ) + self.assertEqual(script, "nvx-snapshot\n") + + no_controller = benchmark.snapshot_request_script( + None, + teardown_mode="guest-exit", + ) + self.assertEqual( + no_controller, + "echo NVX-SNAPSHOT-DISPATCHED\n" + "nvx-snapshot\n" + "echo OPENVMM-SNAPSHOT-RESTORE-OK\n" + "nvx-exit 0\n", + ) + self.assertNotIn( + "nvx-exit", + benchmark.snapshot_request_script(None, teardown_mode="host-terminate"), + ) + + def test_prepare_snapshot_capture_stages_waiting_controller(self): + script = benchmark.prepare_snapshot_capture_script( + 4, + teardown_mode="guest-exit", + network_gateway="10.0.0.1", + ioapic_irq=10, + ) + + self.assertTrue( + script.startswith( + f"cat >{benchmark.SMP_PROBE_PATH} <<'NVX_SMP_PROBE_SCRIPT'\n" + ) + ) + self.assertIn('ping -c 2 -W 1 "10.0.0.1"', script) + self.assertIn("NVX-SMP-PROBE-OK", script) + self.assertIn( + f"cat >{benchmark.SNAPSHOT_CAPTURE_PATH} <<'NVX_SNAPSHOT_CAPTURE_SCRIPT'\n", + script, + ) + self.assertIn("IFS= read -r trigger\n", script) + self.assertIn("echo NVX-SNAPSHOT-DISPATCHED\n", script) + self.assertIn( + "/sbin/nvx-snapshot\necho OPENVMM-SNAPSHOT-RESTORE-OK\nnvx-exit 0\n", + script, + ) + host_terminated = benchmark.prepare_snapshot_capture_script( + 4, teardown_mode="host-terminate" + ) + self.assertNotIn("nvx-exit 0", host_terminated) + self.assertTrue( + script.endswith( + "NVX_SNAPSHOT_CAPTURE_SCRIPT\n" + f"chmod +x {benchmark.SMP_PROBE_PATH} " + f"{benchmark.SNAPSHOT_CAPTURE_PATH}\n" + f"{benchmark.SNAPSHOT_CAPTURE_PATH}\n" + ) + ) + + def test_capture_does_not_wait_for_a_clocksource(self): + # The time ABI registers the tsc clocksource at device_initcall on + # every backend, so no capture waits for Linux to leave tsc-early. + script = benchmark.prepare_snapshot_capture_script( + 1, teardown_mode="guest-exit" + ) + probe = script.split("<<'NVX_SMP_PROBE_SCRIPT'\n", 1)[1] + probe = probe.split("NVX_SMP_PROBE_SCRIPT\n", 1)[0] + self.assertEqual(probe, benchmark.smp_probe_script(1, exit_guest=False)) + for text in ("clocksource", "tsc-early", "clock_tries"): + self.assertNotIn(text, script) + self.assertFalse(hasattr(benchmark, "stable_clocksource_wait_script")) + + def test_output_marker_must_be_a_complete_line(self): + marker = benchmark.RESTORE_MARKER + self.assertFalse( + benchmark.contains_output_line( + b"/ # echo OPENVMM-SNAPSHOT-RESTORE-OK\r\n", + marker, + ) + ) + self.assertTrue( + benchmark.contains_output_line( + b"/ # echo OPENVMM-SNAPSHOT-RESTORE-OK\r\n" + b"OPENVMM-SNAPSHOT-RESTORE-OK\r\n", + marker, + ) + ) + + def test_device_restore_marker_and_trace_parsers(self): + marker = benchmark.parse_device_restore_marker( + "NVX-VIRTIO-RESTORE-PROBE phase=io-success device=net mode=deferred\r" + ) + self.assertEqual( + marker, + {"phase": "io-success", "device": "net", "mode": "deferred"}, + ) + self.assertIsNone(benchmark.parse_device_restore_marker("unrelated")) + + event = benchmark.parse_virtio_restore_event( + "DEBUG virtio_restore: virtio restore lifecycle " + 'event: "queue_start", device_type: 0x1a, trigger: "driver-ok", ' + "queue_index: 0, restored_progress: true, success: true" + ) + self.assertEqual( + event, + { + "event": "queue_start", + "device_type": 0x1A, + "trigger": "driver-ok", + "queue_index": 0, + "restored_progress": True, + "success": True, + }, + ) + self.assertIsNone(benchmark.parse_virtio_restore_event("unrelated")) + + def test_device_restore_commands_use_fixed_attachments(self): + common = ( + Path("openvmm"), + "whp", + Path("vmlinux"), + Path("initrd"), + 256, + ) + console_boot, console_restore = benchmark.device_restore_commands( + *common, "console", "deferred" + ) + self.assertIn( + "nvx_virtio_restore_probe=console,deferred", + console_boot[console_boot.index("--cmdline") + 1], + ) + self.assertEqual(console_boot[-2:], ["--virtio-console", "console"]) + self.assertEqual(console_restore[-2:], ["--virtio-console", "console"]) + + net_boot, net_restore = benchmark.device_restore_commands( + *common, "net", "active" + ) + self.assertIn("--network-profile", net_boot) + self.assertIn("--network-profile", net_restore) + + fs_boot, fs_restore = benchmark.device_restore_commands( + *common, + "virtiofs", + "deferred", + host_directory=Path("host-share"), + ) + self.assertIn("--mount", fs_boot) + self.assertIn("--mount", fs_restore) + + def test_device_restore_sample_rejects_premature_activation(self): + markers = [ + { + "phase": "restore-ready", + "device": "net", + "mode": "deferred", + "observer_elapsed_ms": "8.0", + }, + { + "phase": "trigger", + "device": "net", + "mode": "deferred", + "observer_elapsed_ms": "10.0", + }, + ] + events: list[dict[str, object]] = [ + { + "event": "restore_staged", + "device_type": 1, + "trigger": "inactive-start", + "queue_index": -1, + "restored_progress": False, + "success": True, + "observer_elapsed_ms": 5.0, + }, + { + "event": "private_state_apply", + "device_type": 1, + "trigger": "kick", + "queue_index": -1, + "restored_progress": False, + "success": True, + "observer_elapsed_ms": 9.0, + }, + { + "event": "kick_staged", + "device_type": 1, + "trigger": "kick", + "queue_index": 0, + "restored_progress": False, + "success": True, + "observer_elapsed_ms": 10.5, + }, + ] + for queue_index in range(2): + events.append( + { + "event": "queue_start", + "device_type": 1, + "trigger": "driver-ok", + "queue_index": queue_index, + "restored_progress": False, + "success": True, + "observer_elapsed_ms": 11.0, + } + ) + events.append( + { + "event": "kick_dispatch", + "device_type": 1, + "trigger": "driver-ok", + "queue_index": 0, + "restored_progress": False, + "success": True, + "observer_elapsed_ms": 12.0, + } + ) + sample = cast( + benchmark.DeviceRestoreSample, + { + "process_launch_to_ready_ms": 8.0, + "trigger_to_first_successful_io_ms": 2.0, + "peak_rss_bytes": 1, + "guest_markers": markers, + "events": events, + "queue_start_count": 0, + "staged_kick_dispatch_count": 0, + "stale_premature_callback_count": 0, + "log": "sample.log", + }, + ) + with self.assertRaisesRegex(RuntimeError, "premature"): + benchmark.validate_device_restore_sample(sample, "net", "deferred") + events[1]["observer_elapsed_ms"] = 10.25 + benchmark.validate_device_restore_sample(sample, "net", "deferred") + self.assertEqual(sample["queue_start_count"], 2) + self.assertEqual(sample["staged_kick_dispatch_count"], 1) + self.assertEqual(sample["stale_premature_callback_count"], 0) + + def test_device_restore_profile_writes_standalone_result(self): + args = argparse.Namespace( + processors=1, + restore_devices=["console"], + restore_modes=["active"], + net=None, + memory_mib=128, + timeout=1.0, + warmups=1, + runs=2, + platform="windows-whp-baremetal", + ) + + def sample() -> benchmark.DeviceRestoreSample: + return { + "process_launch_to_ready_ms": 10.0, + "trigger_to_first_successful_io_ms": 2.0, + "peak_rss_bytes": 1024, + "guest_markers": [ + { + "phase": "restore-ready", + "device": "console", + "mode": "active", + "observer_elapsed_ms": "10.0", + }, + { + "phase": "trigger", + "device": "console", + "mode": "active", + "observer_elapsed_ms": "11.0", + }, + ], + "events": [ + { + "event": "restore_staged", + "device_type": 3, + "trigger": "active-start", + "queue_index": -1, + "restored_progress": False, + "success": True, + }, + { + "event": "private_state_apply", + "device_type": 3, + "trigger": "active-start", + "queue_index": -1, + "restored_progress": False, + "success": True, + }, + { + "event": "kick_dispatch", + "device_type": 3, + "trigger": "kick", + "queue_index": 1, + "restored_progress": False, + "success": True, + }, + *[ + { + "event": "queue_start", + "device_type": 3, + "trigger": "active-start", + "queue_index": queue_index, + "restored_progress": True, + "success": True, + } + for queue_index in range(2) + ], + ], + "queue_start_count": 0, + "staged_kick_dispatch_count": 0, + "stale_premature_callback_count": 0, + "log": "", + } + + def restore_sample( + *_args: object, **_kwargs: object + ) -> benchmark.DeviceRestoreSample: + return sample() + + with tempfile.TemporaryDirectory() as temporary: + output_dir = Path(temporary) + with ( + patch.object( + benchmark, + "capture_device_restore_snapshot", + return_value=[ + { + "phase": "capture-ready", + "device": "console", + "mode": "active", + } + ], + ) as capture, + patch.object( + benchmark, + "run_device_restore_sample", + side_effect=restore_sample, + ) as restore, + ): + path = benchmark.benchmark_device_restore_profile( + args, + Path("openvmm"), + Path("vmlinux"), + Path("initrd"), + "whp", + output_dir, + ) + + self.assertEqual(capture.call_count, 1) + self.assertEqual(restore.call_count, 3) + document = json.loads(path.read_text(encoding="utf-8")) + self.assertTrue(document["non_canonical"]) + self.assertTrue(document["assertions"]["zero_stale_premature_callbacks"]) + scenario = document["scenarios"]["console-active"] + self.assertEqual( + scenario["process_launch_to_ready_ms"]["samples"], [10.0, 10.0] + ) + self.assertEqual(scenario["process_launch_to_ready_ms"]["p50"], 10.0) + self.assertEqual(scenario["queue_start_counts"], [2, 2]) + + def test_device_restore_profile_writes_lf_only_virtiofs_seed(self): + args = argparse.Namespace( + processors=1, + restore_devices=["virtiofs"], + restore_modes=["active"], + net=None, + memory_mib=128, + timeout=1.0, + warmups=1, + runs=1, + platform="windows-whp-baremetal", + ) + observed_seeds: list[bytes] = [] + host_directories: list[Path] = [] + + def commands(*_args: object, **kwargs: object) -> tuple[list[str], list[str]]: + host_directory = cast(Path, kwargs["host_directory"]) + host_directories.append(host_directory) + observed_seeds.append( + (host_directory / ".nvx-virtio-restore-host-seed").read_bytes() + ) + return ["boot"], ["restore"] + + sample = cast( + benchmark.DeviceRestoreSample, + { + "process_launch_to_ready_ms": 1.0, + "trigger_to_first_successful_io_ms": 1.0, + "peak_rss_bytes": 1, + "guest_markers": [], + "events": [], + "queue_start_count": 0, + "staged_kick_dispatch_count": 0, + "stale_premature_callback_count": 0, + "log": "sample.log", + }, + ) + + def run_sample( + *_args: object, **_kwargs: object + ) -> benchmark.DeviceRestoreSample: + (host_directories[-1] / ".nvx-virtio-restore-guest-result").write_bytes( + benchmark.VIRTFS_GUEST_TO_HOST + ) + return sample + + with tempfile.TemporaryDirectory() as temporary: + with ( + patch.object( + benchmark, "device_restore_commands", side_effect=commands + ), + patch.object( + benchmark, + "capture_device_restore_snapshot", + return_value=[], + ), + patch.object( + benchmark, + "run_device_restore_sample", + side_effect=run_sample, + ), + patch.object( + benchmark, + "summarize_device_restore_samples", + return_value={"stale_premature_callback_count": 0}, + ), + ): + benchmark.benchmark_device_restore_profile( + args, + Path("openvmm"), + Path("vmlinux"), + Path("initrd"), + "whp", + Path(temporary), + ) + + self.assertEqual(observed_seeds, [benchmark.VIRTFS_HOST_TO_GUEST]) + + def test_device_restore_profile_rejects_reused_output(self): + args = argparse.Namespace( + suite="device-restore-profile", + output=None, + output_dir=None, + platform="linux-kvm-baremetal", + ) + with tempfile.TemporaryDirectory() as temporary: + output_dir = Path(temporary) + (output_dir / "stale.log").write_text("stale", encoding="ascii") + args.output_dir = output_dir + with self.assertRaisesRegex(FileExistsError, "empty output directory"): + benchmark.run_workload_benchmarks( + args, + Path("openvmm"), + Path("vmlinux"), + Path("initrd"), + "kvm", + ) + + def test_benchmark_cpu_set_reserves_host_worker_capacity(self): + benchmark.validate_benchmark_cpu_set(set(range(10)), 8) + with self.assertRaisesRegex(ValueError, "requires at least 10"): + benchmark.validate_benchmark_cpu_set(set(range(9)), 8) + benchmark.validate_benchmark_cpu_set({0, 2, 4}, 1) + benchmark.validate_benchmark_cpu_set(set(range(8)), 8, 0) + with self.assertRaisesRegex(ValueError, "requires at least 8"): + benchmark.validate_benchmark_cpu_set(set(range(7)), 8, 0) + + def test_snapshot_capture_runs_smp_probe_for_selected_count(self): + args = argparse.Namespace( + warmups=0, + runs=1, + timeout=1.0, + processors=8, + teardown_mode="guest-exit", + ) + with patch.object( + benchmark, + "capture_snapshot", + return_value=(1.0, 1.0, 1.0, 1024), + ) as capture: + benchmark.benchmark_snapshot_capture(args, ["openvmm"]) + + self.assertNotIn("backend", capture.call_args.kwargs) + self.assertEqual(capture.call_args.kwargs["processors"], 8) + self.assertEqual(capture.call_args.kwargs["teardown_mode"], "guest-exit") + + def test_network_workload_runs_unmeasured_smp_interrupt_preflight(self): + args = argparse.Namespace( + net=None, + network_memory_mib=128, + processors=4, + warmups=0, + runs=1, + timeout=1.0, + teardown_mode="guest-exit", + ) + result: benchmark.BenchmarkResult = { + "samples_ms": [1.0], + "p50_ms": 1.0, + "p95_ms": 1.0, + "min_ms": 1.0, + "max_ms": 1.0, + "wall_samples_ms": [1.0], + "wall_p50_ms": 1.0, + "wall_p95_ms": 1.0, + "wall_min_ms": 1.0, + "wall_max_ms": 1.0, + "peak_rss_samples_bytes": [1], + "peak_rss_p50_bytes": 1, + "peak_rss_min_bytes": 1, + "peak_rss_max_bytes": 1, + "peak_rss_remeasured_count": 0, + "teardown_samples_ms": [1.0], + "teardown_completed_samples_ms": [1.0], + "teardown_timeout_count": 0, + "teardown_timeout_seconds": 5.0, + "teardown_p50_ms": 1.0, + "teardown_p95_ms": 1.0, + "teardown_min_ms": 1.0, + "teardown_max_ms": 1.0, + } + with ( + patch.object(benchmark, "run_guest_script") as probe, + patch.object(benchmark, "benchmark", return_value=result), + patch.object(benchmark, "capture_automatic_snapshot"), + ): + benchmark.benchmark_network_snapshot_workload( + args, + Path("openvmm"), + Path("vmlinux"), + Path("initrd"), + "kvm", + ) + + self.assertEqual(probe.call_count, 1) + self.assertIn("--network-profile", probe.call_args.args[0]) + self.assertIn("SMP-IOAPIC-OK", probe.call_args.args[1]) + self.assertEqual(probe.call_args.args[2], benchmark.SMP_PROBE_COMPLETION_MARKER) + + def test_shell_snapshot_restore_suite_only_measures_restore(self): + args = argparse.Namespace( + processors=8, + shell_memories=[512], + warmups=1, + runs=5, + timeout=40.0, + teardown_mode="guest-exit", + network_profile=None, + snapshot_profile=True, + ) + result = cast( + benchmark.BenchmarkResult, + { + "samples_ms": [10.0, 12.0], + "profile": { + "schema_version": 1, + "raw_samples": [], + "phases": { + "startup.partition_build": { + "exclusive": True, + "source": "openvmm", + "samples_ms": [1.0], + "p50_ms": 1.0, + "p95_ms": 1.0, + "min_ms": 1.0, + "max_ms": 1.0, + } + }, + }, + }, + ) + with ( + patch.object(benchmark, "capture_snapshot") as capture, + patch.object(benchmark, "benchmark", return_value=result) as run, + patch("sys.stdout", new_callable=io.StringIO) as output, + ): + benchmark.benchmark_shell_snapshot_restore_workload( + args, + Path("openvmm"), + Path("vmlinux"), + Path("initrd"), + "whp", + ) + + capture.assert_called_once() + capture_command = capture.call_args.args[0] + self.assertNotIn("shellsnap", capture_command) + self.assertEqual(capture.call_args.kwargs["processors"], 8) + self.assertEqual(capture.call_args.kwargs["teardown_mode"], "guest-exit") + run.assert_called_once() + restore_command = run.call_args.args[0] + self.assertIn("--restore-snapshot", restore_command) + self.assertEqual( + restore_command[restore_command.index("--processors") + 1], "8" + ) + self.assertIn("== 512 MiB ==", output.getvalue()) + self.assertIn("snapshot restore", output.getvalue()) + self.assertNotIn("cold boot", output.getvalue()) + self.assertIn('marker : "OPENVMM-SNAPSHOT-RESTORE-OK"', output.getvalue()) + self.assertTrue(run.call_args.kwargs["snapshot_profile"]) + self.assertEqual(run.call_args.kwargs["marker"], benchmark.RESTORE_MARKER) + self.assertTrue(run.call_args.kwargs["marker_must_be_line"]) + self.assertTrue(run.call_args.kwargs["guest_exit_prequeued"]) + self.assertIn( + "shell-snapshot-restore/whp/8vcpu/512-mib lifecycle phases:", + output.getvalue(), + ) + + def test_restore_vcpu_suite_reuses_one_capacity_snapshot_for_matrix(self): + args = argparse.Namespace( + processors=8, + memory_mib=512, + warmups=1, + runs=5, + timeout=40.0, + teardown_mode="guest-exit", + snapshot_profile=True, + ) + result = cast( + benchmark.BenchmarkResult, + { + "samples_ms": [10.0, 12.0], + "peak_rss_samples_bytes": [1024, 2048], + "profile": { + "schema_version": 1, + "raw_samples": [], + "phases": { + "restore.saved_state_restore": { + "exclusive": True, + "source": "openvmm", + "samples_ms": [2.0], + "p50_ms": 2.0, + "p95_ms": 2.0, + "min_ms": 2.0, + "max_ms": 2.0, + } + }, + }, + }, + ) + with ( + patch.object(benchmark, "capture_automatic_snapshot") as capture, + patch.object(benchmark, "benchmark", return_value=result) as run, + patch("sys.stdout", new_callable=io.StringIO) as output, + ): + benchmark.benchmark_snapshot_restore_vcpu_workload( + args, + Path("openvmm"), + Path("vmlinux"), + Path("initrd"), + "mshv", + ) + + capture.assert_called_once() + self.assertEqual(capture.call_args.kwargs["timeout"], args.timeout) + capture_command = capture.call_args.args[0] + self.assertEqual( + capture_command[capture_command.index("--processors") + 1], "8" + ) + self.assertTrue(any("maxcpus=1" in argument for argument in capture_command)) + self.assertEqual(run.call_count, 4) + targets: list[int] = [] + for invocation in run.call_args_list: + command = invocation.args[0] + self.assertEqual(invocation.kwargs["timeout"], args.timeout) + self.assertTrue(invocation.kwargs["snapshot_profile"]) + self.assertEqual(command[command.index("--processors") + 1], "8") + targets.append(int(command[command.index("--restore-processors") + 1])) + self.assertEqual(targets, [1, 2, 4, 8]) + self.assertIn("restore-online 8 vCPU", output.getvalue()) + self.assertIn( + "snapshot-restore-vcpu/mshv/capacity-8/online-1 lifecycle phases:", + output.getvalue(), + ) + + args.processors = 4 + with self.assertRaisesRegex(ValueError, "requires --processors 8"): + benchmark.benchmark_snapshot_restore_vcpu_workload( + args, + Path("openvmm"), + Path("vmlinux"), + Path("initrd"), + "kvm", + ) + + def test_benchmark_preserves_exact_process_wall_samples(self): + samples = [ + (10.0, 1024, 2.0, 13.5), + (12.0, 2048, 3.0, 16.5), + ] + with patch.object(benchmark, "measure_once", side_effect=samples): + result = benchmark.benchmark(["openvmm"], warmups=0, runs=2, timeout=1) + + self.assertEqual(result["samples_ms"], [10.0, 12.0]) + self.assertEqual(result["p95_ms"], 12.0) + self.assertEqual(result["wall_samples_ms"], [13.5, 16.5]) + self.assertEqual(result["wall_p50_ms"], 15.0) + self.assertEqual(result["wall_p95_ms"], 16.5) + self.assertEqual( + benchmark.SNAPSHOT_FILENAMES, + ("manifest.bin", "state.bin", "memory.bin"), + ) + + def test_benchmark_remeasures_attempt_without_marker_rss(self): + attempts = iter( + ( + (5.0, None, 1.0, 6.0), + (10.0, None, 7.0, 17.0), + (11.0, 2048, 7.0, 18.0), + (12.0, 4096, 8.0, 20.0), + ) + ) + + def measure_once( + *_args: object, + profile_sink: list[dict[str, object]] | None = None, + **_kwargs: object, + ) -> tuple[float, int | None, float | None, float]: + sample = next(attempts) + if profile_sink is not None: + profile_sink.append({"elapsed_ms": sample[0]}) + return sample + + before_each = MagicMock() + with ( + patch.object(benchmark, "measure_once", side_effect=measure_once), + patch.object( + benchmark, "summarize_lifecycle_profiles", return_value={} + ) as summarize, + patch("sys.stdout", new_callable=io.StringIO) as output, + ): + result = benchmark.benchmark( + ["openvmm"], + warmups=1, + runs=2, + timeout=1, + marker=benchmark.RESTORE_MARKER, + guest_exit_prequeued=True, + snapshot_profile=True, + before_each=before_each, + ) + + self.assertEqual(result["samples_ms"], [11.0, 12.0]) + self.assertEqual(result["peak_rss_samples_bytes"], [2048, 4096]) + self.assertEqual(result["peak_rss_min_bytes"], 2048) + self.assertEqual(result["peak_rss_remeasured_count"], 1) + self.assertEqual(result["teardown_samples_ms"], [7.0, 8.0]) + self.assertEqual(before_each.call_count, 4) + summarize.assert_called_once_with([{"elapsed_ms": 11.0}, {"elapsed_ms": 12.0}]) + self.assertIn("warmup 1/1: 5.000 ms, peak RSS=unavailable", output.getvalue()) + self.assertIn( + "sample 1/2: discarded attempt 1/3 (10.000 ms)", output.getvalue() + ) + + def test_benchmark_rejects_rss_missing_from_every_attempt(self): + with ( + patch.object( + benchmark, "measure_once", return_value=(10.0, None, 7.0, 17.0) + ) as measure, + patch("sys.stdout", new_callable=io.StringIO), + self.assertRaisesRegex( + RuntimeError, + f"{benchmark.PEAK_RSS_SAMPLE_ATTEMPTS} consecutive attempts " + "for sample 1/2", + ), + ): + benchmark.benchmark(["openvmm"], warmups=0, runs=2, timeout=1) + + self.assertEqual(measure.call_count, benchmark.PEAK_RSS_SAMPLE_ATTEMPTS) + + def test_nearest_rank_percentile(self): + self.assertEqual(benchmark.nearest_rank_percentile(range(1, 22), 95), 20) + self.assertEqual(benchmark.nearest_rank_percentile([7.0], 95), 7.0) + with self.assertRaisesRegex(ValueError, "without samples"): + benchmark.nearest_rank_percentile([], 95) + + def test_snapshot_capture_excludes_warmup_and_retains_last_snapshot(self): + with tempfile.TemporaryDirectory() as temporary: + retained = Path(temporary) / "retained" + observed_paths: list[Path] = [] + + def capture( + _command: object, + snapshot_path: Path, + **_kwargs: object, + ) -> tuple[float, float, float, int]: + snapshot_path.mkdir() + observed_paths.append(snapshot_path) + sample = float(len(observed_paths)) + return sample, sample, sample / 10, len(observed_paths) * 1024 + + args = argparse.Namespace( + warmups=1, + runs=2, + timeout=1.0, + processors=1, + teardown_mode="guest-exit", + ) + with patch.object(benchmark, "capture_snapshot", side_effect=capture): + result = benchmark.benchmark_snapshot_capture( + args, + ["openvmm"], + retained_snapshot_path=retained, + ) + + self.assertEqual(result["samples_ms"], [2.0, 3.0]) + self.assertEqual(result["peak_rss_samples_bytes"], [2048, 3072]) + self.assertEqual(result["peak_rss_p50_bytes"], 2560) + self.assertEqual(result["peak_rss_max_bytes"], 3072) + self.assertEqual(observed_paths[-1], retained) + self.assertTrue(retained.is_dir()) + + def test_snapshot_restore_reuses_provided_snapshot(self): + with tempfile.TemporaryDirectory() as temporary: + snapshot = Path(temporary) / "snapshot" + snapshot.mkdir() + args = argparse.Namespace( + warmups=1, + runs=2, + timeout=3.0, + teardown_mode="guest-exit", + network_profile=None, + processors=1, + ) + expected = {"p50_ms": 1.0} + with ( + patch.object(benchmark, "capture_snapshot") as capture, + patch.object(benchmark, "benchmark", return_value=expected) as run, + ): + result = benchmark.benchmark_snapshot_restore( + args, + Path("openvmm"), + "whp", + ["openvmm", "--kernel", "vmlinux"], + snapshot_path=snapshot, + ) + + self.assertIs(result, expected) + capture.assert_not_called() + self.assertIn(str(snapshot), run.call_args.args[0]) + self.assertEqual(run.call_args.kwargs["marker"], benchmark.RESTORE_MARKER) + self.assertTrue(run.call_args.kwargs["marker_must_be_line"]) + self.assertEqual(run.call_args.kwargs["teardown_mode"], "guest-exit") + self.assertTrue(run.call_args.kwargs["guest_exit_prequeued"]) + + def test_native_e2e_measures_and_reuses_snapshot_capture(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + openvmm = root / "openvmm" + executable = openvmm / "target" / "release" / "openvmm" + build_dir = root / "build" + executable.parent.mkdir(parents=True) + build_dir.mkdir() + (openvmm / "Cargo.toml").touch() + executable.touch() + (build_dir / "vmlinux").touch() + (build_dir / "initramfs.cpio.gz").touch() + output = root / "e2e.json" + args = nvx.parse_args( + [ + "benchmark", + "--suite", + "e2e", + "--backend", + "kvm", + "--openvmm-dir", + str(openvmm), + "--nvx-dir", + str(root), + "--cpus", + "0-2", + "--warmups", + "1", + "--runs", + "1", + "--skip-build", + "--output", + str(output), + ] + ) + run_result: benchmark.BenchmarkResult = { + "samples_ms": [10.0], + "p50_ms": 10.0, + "p95_ms": 10.0, + "min_ms": 10.0, + "max_ms": 10.0, + "wall_samples_ms": [12.0], + "wall_p50_ms": 12.0, + "wall_p95_ms": 12.0, + "wall_min_ms": 12.0, + "wall_max_ms": 12.0, + "peak_rss_samples_bytes": [1024], + "peak_rss_p50_bytes": 1024, + "peak_rss_min_bytes": 1024, + "peak_rss_max_bytes": 1024, + "peak_rss_remeasured_count": 0, + "teardown_samples_ms": [2.0], + "teardown_completed_samples_ms": [2.0], + "teardown_timeout_count": 0, + "teardown_timeout_seconds": 5.0, + "teardown_p50_ms": 2.0, + "teardown_p95_ms": 2.0, + "teardown_min_ms": 2.0, + "teardown_max_ms": 2.0, + } + capture_result: benchmark.SnapshotCaptureResult = { + "samples_ms": [3.0], + "p50_ms": 3.0, + "p95_ms": 3.0, + "min_ms": 3.0, + "max_ms": 3.0, + "request_to_publication_samples_ms": [3.0], + "request_to_publication_p50_ms": 3.0, + "request_to_publication_p95_ms": 3.0, + "request_to_publication_min_ms": 3.0, + "request_to_publication_max_ms": 3.0, + "post_publication_exit_samples_ms": [1.0], + "post_publication_exit_p50_ms": 1.0, + "post_publication_exit_p95_ms": 1.0, + "post_publication_exit_min_ms": 1.0, + "post_publication_exit_max_ms": 1.0, + "peak_rss_samples_bytes": [2048], + "peak_rss_p50_bytes": 2048, + "peak_rss_min_bytes": 2048, + "peak_rss_max_bytes": 2048, + } + with ( + patch.object(benchmark, "benchmark", return_value=run_result), + patch.object( + benchmark, + "benchmark_snapshot_capture", + return_value=capture_result, + ) as capture, + patch.object( + benchmark, + "benchmark_snapshot_restore", + return_value=run_result, + ) as restore, + ): + self.assertEqual(benchmark.run_native_linux(args), 0) + + retained = capture.call_args.kwargs["retained_snapshot_path"] + self.assertIsInstance(retained, Path) + self.assertEqual(restore.call_args.kwargs["snapshot_path"], retained) + document = json.loads(output.read_text(encoding="utf-8")) + self.assertEqual( + document["snapshot_capture"]["kvm"]["peak_rss_p50_bytes"], + 2048, + ) + + def test_kvm_worker_e2e_restores_the_measured_snapshot(self): + args = argparse.Namespace( + _stage_dir="/tmp/stage", + cpus="0", + memory_mib=128, + processors=4, + net=None, + suite="e2e", + warmups=1, + runs=1, + timeout=1.0, + teardown_mode="guest-exit", + ) + run_result = cast(benchmark.BenchmarkResult, {"p50_ms": 10.0}) + capture_result = cast(benchmark.SnapshotCaptureResult, {"p50_ms": 3.0}) + with ( + patch.object(benchmark, "benchmark", return_value=run_result), + patch.object( + benchmark, + "benchmark_snapshot_capture", + return_value=capture_result, + ) as capture, + patch.object( + benchmark, + "benchmark_snapshot_restore", + return_value=run_result, + ) as restore, + patch.object(benchmark, "print_summary"), + patch.object(benchmark, "print_snapshot_summary"), + ): + self.assertEqual(benchmark.run_kvm_worker(args), 0) + + retained = capture.call_args.kwargs["retained_snapshot_path"] + self.assertIsInstance(retained, Path) + self.assertEqual(restore.call_args.kwargs["snapshot_path"], retained) + + def test_virtfs_honors_host_termination_mode(self): + guest_exit = benchmark._virtfs_script(1, "guest-exit") + host_terminate = benchmark._virtfs_script(1, "host-terminate") + roundtrip = benchmark._virtfs_roundtrip_script("host-terminate") + + self.assertIn("nvx-exit 0", guest_exit) + self.assertNotIn("nvx-exit 0", host_terminate) + self.assertNotIn("nvx-exit 0", roundtrip) + + def test_virtfs_exchange_uses_lf_only_bytes(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + + def emulate_guest( + *_args: object, **_kwargs: object + ) -> benchmark.GuestCommandResult: + (root / "guest-visible").write_bytes(benchmark.VIRTFS_GUEST_TO_HOST) + deadline = benchmark.time.monotonic() + 1 + while benchmark.time.monotonic() < deadline: + if ( + root / "host-visible" + ).read_bytes() == benchmark.VIRTFS_HOST_TO_GUEST: + break + benchmark.time.sleep(0.001) + else: + self.fail("guest did not observe the byte-exact host update") + return {"text": "", "wall_ms": 1.0, "peak_rss_bytes": 1} + + with patch.object(benchmark, "run_guest_script", side_effect=emulate_guest): + benchmark._run_virtfs_roundtrip( + ["openvmm"], + root, + 1, + 1, + timeout=1, + windows_cpus=None, + teardown_mode="guest-exit", + ) + + self.assertEqual( + (root / "host-visible").read_bytes(), + b"host-to-guest\n", + ) + + def test_virtfs_excludes_declared_warmup(self): + args = argparse.Namespace( + virtfs_memory_mib=128, + payload_mib=1, + processors=1, + warmups=1, + timeout=1.0, + teardown_mode="guest-exit", + ) + io_result: benchmark.GuestCommandResult = { + "text": "1 byte copied, 1 s, 1 MB/s\n1 byte copied, 1 s, 2 MB/s\n", + "wall_ms": 1.0, + "peak_rss_bytes": 1, + } + roundtrip_result: benchmark.GuestCommandResult = { + "text": "", + "wall_ms": 2.0, + "peak_rss_bytes": 1, + } + with ( + patch.object(benchmark, "run_guest_script", return_value=io_result) as io, + patch.object( + benchmark, "_run_virtfs_roundtrip", return_value=roundtrip_result + ) as roundtrip, + ): + benchmark.benchmark_virtfs_workload( + args, + Path("openvmm"), + Path("vmlinux"), + Path("initrd"), + "kvm", + runs=3, + ) + + self.assertEqual(io.call_count, 4) + self.assertEqual(roundtrip.call_count, 4) + + def test_device_io_retains_failure_and_resumes_completed_attempts(self): + args = argparse.Namespace( + processors=1, + warmups=1, + runs=1, + device_io_duration_seconds=1.0, + device_io_size_mib=64, + device_io_port=5201, + net="192.0.2.2/24", + timeout=1.0, + teardown_mode="guest-exit", + ) + + def guest_result(command: list[str], *_args: object, **_kwargs: object): + if "--net" in command: + payload = ( + 'NVX_DEVICE_IO_GUEST_RESULT={"device":"network",' + '"operation":"roundtrip","operations":100,' + '"bytes_per_operation":64,"elapsed_ns":1000000000}\n' + ) + else: + payload = ( + 'NVX_DEVICE_IO_GUEST_RESULT={"device":"file",' + '"operation":"write","operations":100,' + '"bytes_per_operation":4096,"elapsed_ns":1000000000}\n' + 'NVX_DEVICE_IO_GUEST_RESULT={"device":"file",' + '"operation":"read","operations":100,' + '"bytes_per_operation":4096,"elapsed_ns":1000000000}\n' + ) + return {"text": payload, "wall_ms": 1.0, "peak_rss_bytes": 1} + + with tempfile.TemporaryDirectory() as temporary: + output = Path(temporary) / "device-io.log" + with ( + patch.object(benchmark, "host_ipv4_address", return_value="192.0.2.1"), + patch.object(benchmark, "UdpEchoServer") as echo_server, + patch.object( + benchmark, + "run_guest_script", + side_effect=guest_result, + ) as run_guest, + ): + echo_server.return_value.__enter__.return_value = None + benchmark.benchmark_device_io_workload( + args, + Path("openvmm"), + Path("vmlinux"), + Path("initramfs"), + "mshv", + output_path=output, + ) + self.assertEqual(run_guest.call_count, 6) + network_commands = [ + call.args[0] + for call in run_guest.call_args_list + if "--net" in call.args[0] + ] + self.assertEqual(len(network_commands), 2) + self.assertIn("192.0.2.2/24", network_commands[0]) + self.assertIn("microvm", network_commands[0]) + self.assertNotIn("microvm-v2", network_commands[0]) + block_commands = [ + call.args[0] + for call in run_guest.call_args_list + if "--microvm-sandbox-block" in call.args[0] + ] + self.assertEqual(len(block_commands), 2) + self.assertNotIn("--virtio-blk", block_commands[0]) + block_argument = block_commands[0][ + block_commands[0].index("--microvm-sandbox-block") + 1 + ] + self.assertTrue(block_argument.startswith("scratch:file:")) + + benchmark.benchmark_device_io_workload( + args, + Path("openvmm"), + Path("vmlinux"), + Path("initramfs"), + "mshv", + output_path=output, + ) + self.assertEqual(run_guest.call_count, 6) + + records = benchmark._read_device_io_records(output) + self.assertEqual(len(records), 6) + retained = [record for record in records if not record["warmup"]] + self.assertEqual([record["sample_index"] for record in retained], [0, 0, 0]) + + def test_device_io_missing_guest_result_becomes_failure(self): + result: benchmark.GuestCommandResult = { + "text": ( + 'NVX_DEVICE_IO_GUEST_RESULT={"device":"file",' + '"operation":"read","operations":1,' + '"bytes_per_operation":4096,"elapsed_ns":1}\n' + ), + "wall_ms": 1.0, + "peak_rss_bytes": 1, + } + + record = benchmark._device_io_attempt_record("virtio-blk", 0, 0, result, None) + + self.assertEqual(record["status"], "failure") + self.assertIn("missing", str(record["error"])) + + def test_managed_tap_cleanup_uses_openvmm_pid(self): + query: subprocess.CompletedProcess[str] = subprocess.CompletedProcess( + ["ip"], 0, "", "" + ) + removed: subprocess.CompletedProcess[str] = subprocess.CompletedProcess( + ["sudo", "-n", "ip"], 0, "", "" + ) + with ( + patch.object(benchmark.sys, "platform", "linux"), + patch.object(benchmark.os, "geteuid", return_value=1000, create=True), + patch.object( + benchmark.subprocess, "run", side_effect=(query, removed) + ) as run, + ): + benchmark.cleanup_managed_tap(1234) + + self.assertEqual( + run.call_args_list[1].args[0], + [ + "sudo", + "-n", + "ip", + "tuntap", + "del", + "dev", + "ovm1234", + "mode", + "tap", + ], + ) + + def test_performance_suite_writes_four_canonical_logs(self): + with tempfile.TemporaryDirectory() as temporary: + output = Path(temporary) + (output / "snapshot.log").write_text("stale", encoding="ascii") + (output / "snapshot-hello.log").write_text("stale", encoding="ascii") + args = nvx.parse_args( + [ + "benchmark", + "--suite", + "performance", + "--backend", + "whp", + "--platform", + "windows-whp-baremetal", + "--processors", + "8", + "--output-dir", + str(output), + ] + ) + with ( + patch.object(benchmark, "benchmark_cold_start_workload") as cold, + patch.object(benchmark, "benchmark_virtfs_workload") as virtfs, + patch.object(benchmark, "benchmark_shell_snapshot_workload") as shell, + patch.object( + benchmark, "benchmark_network_snapshot_workload" + ) as network, + ): + result = benchmark.run_workload_benchmarks( + args, + Path("openvmm.exe"), + Path("vmlinux"), + Path("initramfs.cpio.gz"), + "whp", + ) + + self.assertEqual(result, 0) + self.assertEqual( + {path.name for path in output.iterdir()}, + { + "benchmark-metadata.json", + "cold-start.log", + "virtfs.log", + "shell-snapshot.log", + "network.log", + }, + ) + metadata = json.loads( + (output / "benchmark-metadata.json").read_text(encoding="utf-8") + ) + self.assertEqual(metadata["platform"], "windows-whp-baremetal") + self.assertEqual(metadata["backend"], "whp") + self.assertEqual(metadata["microvm_abi_version"], 2) + self.assertEqual(metadata["processors"], 8) + self.assertEqual(metadata["host_affinity_set"], args.cpus) + self.assertEqual(metadata["host_cpu_reserve"], args.host_cpu_reserve) + self.assertEqual( + metadata["scratch_directory"], + str(Path(tempfile.gettempdir()).resolve()), + ) + cold.assert_called_once() + self.assertEqual(virtfs.call_args.kwargs["runs"], 3) + shell.assert_called_once() + network.assert_called_once() + + def test_restore_only_suite_writes_only_its_canonical_log(self): + with tempfile.TemporaryDirectory() as temporary: + output = Path(temporary) + (output / "cold-start.log").write_text("stale", encoding="ascii") + (output / "shell-snapshot.log").write_text("stale", encoding="ascii") + (output / "acceptance.json").write_text("stale", encoding="ascii") + args = nvx.parse_args( + [ + "benchmark", + "--suite", + "shell-snapshot-restore", + "--backend", + "whp", + "--platform", + "windows-whp-virtual-machine", + "--processors", + "8", + "--shell-memories", + "512", + "--output-dir", + str(output), + ] + ) + with patch.object( + benchmark, "benchmark_shell_snapshot_restore_workload" + ) as restore: + result = benchmark.run_workload_benchmarks( + args, + Path("openvmm.exe"), + Path("vmlinux"), + Path("initramfs.cpio.gz"), + "whp", + ) + + self.assertEqual(result, 0) + self.assertEqual( + {path.name for path in output.iterdir()}, + {"benchmark-metadata.json", "shell-snapshot-restore.log"}, + ) + restore.assert_called_once() + + def test_mshv_performance_suite_includes_network(self): + with tempfile.TemporaryDirectory() as temporary: + output = Path(temporary) + args = nvx.parse_args( + [ + "benchmark", + "--suite", + "performance", + "--backend", + "mshv", + "--output-dir", + str(output), + ] + ) + with ( + patch.object(benchmark, "benchmark_cold_start_workload"), + patch.object(benchmark, "benchmark_virtfs_workload"), + patch.object(benchmark, "benchmark_shell_snapshot_workload"), + patch.object( + benchmark, "benchmark_network_snapshot_workload" + ) as network, + ): + benchmark.run_workload_benchmarks( + args, + Path("openvmm"), + Path("vmlinux"), + Path("initramfs.cpio.gz"), + "mshv", + ) + + self.assertEqual( + {path.name for path in output.iterdir()}, + { + "benchmark-metadata.json", + "cold-start.log", + "virtfs.log", + "shell-snapshot.log", + "network.log", + }, + ) + network.assert_called_once() + + def test_mshv_runs_explicit_network_snapshot_suite(self): + args = nvx.parse_args( + ["benchmark", "--suite", "network-snapshot", "--backend", "mshv"] + ) + + with patch.object(benchmark, "benchmark_network_snapshot_workload") as network: + benchmark.run_workload_benchmarks( + args, + Path("openvmm"), + Path("vmlinux"), + Path("initramfs.cpio.gz"), + "mshv", + ) + + network.assert_called_once() + + def test_performance_suite_defaults_to_data_runs_platform(self): + with tempfile.TemporaryDirectory() as temporary: + repository = Path(temporary) + backend = "whp" if os.name == "nt" else "kvm" + platform = "windows-whp" if os.name == "nt" else "linux-kvm" + args = nvx.parse_args( + [ + "benchmark", + "--suite", + "performance", + "--backend", + backend, + "--nvx-dir", + str(repository), + ] + ) + with ( + patch.object(benchmark, "benchmark_cold_start_workload"), + patch.object(benchmark, "benchmark_virtfs_workload"), + patch.object(benchmark, "benchmark_shell_snapshot_workload"), + patch.object(benchmark, "benchmark_network_snapshot_workload"), + ): + benchmark.run_workload_benchmarks( + args, + Path("openvmm.exe"), + Path("vmlinux"), + Path("initramfs.cpio.gz"), + backend, + ) + + output = repository / "data" / "runs" / f"{platform}-microvm-v2-1vcpu" + self.assertEqual( + {path.name for path in output.iterdir()}, + { + "benchmark-metadata.json", + "cold-start.log", + "virtfs.log", + "shell-snapshot.log", + "network.log", + }, + ) + + def test_device_io_defaults_to_dedicated_abi_v2_directory(self): + with tempfile.TemporaryDirectory() as temporary: + repository = Path(temporary).resolve() + backend = "whp" if os.name == "nt" else "kvm" + platform = "windows-whp" if os.name == "nt" else "linux-kvm" + args = nvx.parse_args( + [ + "benchmark", + "--suite", + "device-io", + "--backend", + backend, + "--nvx-dir", + str(repository), + ] + ) + with ( + patch.object( + benchmark, + "write_benchmark_metadata", + return_value=Path("benchmark-metadata.json"), + ) as metadata, + patch.object( + benchmark, "benchmark_device_io_workload", return_value=0 + ) as device_io, + ): + benchmark.run_workload_benchmarks( + args, + Path("openvmm.exe"), + Path("vmlinux"), + Path("initramfs.cpio.gz"), + backend, + ) + + output = ( + repository + / "data" + / "runs" + / platform + / "microvm-v2" + / "1vcpu" + / "device-io" + ) + self.assertEqual( + device_io.call_args.kwargs["output_path"], + output / "device-io.log", + ) + self.assertEqual(metadata.call_args.args[1], output) + + def test_package_command_forwards_parsed_options(self): + args = nvx.parse_args( + [ + "package", + "--version", + "2.0.0", + "--destination", + "dist/custom", + "--binary-only", + "--force", + ] + ) + + with patch.object(nvx, "package_release") as package_release: + args.handler(args) + + package_release.assert_called_once_with( + version="2.0.0", + destination=Path("dist/custom"), + include_source=False, + force=True, + ) + + def test_archive_release_command_forwards_paths(self): + args = nvx.parse_args( + [ + "archive-release", + "--source", + "dist/package", + "--destination", + "dist/package.zip", + ] + ) + + with patch.object(nvx, "create_release_archive") as create_archive: + args.handler(args) + + create_archive.assert_called_once_with( + Path("dist/package"), + Path("dist/package.zip"), + ) + + def test_download_command_selects_host_release(self): + args = nvx.parse_args(["download", "--repository", "example/nvx"]) + expected_platform = "windows-whp" if os.name == "nt" else "linux-kvm" + + with patch.object(nvx, "download_checkout_release") as download_release: + args.handler(args) + + download_release.assert_called_once_with("example/nvx", expected_platform) + + def test_download_command_defaults_to_integration_repository(self): + args = nvx.parse_args(["download"]) + expected_platform = "windows-whp" if os.name == "nt" else "linux-kvm" + + with patch.object(nvx, "download_checkout_release") as download_release: + args.handler(args) + + download_release.assert_called_once_with("microsoft/nvx", expected_platform) + + +AUTHORIZATION_VALUE = "Bearer placeholder-value" + + +def _git_history(*commits: tuple[str, str | None]) -> Callable[..., str]: + """Fakes git_output for HEAD's first-parent history and its VERSION files.""" + + versions = dict(commits) + + def git_output(*arguments: str) -> str: + if arguments[:2] == ("rev-list", "--first-parent") and arguments[3:] == ( + "HEAD", + ): + limit = int(arguments[2].removeprefix("--max-count=")) + return "\n".join(list(versions)[:limit]) + if arguments[0] != "show" or not arguments[1].endswith(":VERSION"): + raise AssertionError(f"unexpected git command: {arguments}") + version = versions[arguments[1].removesuffix(":VERSION")] + if version is None: + raise subprocess.CalledProcessError(128, ["git", *arguments]) + return version + + return git_output + + +def _missing_release() -> urllib.error.HTTPError: + return urllib.error.HTTPError( + "https://api.github.invalid/releases/tags/missing", + 404, + "Not Found", + http.client.HTTPMessage(), + io.BytesIO(b'{"message": "Not Found"}'), + ) + + +class ReleaseTests(unittest.TestCase): + def test_alpine_source_validation_rejects_malformed_manifest(self): + with tempfile.TemporaryDirectory() as temporary: + source_dir = Path(temporary) + alpine_dir = source_dir / "alpine" + alpine_dir.mkdir() + (alpine_dir / "manifest.json").write_text("{", encoding="utf-8") + + with ( + patch.object(BuildConstants, "SOURCE_DIR", source_dir), + self.assertRaisesRegex( + common.ScriptError, + "invalid collected Alpine source manifest", + ), + ): + release._validate_alpine_sources([]) + + def test_alpine_source_validation_checks_recorded_executables(self): + with tempfile.TemporaryDirectory() as temporary: + source_dir = Path(temporary) + alpine_dir = source_dir / "alpine" + recipe = alpine_dir / "recipes" / "c0ffee" / "main" / "foo" + recipe.mkdir(parents=True) + (recipe / "APKBUILD").write_bytes(b"pkgname=foo\n") + (recipe / "foo.post-install").write_bytes(b"#!/bin/sh\n") + installer = "recipes/c0ffee/main/foo/foo.post-install" + current = AlpineBuildConstants.SOURCE_MANIFEST_FORMAT + cases: dict[str, tuple[dict[str, object], str | None]] = { + "recorded executable": ( + {"format": current, "executables": [installer]}, + None, + ), + "older format": ( + {"format": 1, "executables": [installer]}, + "unsupported manifest format", + ), + "missing executables": ({"format": current}, "invalid executables"), + "unlisted executable": ( + {"format": current, "executables": [f"{installer}.missing"]}, + "not a checksummed recipe file", + ), + "executable outside recipes": ( + {"format": current, "executables": ["manifest.json"]}, + "not a checksummed recipe file", + ), + } + for name, (document, error) in cases.items(): + (alpine_dir / "manifest.json").write_text( + json.dumps({"packages": [], **document}), encoding="utf-8" + ) + common.write_sha256_sums(alpine_dir) + with ( + self.subTest(case=name), + patch.object(BuildConstants, "SOURCE_DIR", source_dir), + ): + if error is None: + self.assertEqual( + release._validate_alpine_sources([]), {installer} + ) + continue + with self.assertRaisesRegex(common.ScriptError, error): + release._validate_alpine_sources([]) + + def test_alpine_source_archive_applies_recorded_modes(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + source_dir = root / "sources" + recipe = source_dir / "alpine" / "recipes" / "c0ffee" / "main" / "foo" + recipe.mkdir(parents=True) + for name in ("APKBUILD", "foo.post-install"): + (recipe / name).write_bytes(b"#!/bin/sh\n") + package_manifest = root / "initramfs.cpio.gz.packages.json" + package_manifest.write_text("{}", encoding="utf-8") + # Report every mode as Windows does, whatever the host stores. + for path in (*source_dir.rglob("*"), package_manifest): + path.chmod(0o777 if path.is_dir() else 0o666) + output = root / "alpine-source.tar.gz" + + with ( + patch.object(BuildConstants, "SOURCE_DIR", source_dir), + contextlib.redirect_stdout(io.StringIO()), + ): + release._alpine_source_archive( + output, + "1.0.0", + [package_manifest], + frozenset({"recipes/c0ffee/main/foo/foo.post-install"}), + ) + + with tarfile.open(output, "r:gz") as package: + modes = {member.name: member.mode for member in package.getmembers()} + prefix = "nvx-alpine-source-1.0.0" + self.assertEqual( + modes, + { + f"{prefix}/sources": 0o755, + f"{prefix}/sources/recipes": 0o755, + f"{prefix}/sources/recipes/c0ffee": 0o755, + f"{prefix}/sources/recipes/c0ffee/main": 0o755, + f"{prefix}/sources/recipes/c0ffee/main/foo": 0o755, + f"{prefix}/sources/recipes/c0ffee/main/foo/APKBUILD": 0o644, + f"{prefix}/sources/recipes/c0ffee/main/foo/foo.post-install": ( + 0o755 + ), + f"{prefix}/manifests/initramfs.cpio.gz.packages.json": 0o644, + }, + ) + + def test_downloads_release_built_from_head(self): + published = { + "draft": False, + "prerelease": True, + "tag_name": "v1.2.3-dev.aaaaaaaaaaaa", + "assets": [ + { + "name": "nvx-invalid-linux-kvm.tar.gz", + "url": "https://api.example.invalid/invalid", + "size": True, + }, + { + "name": "nvx-1.2.3-windows-whp.zip", + "url": "https://api.example.invalid/windows", + "size": 200, + }, + { + "name": "nvx-1.2.3-linux-kvm.tar.gz", + "url": "https://api.example.invalid/linux", + "size": 300, + }, + ], + } + + with ( + patch.object( + release, + "git_output", + side_effect=_git_history(("a" * 40, "1.2.3"), ("b" * 40, "1.2.3")), + ) as git_output, + patch( + "nvx_tools.release.urllib.request.OpenerDirector.open", + return_value=io.BytesIO(json.dumps(published).encode("utf-8")), + ) as opener_open, + contextlib.redirect_stdout(io.StringIO()) as stdout, + ): + asset = release._checkout_release_asset( + "example/nvx", + "linux-kvm", + "token", + ) + + self.assertEqual( + git_output.call_args_list[0], + call( + "rev-list", + "--first-parent", + f"--max-count={release._RELEASE_SEARCH_COMMITS}", + "HEAD", + ), + ) + self.assertEqual(opener_open.call_count, 1) + self.assertEqual( + opener_open.call_args.args[0].full_url, + "https://api.github.com/repos/example/nvx/releases/tags/" + "v1.2.3-dev.aaaaaaaaaaaa", + ) + self.assertEqual( + asset, + release._ReleaseAsset( + "v1.2.3-dev.aaaaaaaaaaaa", + "nvx-1.2.3-linux-kvm.tar.gz", + "https://api.example.invalid/linux", + 300, + ), + ) + self.assertEqual(stdout.getvalue(), "") + + def test_skips_draft_release_built_from_head(self): + def response(commit: str, draft: bool) -> io.BytesIO: + tag = f"v1.2.3-dev.{commit[:12]}" + document = { + "draft": draft, + "tag_name": tag, + "assets": [ + { + "name": "nvx-1.2.3-linux-kvm.tar.gz", + "url": f"https://api.example.invalid/{tag}/linux-kvm.tar.gz", + "size": 300, + } + ], + } + return io.BytesIO(json.dumps(document).encode("utf-8")) + + with ( + patch.object( + release, + "git_output", + side_effect=_git_history(("a" * 40, "1.2.3"), ("b" * 40, "1.2.3")), + ), + patch( + "nvx_tools.release.urllib.request.OpenerDirector.open", + side_effect=[ + response("a" * 40, True), + response("b" * 40, False), + ], + ), + contextlib.redirect_stdout(io.StringIO()) as stdout, + ): + asset = release._checkout_release_asset( + "example/nvx", + "linux-kvm", + "token", + ) + + self.assertEqual(asset.tag, "v1.2.3-dev.bbbbbbbbbbbb") + self.assertEqual( + stdout.getvalue(), + ">> no linux-kvm release for HEAD; using v1.2.3-dev.bbbbbbbbbbbb, " + "1 first-parent commit earlier\n", + ) + + def test_walks_first_parent_history_to_the_nearest_release(self): + def published(version: str, commit: str, *packages: str) -> io.BytesIO: + tag = f"v{version}-dev.{commit[:12]}" + document = { + "tag_name": tag, + "assets": [ + { + "name": f"nvx-{version}-{package}", + "url": f"https://api.example.invalid/{tag}/{package}", + "size": 100, + } + for package in packages + ], + } + return io.BytesIO(json.dumps(document).encode("utf-8")) + + # HEAD has no release, b has no VERSION file, c's release has no KVM + # package, and d's release predates a version bump. + history = _git_history( + ("a" * 40, "1.2.3"), + ("b" * 40, None), + ("c" * 40, "1.2.3"), + ("d" * 40, "1.2.2"), + ("e" * 40, "1.2.2"), + ) + responses = [ + _missing_release(), + published("1.2.3", "c" * 40, "windows-whp.zip"), + published("1.2.2", "d" * 40, "windows-whp.zip", "linux-kvm.tar.gz"), + ] + + with ( + patch.object(release, "git_output", side_effect=history), + patch( + "nvx_tools.release.urllib.request.OpenerDirector.open", + side_effect=responses, + ) as opener_open, + contextlib.redirect_stdout(io.StringIO()) as stdout, + ): + asset = release._checkout_release_asset("example/nvx", "linux-kvm", None) + + tags = "https://api.github.com/repos/example/nvx/releases/tags" + self.assertEqual( + [request.args[0].full_url for request in opener_open.call_args_list], + [ + f"{tags}/v1.2.3-dev.aaaaaaaaaaaa", + f"{tags}/v1.2.3-dev.cccccccccccc", + f"{tags}/v1.2.2-dev.dddddddddddd", + ], + ) + self.assertEqual(asset.tag, "v1.2.2-dev.dddddddddddd") + self.assertEqual( + asset.url, + "https://api.example.invalid/v1.2.2-dev.dddddddddddd/linux-kvm.tar.gz", + ) + self.assertEqual( + stdout.getvalue(), + ">> no linux-kvm release for HEAD; using v1.2.2-dev.dddddddddddd, " + "3 first-parent commits earlier\n", + ) + + def test_reports_when_no_searched_commit_has_a_release(self): + with ( + patch.object( + release, + "git_output", + side_effect=_git_history(("a" * 40, "1.2.3"), ("b" * 40, "1.2.3")), + ), + patch( + "nvx_tools.release.urllib.request.OpenerDirector.open", + side_effect=[_missing_release(), _missing_release(), io.BytesIO(b"{}")], + ) as opener_open, + self.assertRaisesRegex( + release.ScriptError, + r"no example/nvx release with an NVX package for linux-kvm was built " + r"from HEAD or its first-parent ancestors \(2 commits searched\)", + ), + ): + release._checkout_release_asset("example/nvx", "linux-kvm", None) + + self.assertEqual( + opener_open.call_args.args[0].full_url, + "https://api.github.com/repos/example/nvx", + ) + + def test_download_requires_a_git_checkout(self): + failure = subprocess.CalledProcessError( + 128, + ["git", "rev-list"], + stderr="fatal: not a git repository (or any of the parent directories)\n", + ) + + with ( + patch.object(release, "git_output", side_effect=failure), + patch( + "nvx_tools.release.urllib.request.OpenerDirector.open" + ) as opener_open, + self.assertRaisesRegex( + release.ScriptError, + "download needs a Git checkout to find its release: " + "fatal: not a git repository", + ), + ): + release._checkout_release_asset("example/nvx", "linux-kvm", None) + + opener_open.assert_not_called() + + def test_version_lookup_reports_git_execution_failures(self): + commit = "a" * 40 + failures = ( + OSError("git unavailable"), + subprocess.TimeoutExpired(["git", "show"], 30), + ) + for failure in failures: + with ( + self.subTest(failure=type(failure).__name__), + patch.object(release, "git_output", side_effect=failure), + self.assertRaisesRegex( + release.ScriptError, + f"download failed to read VERSION at {commit}", + ) as raised, + ): + release._version_at(commit) + + self.assertIs(raised.exception.__cause__, failure) + + def test_first_parent_history_and_versions_come_from_git(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + environment = { + key: value + for key, value in os.environ.items() + if not key.startswith("GIT_CONFIG_") + } + environment.update( + GIT_AUTHOR_NAME="NVX Tests", + GIT_AUTHOR_EMAIL="nvx-tests@example.com", + GIT_COMMITTER_NAME="NVX Tests", + GIT_COMMITTER_EMAIL="nvx-tests@example.com", + ) + + def git(*args: str, data: bytes = b"") -> str: + result = subprocess.run( + ["git", "-C", str(root), *args], + input=data, + env=environment, + capture_output=True, + check=True, + ) + return result.stdout.decode("ascii").strip() + + def commit(version: str | None, *parents: str) -> str: + entries = b"" + if version is not None: + blob = git( + "hash-object", "-w", "--stdin", data=f"{version}\n".encode() + ) + entries = f"100644 blob {blob}\tVERSION\n".encode() + tree = git("mktree", data=entries) + parent_arguments = [ + argument for parent in parents for argument in ("-p", parent) + ] + return git( + "commit-tree", "--no-gpg-sign", tree, *parent_arguments, "-m", "x" + ) + + git("init", "-q") + first = commit("1.2.2") + merged = commit("9.9.9", first) + second = commit("1.2.3", first) + merge = commit("1.2.3", second, merged) + head = commit(None, merge) + git("update-ref", "HEAD", head) + + with patch.object(BuildConstants, "REPO_ROOT", root): + commits = release._first_parent_commits(10) + limited = release._first_parent_commits(2) + versions = [release._version_at(commit_id) for commit_id in commits] + + self.assertEqual(commits, [head, merge, second, first]) + self.assertEqual(limited, [head, merge]) + self.assertEqual(versions, [None, "1.2.3", "1.2.3", "1.2.2"]) + + def test_forbidden_release_query_reports_github_message_and_hint(self): + error = urllib.error.HTTPError( + "https://api.github.invalid/releases", + 403, + "Forbidden", + http.client.HTTPMessage(), + io.BytesIO( + json.dumps( + {"message": "Resource protected by organization SAML enforcement."} + ).encode("utf-8") + ), + ) + + with ( + patch.object( + release, + "git_output", + side_effect=_git_history(("a" * 40, "1.2.3")), + ), + patch( + "nvx_tools.release.urllib.request.OpenerDirector.open", + side_effect=error, + ), + self.assertRaises(release.ScriptError) as context, + ): + release._checkout_release_asset("example/nvx", "linux-kvm", "token") + + message = str(context.exception) + self.assertIn("HTTP 403", message) + self.assertIn("organization SAML enforcement", message) + self.assertIn("read access to the repository contents", message) + + def test_unauthenticated_release_query_reports_token_hint(self): + error = urllib.error.HTTPError( + "https://api.github.invalid/releases", + 404, + "Not Found", + http.client.HTTPMessage(), + io.BytesIO(b"not json"), + ) + + # The release lookup's 404 is a miss; the repository's 404 is the error. + with ( + patch.object( + release, + "git_output", + side_effect=_git_history(("a" * 40, "1.2.3")), + ), + patch( + "nvx_tools.release.urllib.request.OpenerDirector.open", + side_effect=error, + ) as opener_open, + self.assertRaisesRegex(release.ScriptError, "HTTP 404.*set GH_TOKEN"), + ): + release._checkout_release_asset("example/nvx", "linux-kvm", None) + + self.assertEqual(opener_open.call_count, 2) + + def test_exhausted_rate_limit_reports_retry_hint(self): + headers = http.client.HTTPMessage() + headers["x-ratelimit-remaining"] = "0" + error = urllib.error.HTTPError( + "https://api.github.invalid/releases", + 403, + "Forbidden", + headers, + io.BytesIO(b""), + ) + + with ( + patch.object( + release, + "git_output", + side_effect=_git_history(("a" * 40, "1.2.3")), + ), + patch( + "nvx_tools.release.urllib.request.OpenerDirector.open", + side_effect=error, + ), + self.assertRaisesRegex(release.ScriptError, "rate limit is exhausted"), + ): + release._checkout_release_asset("example/nvx", "linux-kvm", "token") + + def test_forbidden_query_falls_back_to_public_release(self): + error = urllib.error.HTTPError( + "https://api.github.invalid/releases", + 403, + "Forbidden", + http.client.HTTPMessage(), + io.BytesIO( + json.dumps( + {"message": "Resource protected by organization SAML enforcement."} + ).encode("utf-8") + ), + ) + public_response = io.BytesIO( + json.dumps( + { + "draft": False, + "tag_name": "v1.2.3-dev.aaaaaaaaaaaa", + "assets": [ + { + "name": "nvx-1.2.3-linux-kvm.tar.gz", + "url": "https://api.example.invalid/linux", + "size": 300, + } + ], + } + ).encode("utf-8") + ) + + with ( + patch.object( + release, + "git_output", + side_effect=_git_history(("a" * 40, "1.2.3")), + ), + patch( + "nvx_tools.release.urllib.request.OpenerDirector.open", + side_effect=[error, public_response], + ) as opener_open, + patch("sys.stderr", io.StringIO()) as stderr, + ): + asset, download_token = release._checkout_release_asset_with_fallback( + "example/nvx", + "linux-kvm", + "token", + ) + + self.assertEqual(opener_open.call_count, 2) + authenticated_request = opener_open.call_args_list[0].args[0] + public_request = opener_open.call_args_list[1].args[0] + self.assertIsNotNone(authenticated_request.get_header("Authorization")) + self.assertIsNone(public_request.get_header("Authorization")) + self.assertEqual(asset.tag, "v1.2.3-dev.aaaaaaaaaaaa") + self.assertIsNone(download_token) + self.assertIn("retrying without credentials", stderr.getvalue()) + + def test_asset_download_drops_credentials_across_origins(self): + handler = common._CrossOriginRedirectHandler() + request = urllib.request.Request( + "https://api.github.invalid/assets/1", + headers={"Authorization": AUTHORIZATION_VALUE, "Accept": "*/*"}, + ) + + redirected = handler.redirect_request( + request, + io.BytesIO(b""), + 302, + "Found", + http.client.HTTPMessage(), + "https://objects.github.invalid/assets/1?signature=abc", + ) + same_host = handler.redirect_request( + request, + io.BytesIO(b""), + 302, + "Found", + http.client.HTTPMessage(), + "https://api.github.invalid/assets/2", + ) + downgraded = handler.redirect_request( + request, + io.BytesIO(b""), + 302, + "Found", + http.client.HTTPMessage(), + "http://api.github.invalid/assets/3", + ) + + self.assertIsNotNone(redirected) + self.assertIsNotNone(same_host) + self.assertIsNotNone(downgraded) + assert ( + redirected is not None and same_host is not None and downgraded is not None + ) + self.assertIsNone(redirected.get_header("Authorization")) + self.assertEqual(redirected.get_header("Accept"), "*/*") + self.assertEqual(same_host.get_header("Authorization"), AUTHORIZATION_VALUE) + self.assertIsNone(downgraded.get_header("Authorization")) + + def test_rejected_token_falls_back_to_public_release(self): + asset = release._ReleaseAsset( + "v1.2.3", + "nvx-1.2.3-linux-kvm.tar.gz", + "https://api.github.invalid/assets/1", + 4, + ) + authenticated_error = release._GitHubReleaseQueryError( + 403, + "authenticated query failed", + ) + stderr = io.StringIO() + + def write_archive( + _url: str, + destination: Path, + **kwargs: object, + ) -> None: + self.assertIsInstance( + kwargs["opener"], + urllib.request.OpenerDirector, + ) + self.assertNotIn("Authorization", cast(dict[str, str], kwargs["headers"])) + destination.write_bytes(b"data") + + with ( + patch.dict(os.environ, {"GH_TOKEN": "token"}), + patch.object( + release, + "_checkout_release_asset", + side_effect=[authenticated_error, asset], + ) as latest_release_asset, + patch.object(release, "download", side_effect=write_archive) as download, + patch.object(release, "_install_release_archive") as install, + patch("sys.stderr", stderr), + ): + release.download_checkout_release("example/nvx", "linux-kvm") + + self.assertEqual( + latest_release_asset.call_args_list, + [ + call("example/nvx", "linux-kvm", "token"), + call("example/nvx", "linux-kvm", None), + ], + ) + self.assertEqual(download.call_count, 1) + install.assert_called_once() + self.assertIn("retrying without credentials", stderr.getvalue()) + + def test_failed_public_fallback_preserves_authenticated_error(self): + authenticated_error = release._GitHubReleaseQueryError( + 403, + "authenticated query failed", + ) + public_error = release._GitHubReleaseQueryError( + 404, + "public query failed", + ) + + with ( + patch.dict(os.environ, {"GH_TOKEN": "token"}), + patch.object( + release, + "_checkout_release_asset", + side_effect=[authenticated_error, public_error], + ), + patch("sys.stderr", io.StringIO()), + self.assertRaisesRegex( + release.ScriptError, + "authenticated query failed", + ), + ): + release.download_checkout_release("example/nvx", "linux-kvm") + + def test_authenticated_asset_download_uses_credential_safe_opener(self): + asset = release._ReleaseAsset( + "v1.2.3", + "nvx-1.2.3-linux-kvm.tar.gz", + "https://api.github.invalid/assets/1", + 4, + ) + + def write_archive( + _url: str, + destination: Path, + **kwargs: object, + ) -> None: + self.assertIsInstance( + kwargs["opener"], + urllib.request.OpenerDirector, + ) + self.assertEqual( + kwargs["headers"], + release._github_headers("token", "application/octet-stream"), + ) + destination.write_bytes(b"data") + + with ( + patch.dict(os.environ, {"GH_TOKEN": "token"}), + patch.object(release, "_checkout_release_asset", return_value=asset), + patch.object(release, "download", side_effect=write_archive) as download, + patch.object(release, "_install_release_archive") as install, + ): + release.download_checkout_release("example/nvx", "linux-kvm") + + self.assertEqual(download.call_count, 1) + install.assert_called_once() + + def test_binary_package_stages_files_and_checksums(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + paths, kernel_inputs, revision = _write_release_fixture(root) + build_dir = paths["build"] + source_dir = paths["source"] + openvmm_dir = paths["openvmm"] + binary = paths["binary"] + destination = root / "staged" + stderr = io.StringIO() + + def artifact_path(name: str) -> Path: + return build_dir / name + + with ( + patch.object(BuildConstants, "REPO_ROOT", root), + patch.object(BuildConstants, "SOURCE_DIR", source_dir), + patch.object( + UbuntuBuildConstants, + "PACKAGE_LOCK", + root / "ubuntu" / "packages.lock.json", + ), + patch.object(OpenVMMBuildConstants, "DIRECTORY", openvmm_dir), + patch.object( + release, + "artifact_path", + side_effect=artifact_path, + ), + patch.object(release, "openvmm_binary_path", return_value=binary), + patch.object( + release, + "kernel_provenance_inputs", + return_value=kernel_inputs, + ), + patch.object( + release, + "openvmm_git_state", + return_value=(revision, True), + ), + patch("sys.stderr", stderr), + ): + release.package_release( + version="1.0.0", + destination=destination, + include_source=False, + force=False, + ) + + self.assertTrue((destination / "bin" / binary.name).is_file()) + for name in ReleaseBuildConstants.GUEST_ARTIFACT_NAMES: + self.assertTrue((destination / "guest" / name).is_file()) + self.assertTrue( + ( + destination / "provenance" / OpenVMMBuildConstants.PROVENANCE_NAME + ).is_file() + ) + self.assertTrue( + ( + destination / "provenance" / KernelBuildConstants.PROVENANCE_NAME + ).is_file() + ) + self.assertTrue( + ( + destination / "provenance" / InitramfsBuildConstants.PROVENANCE_NAME + ).is_file() + ) + manifest = json.loads( + (destination / "SOURCE-MANIFEST.json").read_text(encoding="utf-8") + ) + self.assertEqual(manifest["distribution"]["version"], "1.0.0") + self.assertNotIn("guest_agent", manifest) + self.assertEqual(manifest["openvmm"]["source_revision"], revision) + self.assertEqual( + manifest["openvmm"]["executable_sha256"], + common.sha256_file(destination / "bin" / binary.name), + ) + self.assertEqual( + manifest["openvmm"]["microvm_abi_version"], + 2, + ) + self.assertEqual( + manifest["openvmm"]["control_session_protocol_version"], + 1, + ) + self.assertEqual( + manifest["openvmm"]["control_contract_revision"], + "nvx-microvm-v2-control-v2", + ) + self.assertEqual( + manifest["linux"]["kernel_sha256"], + common.sha256_file(destination / "guest" / "vmlinux"), + ) + self.assertEqual( + manifest["linux"]["config_sha256"], + common.sha256_file(destination / "guest" / "vmlinux.config"), + ) + self.assertEqual( + manifest["azurelinux"]["initramfs_sha256"], + common.sha256_file( + destination / "guest" / AzureLinuxBuildConstants.INITRAMFS_NAME + ), + ) + self.assertEqual( + manifest["azurelinux"]["initramfs_package_manifest_sha256"], + common.sha256_file( + destination + / "guest" + / AzureLinuxBuildConstants.PACKAGE_MANIFEST_NAME + ), + ) + common.verify_sha256_sums(destination) + self.assertIn("binary-only package", stderr.getvalue()) + + def test_source_package_omits_azure_linux_artifacts(self): + for azurelinux_built in (True, False): + with ( + self.subTest(azurelinux_built=azurelinux_built), + tempfile.TemporaryDirectory() as temporary, + ): + root = Path(temporary) + paths, kernel_inputs, revision = _write_release_fixture(root) + build_dir = paths["build"] + if not azurelinux_built: + for name in ReleaseBuildConstants.AZURELINUX_ARTIFACT_NAMES: + (build_dir / name).unlink() + self._package_source_release_without_azure_linux( + root, paths, kernel_inputs, revision + ) + + def _package_source_release_without_azure_linux( + self, + root: Path, + paths: dict[str, Path], + kernel_inputs: dict[str, object], + revision: str, + ) -> None: + build_dir = paths["build"] + source_dir = paths["source"] + openvmm_dir = paths["openvmm"] + binary = paths["binary"] + destination = root / "staged" + stderr = io.StringIO() + linux_source_archive = ( + source_dir + / KernelBuildConstants.SOURCE_DIRECTORY_NAME + / KernelBuildConstants.SOURCE_ARCHIVE_NAME + ) + linux_source_archive.parent.mkdir(parents=True, exist_ok=True) + linux_source_archive.write_bytes(b"linux-source") + + def artifact_path(name: str) -> Path: + return build_dir / name + + def write_source_archive(output: Path, *_args: object) -> None: + output.parent.mkdir(parents=True, exist_ok=True) + output.write_bytes(output.name.encode("ascii")) + + with ( + patch.object(BuildConstants, "REPO_ROOT", root), + patch.object(BuildConstants, "SOURCE_DIR", source_dir), + patch.object( + UbuntuBuildConstants, + "PACKAGE_LOCK", + root / "ubuntu" / "packages.lock.json", + ), + patch.object(OpenVMMBuildConstants, "DIRECTORY", openvmm_dir), + patch.object( + release, + "artifact_path", + side_effect=artifact_path, + ), + patch.object(release, "openvmm_binary_path", return_value=binary), + patch.object( + release, + "kernel_provenance_inputs", + return_value=kernel_inputs, + ), + patch.object( + release, + "openvmm_git_state", + return_value=(revision, True), + ), + patch.object(release, "_validate_alpine_sources"), + patch.object(release, "_validate_ubuntu_sources"), + patch.object(release, "_validate_linux_source_archive"), + patch.object( + release, + "_project_source_archive", + side_effect=write_source_archive, + ) as project_source_archive, + patch.object( + release, + "_alpine_source_archive", + side_effect=write_source_archive, + ), + patch.object( + release, + "_ubuntu_source_archive", + side_effect=write_source_archive, + ), + patch("sys.stderr", stderr), + ): + release.package_release( + version="1.0.0", + destination=destination, + include_source=True, + force=False, + ) + + for name in ReleaseBuildConstants.AZURELINUX_ARTIFACT_NAMES: + self.assertFalse((destination / "guest" / name).exists()) + for name in ReleaseBuildConstants.GUEST_ARTIFACT_NAMES: + if name in ReleaseBuildConstants.AZURELINUX_ARTIFACT_NAMES: + continue + self.assertTrue((destination / "guest" / name).is_file()) + self.assertNotIn( + build_dir / AzureLinuxBuildConstants.PACKAGE_MANIFEST_NAME, + project_source_archive.call_args.args[2], + ) + manifest = json.loads( + (destination / "SOURCE-MANIFEST.json").read_text(encoding="utf-8") + ) + self.assertNotIn("azurelinux", manifest) + common.verify_sha256_sums(destination) + self.assertIn("omits the Azure Linux guest", stderr.getvalue()) + + def test_runtime_provenance_validation_uses_required_paths(self): + build_dir = Path("build") + binary = Path("openvmm") + + def artifact_path(name: str) -> Path: + return build_dir / name + + def require_file(path: Path, _description: str) -> Path: + return path + + with ( + patch.object(release, "artifact_path", side_effect=artifact_path), + patch.object(release, "openvmm_binary_path", return_value=binary), + patch.object( + release, + "require_file", + side_effect=require_file, + ) as require, + patch.object(release, "_validate_openvmm_provenance") as openvmm, + patch.object(release, "_validate_kernel_provenance"), + patch.object(release, "_validate_initramfs_provenance"), + ): + release.validate_runtime_artifact_provenance() + + self.assertEqual( + require.call_args_list[-3:], + [ + call( + build_dir / OpenVMMBuildConstants.PROVENANCE_NAME, + "OpenVMM build provenance", + ), + call( + build_dir / KernelBuildConstants.PROVENANCE_NAME, + "kernel build provenance", + ), + call( + build_dir / InitramfsBuildConstants.PROVENANCE_NAME, + "initramfs build provenance", + ), + ], + ) + openvmm.assert_called_once_with( + binary, build_dir / OpenVMMBuildConstants.PROVENANCE_NAME + ) + + def test_package_rejects_dirty_openvmm_provenance(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + paths, _kernel_inputs, revision = _write_release_fixture(root) + + with ( + patch.object( + release, + "openvmm_git_state", + return_value=(revision, False), + ), + self.assertRaisesRegex( + common.ScriptError, + "current clean pinned source", + ), + ): + release._validate_openvmm_provenance( + paths["binary"], + paths["build"] / OpenVMMBuildConstants.PROVENANCE_NAME, + ) + + def test_package_rejects_stale_kernel_provenance(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + paths, kernel_inputs, _revision = _write_release_fixture(root) + kernel = paths["build"] / "vmlinux" + kernel.write_bytes(b"new kernel bytes") + + with ( + patch.object( + release, + "kernel_provenance_inputs", + return_value=kernel_inputs, + ), + self.assertRaisesRegex( + common.ScriptError, + "kernel build provenance", + ), + ): + release._validate_kernel_provenance( + kernel, + paths["build"] / "vmlinux.config", + paths["build"] / KernelBuildConstants.PROVENANCE_NAME, + ) + + def test_package_rejects_stale_initramfs_provenance(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + paths, _kernel_inputs, _revision = _write_release_fixture(root) + initramfs = paths["build"] / "initramfs.cpio.gz" + initramfs.write_bytes(b"new initramfs bytes") + + with self.assertRaisesRegex( + common.ScriptError, + "initramfs build provenance", + ): + release._validate_initramfs_provenance( + initramfs, + paths["build"] / "initramfs.cpio.gz.packages.json", + paths["build"] / InitramfsBuildConstants.PROVENANCE_NAME, + ) + + def test_package_rejects_stale_initramfs_package_manifest_provenance(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + paths, _kernel_inputs, _revision = _write_release_fixture(root) + package_manifest = paths["build"] / "initramfs.cpio.gz.packages.json" + package_manifest.write_bytes(b"new package manifest bytes") + + with self.assertRaisesRegex( + common.ScriptError, + "initramfs build provenance", + ): + release._validate_initramfs_provenance( + paths["build"] / "initramfs.cpio.gz", + package_manifest, + paths["build"] / InitramfsBuildConstants.PROVENANCE_NAME, + ) + + def test_guest_release_inputs_reject_stale_ubuntu_artifacts(self): + for artifact_name in ( + "initramfs-ubuntu.cpio.gz", + "ubuntu-distro.erofs", + ): + with ( + self.subTest(artifact=artifact_name), + tempfile.TemporaryDirectory() as temporary, + ): + root = Path(temporary) + paths, _kernel_inputs, _revision = _write_release_fixture(root) + build_dir = paths["build"] + (build_dir / artifact_name).write_bytes(b"stale artifact") + with ( + patch.object( + release, + "artifact_path", + side_effect=build_dir.joinpath, + ), + self.assertRaisesRegex( + common.ScriptError, + "Ubuntu artifact manifest does not match", + ), + ): + release._guest_release_inputs(include_azurelinux=True) + + def test_guest_release_inputs_reject_stale_azurelinux_initramfs(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + paths, _kernel_inputs, _revision = _write_release_fixture(root) + build_dir = paths["build"] + (build_dir / AzureLinuxBuildConstants.INITRAMFS_NAME).write_bytes( + b"stale artifact" + ) + with ( + patch.object( + release, + "artifact_path", + side_effect=build_dir.joinpath, + ), + self.assertRaisesRegex( + common.ScriptError, + "Azure Linux initramfs manifest is invalid", + ), + ): + release._guest_release_inputs(include_azurelinux=True) + + def test_guest_release_inputs_reject_unpinned_azurelinux_image(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + paths, _kernel_inputs, _revision = _write_release_fixture(root) + build_dir = paths["build"] + manifest_path = build_dir / AzureLinuxBuildConstants.PACKAGE_MANIFEST_NAME + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + manifest["image"] = "mcr.microsoft.com/azurelinux/base/core@sha256:" + ( + "0" * 64 + ) + manifest_path.write_text(json.dumps(manifest), encoding="utf-8") + with ( + patch.object( + release, + "artifact_path", + side_effect=build_dir.joinpath, + ), + self.assertRaisesRegex( + common.ScriptError, + "Azure Linux initramfs manifest is invalid", + ), + ): + release._guest_release_inputs(include_azurelinux=True) + + def test_guest_release_inputs_reject_stale_azurelinux_build_inputs(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + paths, _kernel_inputs, _revision = _write_release_fixture(root) + build_dir = paths["build"] + manifest_path = build_dir / AzureLinuxBuildConstants.PACKAGE_MANIFEST_NAME + current_manifest = manifest_path.read_text(encoding="utf-8") + with ( + patch.object(BuildConstants, "REPO_ROOT", root), + patch.object( + UbuntuBuildConstants, + "PACKAGE_LOCK", + root / "ubuntu" / "packages.lock.json", + ), + patch.object( + release, + "artifact_path", + side_effect=build_dir.joinpath, + ), + ): + release._guest_release_inputs(include_azurelinux=True) + with self.subTest(case="missing digest"): + document = json.loads(current_manifest) + del document["input_sha256"] + manifest_path.write_text(json.dumps(document), encoding="utf-8") + with self.assertRaisesRegex( + common.ScriptError, + "Azure Linux initramfs manifest does not match the " + "current build inputs", + ): + release._guest_release_inputs(include_azurelinux=True) + manifest_path.write_text(current_manifest, encoding="utf-8") + with self.subTest(case="edited Dockerfile"): + dockerfile = root / "docker" / "Dockerfile" + dockerfile.write_text( + dockerfile.read_text(encoding="utf-8") + "# changed\n", + encoding="utf-8", + ) + with self.assertRaisesRegex( + common.ScriptError, + "Azure Linux initramfs manifest does not match the " + "current build inputs", + ): + release._guest_release_inputs(include_azurelinux=True) + + def test_guest_release_inputs_require_azurelinux_only_when_included(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + paths, _kernel_inputs, _revision = _write_release_fixture(root) + build_dir = paths["build"] + for name in ReleaseBuildConstants.AZURELINUX_ARTIFACT_NAMES: + (build_dir / name).unlink() + with patch.object( + release, + "artifact_path", + side_effect=build_dir.joinpath, + ): + guest_names, _alpine, _ubuntu, azurelinux_manifests = ( + release._guest_release_inputs(include_azurelinux=False) + ) + with self.assertRaisesRegex( + common.ScriptError, + "required guest artifact initramfs-azurelinux.cpio.gz not found", + ): + release._guest_release_inputs(include_azurelinux=True) + + self.assertEqual( + guest_names, + [ + name + for name in ReleaseBuildConstants.GUEST_ARTIFACT_NAMES + if name not in ReleaseBuildConstants.AZURELINUX_ARTIFACT_NAMES + ], + ) + self.assertEqual(azurelinux_manifests, []) + + def test_guest_release_inputs_reject_stale_ubuntu_source_inputs(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + paths, _kernel_inputs, _revision = _write_release_fixture(root) + build_dir = paths["build"] + manifest_path = build_dir / "initramfs-ubuntu.cpio.gz.packages.json" + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + manifest["input_sha256"] = "0" * 64 + manifest_path.write_text(json.dumps(manifest), encoding="utf-8") + with ( + patch.object( + release, + "artifact_path", + side_effect=build_dir.joinpath, + ), + self.assertRaisesRegex( + common.ScriptError, + "Ubuntu artifact manifest does not match", + ), + ): + release._guest_release_inputs(include_azurelinux=True) + + def test_package_rejects_tampered_source_metadata_without_replacing_output(self): + cases: tuple[tuple[str, str, object, str], ...] = ( + ("root", "format", 2, "format must be 1"), + ("linux", "version", "6.18.37", "Linux version"), + ( + "linux", + "upstream_archive_sha256", + "0" * 64, + "Linux upstream_archive_sha256", + ), + ("linux", "patches", [], "Linux patches"), + ( + "alpine", + "minirootfs_sha256", + "0" * 64, + "Alpine minirootfs_sha256", + ), + ) + for section, field, invalid_value, error in cases: + with ( + self.subTest(section=section, field=field), + tempfile.TemporaryDirectory() as temporary, + ): + root = Path(temporary) + paths, kernel_inputs, revision = _write_release_fixture(root) + fixture_build_dir = paths["build"] + manifest_path = root / "SOURCE-MANIFEST.json" + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + if section == "root": + manifest[field] = invalid_value + else: + manifest_section = cast(dict[str, object], manifest[section]) + manifest_section[field] = invalid_value + manifest_path.write_text(json.dumps(manifest), encoding="utf-8") + destination = root / "dist" / "1.0.0" + destination.mkdir(parents=True) + marker = destination / "prior.txt" + marker.write_text("prior release", encoding="utf-8") + + def artifact_path( + name: str, + build_dir: Path = fixture_build_dir, + ) -> Path: + return build_dir / name + + with ( + patch.object(BuildConstants, "REPO_ROOT", root), + patch.object(BuildConstants, "SOURCE_DIR", paths["source"]), + patch.object( + UbuntuBuildConstants, + "PACKAGE_LOCK", + root / "ubuntu" / "packages.lock.json", + ), + patch.object(OpenVMMBuildConstants, "DIRECTORY", paths["openvmm"]), + patch.object( + release, + "artifact_path", + side_effect=artifact_path, + ), + patch.object( + release, + "openvmm_binary_path", + return_value=paths["binary"], + ), + patch.object( + release, + "kernel_provenance_inputs", + return_value=kernel_inputs, + ), + patch.object( + release, + "openvmm_git_state", + return_value=(revision, True), + ), + patch("sys.stderr", io.StringIO()), + self.assertRaisesRegex( + common.ScriptError, + error, + ), + ): + release.package_release( + version="1.0.0", + destination=destination, + include_source=False, + force=True, + ) + + self.assertEqual( + marker.read_text(encoding="utf-8"), + "prior release", + ) + + def test_failed_staged_verification_preserves_existing_release(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + paths, kernel_inputs, revision = _write_release_fixture(root) + build_dir = paths["build"] + destination = root / "dist" / "1.0.0" + destination.mkdir(parents=True) + marker = destination / "prior.txt" + marker.write_text("prior release", encoding="utf-8") + + def artifact_path(name: str) -> Path: + return build_dir / name + + original_copy = release._copy_release_file + + def corrupt_packaged_binary(source: Path, target: Path) -> None: + original_copy(source, target) + if target.name == paths["binary"].name: + target.write_bytes(b"corrupt") + + with ( + patch.object(BuildConstants, "REPO_ROOT", root), + patch.object(BuildConstants, "SOURCE_DIR", paths["source"]), + patch.object( + UbuntuBuildConstants, + "PACKAGE_LOCK", + root / "ubuntu" / "packages.lock.json", + ), + patch.object(OpenVMMBuildConstants, "DIRECTORY", paths["openvmm"]), + patch.object(release, "artifact_path", side_effect=artifact_path), + patch.object( + release, + "openvmm_binary_path", + return_value=paths["binary"], + ), + patch.object( + release, + "kernel_provenance_inputs", + return_value=kernel_inputs, + ), + patch.object( + release, + "openvmm_git_state", + return_value=(revision, True), + ), + patch.object( + release, + "_copy_release_file", + side_effect=corrupt_packaged_binary, + ), + patch("sys.stderr", io.StringIO()), + self.assertRaisesRegex( + common.ScriptError, + "packaged OpenVMM executable", + ), + ): + release.package_release( + version="1.0.0", + destination=destination, + include_source=False, + force=True, + ) + + self.assertEqual(marker.read_text(encoding="utf-8"), "prior release") + self.assertEqual( + [path.name for path in destination.iterdir()], ["prior.txt"] + ) + + def test_publication_failure_restores_existing_release(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary).resolve() + destination = root / "dist" / "release" + destination.mkdir(parents=True) + (destination / "prior.txt").write_text("prior", encoding="utf-8") + staging = root / "staging" + staging.mkdir() + (staging / "new.txt").write_text("new", encoding="utf-8") + original_replace = Path.replace + + def fail_staging_replace(source: Path, target: Path) -> Path: + if source == staging: + raise OSError("injected publication failure") + return original_replace(source, target) + + with ( + patch.object(BuildConstants, "REPO_ROOT", root), + patch.object(Path, "replace", new=fail_staging_replace), + self.assertRaisesRegex( + common.ScriptError, + "failed to publish staged release", + ), + ): + release._publish_release_directory( + staging, + destination, + force=True, + ) + + self.assertEqual( + (destination / "prior.txt").read_text(encoding="utf-8"), + "prior", + ) + self.assertFalse(staging.exists()) + + def test_failed_rollback_preserves_prior_backup_and_staging(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary).resolve() + destination = root / "dist" / "release" + destination.mkdir(parents=True) + (destination / "prior.txt").write_text("prior", encoding="utf-8") + staging = root / "staging" + staging.mkdir() + (staging / "new.txt").write_text("new", encoding="utf-8") + original_replace = Path.replace + + def fail_publication_and_restore(source: Path, target: Path) -> Path: + if source == staging or source.name.startswith(".release.backup-"): + raise OSError("injected rename failure") + return original_replace(source, target) + + with ( + patch.object(BuildConstants, "REPO_ROOT", root), + patch.object(Path, "replace", new=fail_publication_and_restore), + self.assertRaisesRegex( + common.ScriptError, + "prior release remains", + ), + ): + release._publish_release_directory( + staging, + destination, + force=True, + ) + + backups = list(destination.parent.glob(".release.backup-*")) + self.assertEqual(len(backups), 1) + self.assertEqual( + (backups[0] / "prior.txt").read_text(encoding="utf-8"), + "prior", + ) + self.assertEqual( + (staging / "new.txt").read_text(encoding="utf-8"), + "new", + ) + + def test_publication_rechecks_no_force_after_staging(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + destination = root / "release" + destination.mkdir() + (destination / "prior.txt").write_text("prior", encoding="utf-8") + staging = root / "staging" + staging.mkdir() + (staging / "new.txt").write_text("new", encoding="utf-8") + + with self.assertRaisesRegex( + common.ScriptError, + "pass --force", + ): + release._publish_release_directory( + staging, + destination, + force=False, + ) + + self.assertEqual( + (destination / "prior.txt").read_text(encoding="utf-8"), + "prior", + ) + self.assertFalse(staging.exists()) + + def test_publication_rechecks_force_destination_scope(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + destination = root / "outside-dist" + destination.mkdir() + (destination / "prior.txt").write_text("prior", encoding="utf-8") + staging = root / "staging" + staging.mkdir() + (staging / "new.txt").write_text("new", encoding="utf-8") + + with ( + patch.object(BuildConstants, "REPO_ROOT", root), + self.assertRaisesRegex( + common.ScriptError, + "below dist", + ), + ): + release._publish_release_directory( + staging, + destination, + force=True, + ) + + self.assertEqual( + (destination / "prior.txt").read_text(encoding="utf-8"), + "prior", + ) + self.assertFalse(staging.exists()) + + def test_release_archive_installs_runtime_layout(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + package_root = root / "package" / "nvx-1.2.3-test" + binary_name = "openvmm.exe" if os.name == "nt" else "openvmm" + binary = package_root / "bin" / binary_name + binary.parent.mkdir(parents=True) + binary.write_bytes(b"openvmm") + for name in ReleaseBuildConstants.GUEST_ARTIFACT_NAMES: + guest = package_root / "guest" / name + guest.parent.mkdir(parents=True, exist_ok=True) + guest.write_bytes(name.encode("ascii")) + for name in ( + OpenVMMBuildConstants.PROVENANCE_NAME, + KernelBuildConstants.PROVENANCE_NAME, + InitramfsBuildConstants.PROVENANCE_NAME, + ): + provenance = package_root / "provenance" / name + provenance.parent.mkdir(parents=True, exist_ok=True) + provenance.write_bytes(name.encode("ascii")) + common.write_sha256_sums(package_root) + + if os.name == "nt": + archive_path = root / "nvx-1.2.3-windows-whp.zip" + with zipfile.ZipFile(archive_path, "w") as package: + for path in package_root.rglob("*"): + package.write(path, path.relative_to(package_root.parent)) + else: + archive_path = root / "nvx-1.2.3-linux-kvm.tar.gz" + with tarfile.open(archive_path, "w:gz") as package: + package.add(package_root, arcname=package_root.name) + + build_dir = root / "runtime" / "build" + binary_destination = ( + root / "runtime" / "openvmm" / "target" / "release" / binary_name + ) + + def artifact_path(name: str) -> Path: + return build_dir / name + + with ( + patch.object( + release, + "artifact_path", + side_effect=artifact_path, + ), + patch.object( + release, + "openvmm_binary_path", + return_value=binary_destination, + ), + ): + release._install_release_archive(archive_path) + + self.assertEqual(binary_destination.read_bytes(), b"openvmm") + for name in ReleaseBuildConstants.GUEST_ARTIFACT_NAMES: + self.assertEqual( + (build_dir / name).read_bytes(), + name.encode("ascii"), + ) + for name in ( + OpenVMMBuildConstants.PROVENANCE_NAME, + KernelBuildConstants.PROVENANCE_NAME, + InitramfsBuildConstants.PROVENANCE_NAME, + ): + self.assertEqual( + (build_dir / name).read_bytes(), + name.encode("ascii"), + ) + + def test_source_release_archive_removes_stale_azure_artifacts(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + package_root = root / "package" / "nvx-1.2.3-test" + binary_name = "openvmm.exe" if os.name == "nt" else "openvmm" + binary = package_root / "bin" / binary_name + binary.parent.mkdir(parents=True) + binary.write_bytes(b"openvmm") + for name in ReleaseBuildConstants.GUEST_ARTIFACT_NAMES: + if name in ReleaseBuildConstants.AZURELINUX_ARTIFACT_NAMES: + continue + guest = package_root / "guest" / name + guest.parent.mkdir(parents=True, exist_ok=True) + guest.write_bytes(name.encode("ascii")) + for name in ( + OpenVMMBuildConstants.PROVENANCE_NAME, + KernelBuildConstants.PROVENANCE_NAME, + InitramfsBuildConstants.PROVENANCE_NAME, + ): + provenance = package_root / "provenance" / name + provenance.parent.mkdir(parents=True, exist_ok=True) + provenance.write_bytes(name.encode("ascii")) + (package_root / "SOURCE-MANIFEST.json").write_text( + json.dumps({"linux": {}, "alpine": {}, "ubuntu": {}}), + encoding="utf-8", + ) + common.write_sha256_sums(package_root) + + if os.name == "nt": + archive_path = root / "nvx-1.2.3-windows-whp.zip" + with zipfile.ZipFile(archive_path, "w") as package: + for path in package_root.rglob("*"): + package.write(path, path.relative_to(package_root.parent)) + else: + archive_path = root / "nvx-1.2.3-linux-kvm.tar.gz" + with tarfile.open(archive_path, "w:gz") as package: + package.add(package_root, arcname=package_root.name) + + build_dir = root / "runtime" / "build" + binary_destination = ( + root / "runtime" / "openvmm" / "target" / "release" / binary_name + ) + build_dir.mkdir(parents=True) + for name in ReleaseBuildConstants.GUEST_ARTIFACT_NAMES: + (build_dir / name).write_bytes(b"stale " + name.encode("ascii")) + + def artifact_path(name: str) -> Path: + return build_dir / name + + with ( + patch.object( + release, + "artifact_path", + side_effect=artifact_path, + ), + patch.object( + release, + "openvmm_binary_path", + return_value=binary_destination, + ), + ): + release._install_release_archive(archive_path) + + self.assertEqual(binary_destination.read_bytes(), b"openvmm") + for name in ReleaseBuildConstants.GUEST_ARTIFACT_NAMES: + destination = build_dir / name + if name in ReleaseBuildConstants.AZURELINUX_ARTIFACT_NAMES: + self.assertFalse(destination.exists()) + else: + self.assertEqual(destination.read_bytes(), name.encode("ascii")) + + +class PositiveIntTests(unittest.TestCase): + def test_accepts_positive_integer(self): + self.assertEqual(common.positive_int("1"), 1) + + def test_rejects_zero_and_negative_values(self): + for value in ("0", "-1"): + with self.subTest(value=value): + with self.assertRaisesRegex( + argparse.ArgumentTypeError, "^must be greater than zero$" + ): + common.positive_int(value) + + def test_rejects_non_integer_input(self): + with self.assertRaises(ValueError): + common.positive_int("not-an-int") + + def test_accepts_custom_error_message(self): + with self.assertRaisesRegex(argparse.ArgumentTypeError, "^must be at least 1$"): + common.positive_int("0", message="must be at least 1") + + +class PositiveFloatTests(unittest.TestCase): + def test_accepts_positive_finite_float(self): + self.assertEqual(benchmark.positive_float("1.5"), 1.5) + + def test_rejects_nonpositive_and_nonfinite_values(self): + for value in ("0", "-1", "nan", "inf"): + with self.subTest(value=value): + with self.assertRaisesRegex( + argparse.ArgumentTypeError, + "^must be finite and greater than 0$", + ): + benchmark.positive_float(value) + + +class GitOutputTests(unittest.TestCase): + def test_runs_git_from_repository_root_and_strips_output(self): + completed = MagicMock(stdout=" output \n") + with patch.object(common.subprocess, "run", return_value=completed) as run: + self.assertEqual(common.git_output("status", "--short"), "output") + run.assert_called_once_with( + ["git", "-C", str(BuildConstants.REPO_ROOT), "status", "--short"], + check=True, + capture_output=True, + text=True, + encoding="utf-8", + errors="strict", + timeout=30.0, + ) + + +class SharedCommandTests(unittest.TestCase): + def test_commands_that_cannot_start_raise_script_errors(self): + with tempfile.TemporaryDirectory() as temporary: + missing = Path(temporary) / "missing-command" + for run in (common.run_checked, common.run_capture): + with self.subTest(helper=run.__name__): + with self.assertRaisesRegex( + common.ScriptError, + f"^failed to run command {re.escape(str(missing))} --version: ", + ) as raised: + run([missing, "--version"]) + self.assertIsInstance(raised.exception.__cause__, FileNotFoundError) + + +class SharedFileTests(unittest.TestCase): + def test_checksum_manifest_detects_modified_file(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + payload = root / "nested" / "payload.txt" + payload.parent.mkdir() + payload.write_text("original", encoding="ascii") + + common.write_sha256_sums(root) + common.verify_sha256_sums(root) + payload.write_text("modified", encoding="ascii") + + with self.assertRaisesRegex(common.ScriptError, "checksum mismatch"): + common.verify_sha256_sums(root) + + def test_checksum_manifest_rejects_unlisted_tampered_provenance(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + payload = root / "bin" / "openvmm" + provenance = root / "provenance" / "openvmm.provenance.json" + payload.parent.mkdir() + provenance.parent.mkdir() + payload.write_bytes(b"openvmm") + provenance.write_bytes(b"original provenance") + common.write_sha256_sums(root) + checksum_file = root / "SHA256SUMS" + checksum_file.write_text( + "\n".join( + line + for line in checksum_file.read_text(encoding="ascii").splitlines() + if not line.endswith("provenance/openvmm.provenance.json") + ) + + "\n", + encoding="ascii", + ) + provenance.write_bytes(b"tampered provenance") + + with self.assertRaisesRegex( + common.ScriptError, + "unlisted file.*openvmm.provenance.json", + ): + common.verify_sha256_sums(root) + + def test_checksum_manifest_rejects_missing_and_duplicate_paths(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + payload = root / "payload" + payload.write_bytes(b"payload") + common.write_sha256_sums(root) + checksum_file = root / "SHA256SUMS" + line = checksum_file.read_text(encoding="ascii") + + payload.unlink() + with self.assertRaisesRegex(common.ScriptError, "invalid checksum path"): + common.verify_sha256_sums(root) + + payload.write_bytes(b"payload") + checksum_file.write_text(line + line, encoding="ascii") + with self.assertRaisesRegex(common.ScriptError, "duplicate checksum path"): + common.verify_sha256_sums(root) + + def test_checksum_manifest_rejects_non_ascii_text(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + (root / "SHA256SUMS").write_bytes(b"\xff") + + with self.assertRaisesRegex(common.ScriptError, "only ASCII text"): + common.verify_sha256_sums(root) + + def test_checksum_manifest_rejects_unsafe_paths_and_symlinks(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + payload = root / "payload" + payload.write_bytes(b"payload") + common.write_sha256_sums(root) + checksum_file = root / "SHA256SUMS" + checksum = hashlib.sha256(b"payload").hexdigest() + checksum_file.write_text( + f"{checksum} ../payload\n", + encoding="ascii", + ) + with self.assertRaisesRegex(common.ScriptError, "malformed checksum line"): + common.verify_sha256_sums(root) + + common.write_sha256_sums(root) + link = root / "payload-link" + try: + link.symlink_to(payload) + except OSError as error: + self.skipTest(f"symlinks are unavailable: {error}") + with self.assertRaisesRegex(common.ScriptError, "symlink is not allowed"): + common.verify_sha256_sums(root) + + def test_checksum_writer_rejects_trees_the_verifier_rejects(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + payload = root / "payload" + payload.write_bytes(b"payload") + nested = root / "nested" / "SHA256SUMS" + nested.parent.mkdir() + nested.write_bytes(b"nested") + common.write_sha256_sums(root) + self.assertIn( + "nested/SHA256SUMS", + dict(common.verify_sha256_sums(root).files), + ) + checksums = (root / "SHA256SUMS").read_bytes() + + link = root / "payload-link" + try: + link.symlink_to(payload) + except OSError as error: + self.skipTest(f"symlinks are unavailable: {error}") + with self.assertRaisesRegex( + common.ScriptError, + "symlink is not allowed in checksummed tree: payload-link", + ): + common.write_sha256_sums(root) + self.assertEqual((root / "SHA256SUMS").read_bytes(), checksums) + + link.unlink() + (root / "SHA256SUMS").unlink() + (root / "SHA256SUMS").symlink_to(payload) + with self.assertRaisesRegex( + common.ScriptError, + "symlink is not allowed in checksummed tree: SHA256SUMS", + ): + common.write_sha256_sums(root) + self.assertEqual(payload.read_bytes(), b"payload") + + def test_checksum_writer_rejects_special_files(self): + if sys.platform == "win32": + self.skipTest("FIFOs are unavailable on Windows") + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + os.mkfifo(root / "pipe") + + with self.assertRaisesRegex( + common.ScriptError, + "special file is not allowed in checksummed tree: pipe", + ): + common.write_sha256_sums(root) + + self.assertFalse((root / "SHA256SUMS").exists()) + + def test_release_archives_are_reproducible_with_fixed_layout_and_modes(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + source = root / "nvx-1.0.0-test" + executable = source / "bin" / "openvmm" + data = source / "provenance" / "openvmm.provenance.json" + executable.parent.mkdir(parents=True) + data.parent.mkdir() + executable.write_bytes(b"openvmm") + data.write_bytes(b"provenance") + executable.chmod(0o755) + data.chmod(0o644) + common.write_sha256_sums(source) + + for suffix in (".tar.gz", ".zip"): + with self.subTest(suffix=suffix): + first = root / f"first{suffix}" + second = root / f"second{suffix}" + release.create_release_archive(source, first) + os.utime(executable, (1000, 1000)) + release.create_release_archive(source, second) + self.assertEqual(first.read_bytes(), second.read_bytes()) + + if suffix == ".tar.gz": + with tarfile.open(first, "r:gz") as package: + member_list = package.getmembers() + members = {member.name: member for member in member_list} + self.assertEqual( + [member.name for member in member_list], + sorted(member.name for member in member_list), + ) + self.assertEqual( + {member.name for member in member_list if member.isfile()}, + { + f"{source.name}/SHA256SUMS", + f"{source.name}/bin/openvmm", + (f"{source.name}/provenance/openvmm.provenance.json"), + }, + ) + self.assertEqual( + members[f"{source.name}/bin/openvmm"].mode, + 0o755, + ) + self.assertEqual( + members[ + f"{source.name}/provenance/openvmm.provenance.json" + ].mode, + 0o644, + ) + self.assertTrue( + all(member.mtime == 0 for member in members.values()) + ) + else: + with zipfile.ZipFile(first) as package: + member_list = package.infolist() + members = { + member.filename: member for member in member_list + } + self.assertEqual( + [member.filename for member in member_list], + sorted(member.filename for member in member_list), + ) + self.assertEqual( + { + member.filename + for member in member_list + if not member.is_dir() + }, + { + f"{source.name}/SHA256SUMS", + f"{source.name}/bin/openvmm", + (f"{source.name}/provenance/openvmm.provenance.json"), + }, + ) + executable_mode = ( + members[f"{source.name}/bin/openvmm"].external_attr >> 16 + ) & 0o777 + data_mode = ( + members[ + f"{source.name}/provenance/openvmm.provenance.json" + ].external_attr + >> 16 + ) & 0o777 + self.assertEqual(executable_mode, 0o755) + self.assertEqual(data_mode, 0o644) + self.assertTrue( + all( + member.date_time == (1980, 1, 1, 0, 0, 0) + for member in members.values() + ) + ) + + def test_release_archive_rejects_internally_inconsistent_staging(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + source = root / "nvx-1.0.0-test" + payload = source / "bin" / "openvmm" + payload.parent.mkdir(parents=True) + payload.write_bytes(b"original") + common.write_sha256_sums(source) + destination = root / "release.tar.gz" + destination.write_bytes(b"prior archive") + create_archive = archive.create_reproducible_release_archive + + def archive_mutated_source(source: Path, output: Path) -> None: + snapshot_payload = source / "bin" / "openvmm" + snapshot_payload.write_bytes(b"replacement") + create_archive(source, output) + snapshot_payload.write_bytes(b"original") + + with ( + patch.object( + release, + "create_reproducible_release_archive", + side_effect=archive_mutated_source, + ), + self.assertRaisesRegex(common.ScriptError, "checksum mismatch"), + ): + release.create_release_archive(source, destination) + + self.assertEqual(destination.read_bytes(), b"prior archive") + self.assertEqual(payload.read_bytes(), b"original") + self.assertEqual( + list(root.glob(".staging-*-release.tar.gz")), + [], + ) + + def test_release_snapshot_rejects_coherent_source_replacement(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + source = root / "nvx-1.0.0-test" + payload = source / "bin" / "openvmm" + payload.parent.mkdir(parents=True) + payload.write_bytes(b"original") + common.write_sha256_sums(source) + checksum_file = source / "SHA256SUMS" + original_checksum = checksum_file.read_bytes() + replacement = b"coherent replacement" + replacement_checksum = ( + f"{hashlib.sha256(replacement).hexdigest()} bin/openvmm\n" + ).encode("ascii") + destination = root / "release.tar.gz" + destination.write_bytes(b"prior archive") + copy_pinned_file = release._copy_pinned_regular_file + + def replace_between_copies( + source_root: Path, + snapshot_root: Path, + relative_name: str, + expected_sha256: str, + ) -> None: + copy_pinned_file( + source_root, + snapshot_root, + relative_name, + expected_sha256, + ) + if relative_name == "SHA256SUMS": + payload.write_bytes(replacement) + checksum_file.write_bytes(replacement_checksum) + + try: + with ( + patch.object( + release, + "_copy_pinned_regular_file", + side_effect=replace_between_copies, + ), + self.assertRaisesRegex( + common.ScriptError, + "release snapshot source changed", + ), + ): + release.create_release_archive(source, destination) + finally: + if payload.exists(): + payload.write_bytes(b"original") + if checksum_file.exists(): + checksum_file.write_bytes(original_checksum) + + self.assertEqual(destination.read_bytes(), b"prior archive") + self.assertEqual(payload.read_bytes(), b"original") + self.assertEqual(checksum_file.read_bytes(), original_checksum) + self.assertEqual(list(root.glob(".snapshot-*")), []) + self.assertEqual(list(root.glob(".staging-*-release.tar.gz")), []) + + def test_source_archives_are_reproducible(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + source = root / "source" + source.mkdir() + (source / "payload.txt").write_text("payload", encoding="ascii") + cache = source / "__pycache__" + cache.mkdir() + (cache / "ignored.pyc").write_bytes(b"cache") + first = root / "first.tar.gz" + second = root / "second.tar.gz" + + archive.create_reproducible_tar_gz(first, [(source, "bundle")]) + os.utime(source / "payload.txt", (1000, 1000)) + archive.create_reproducible_tar_gz(second, [(source, "bundle")]) + + self.assertEqual(first.read_bytes(), second.read_bytes()) + with tarfile.open(first, "r:gz") as package: + members = package.getmembers() + self.assertNotIn("bundle/__pycache__", {member.name for member in members}) + self.assertTrue(all(member.mtime == 0 for member in members)) + self.assertTrue(all(member.uid == member.gid == 0 for member in members)) + + +class DownloadTests(unittest.TestCase): + def test_rejects_nonpositive_attempts(self): + with tempfile.TemporaryDirectory() as temporary: + destination = Path(temporary) / "archive.tar.xz" + with ( + patch("nvx_tools.common.urllib.request.urlopen") as urlopen, + self.assertRaisesRegex( + common.ScriptError, "download attempts must be positive" + ), + ): + common.download( + "https://example.invalid/archive.tar.xz", + destination, + attempts=0, + ) + + urlopen.assert_not_called() + self.assertFalse(destination.exists()) + + def test_retries_checksum_mismatch(self): + payload = b"verified archive" + expected_sha256 = hashlib.sha256(payload).hexdigest() + responses = [io.BytesIO(b"corrupt archive"), io.BytesIO(payload)] + + with tempfile.TemporaryDirectory() as temporary: + destination = Path(temporary) / "archive.tar.xz" + with patch( + "nvx_tools.common.urllib.request.urlopen", side_effect=responses + ) as urlopen: + common.download( + "https://example.invalid/archive.tar.xz", + destination, + expected_sha256=expected_sha256, + ) + + self.assertEqual(destination.read_bytes(), payload) + self.assertEqual(urlopen.call_count, 2) + self.assertFalse(destination.with_name("archive.tar.xz.part").exists()) + + def test_does_not_publish_checksum_mismatch(self): + payload = b"verified archive" + expected_sha256 = hashlib.sha256(payload).hexdigest() + responses = [io.BytesIO(b"corrupt archive") for _ in range(3)] + + with tempfile.TemporaryDirectory() as temporary: + destination = Path(temporary) / "archive.tar.xz" + with ( + patch( + "nvx_tools.common.urllib.request.urlopen", + side_effect=responses, + ), + self.assertRaisesRegex(common.ScriptError, "SHA-256 is"), + ): + common.download( + "https://example.invalid/archive.tar.xz", + destination, + expected_sha256=expected_sha256, + ) + + self.assertFalse(destination.exists()) + self.assertFalse(destination.with_name("archive.tar.xz.part").exists()) + + def test_replaces_mismatched_cached_archive(self): + payload = b"verified archive" + expected_sha256 = hashlib.sha256(payload).hexdigest() + + with tempfile.TemporaryDirectory() as temporary: + destination = Path(temporary) / "archive.tar.xz" + destination.write_bytes(b"corrupt archive") + + def write_verified( + _url: str, + output: Path, + *, + expected_sha256: str, + ) -> None: + self.assertFalse(output.exists()) + self.assertEqual( + expected_sha256, + hashlib.sha256(payload).hexdigest(), + ) + output.write_bytes(payload) + + with patch.object(common, "download", side_effect=write_verified): + common.download_verified( + "https://example.invalid/archive.tar.xz", + destination, + expected_sha256, + ) + + self.assertEqual(destination.read_bytes(), payload) + + +if __name__ == "__main__": + unittest.main() From ba162d0188685b01072f144063fd68a97bdaf680 Mon Sep 17 00:00:00 2001 From: Akhil Gupta Date: Sun, 11 Oct 2026 16:09:34 -0700 Subject: [PATCH 2/5] Delete scripts/nvx_tools/release.py --- scripts/nvx_tools/release.py | 1786 ---------------------------------- 1 file changed, 1786 deletions(-) delete mode 100644 scripts/nvx_tools/release.py diff --git a/scripts/nvx_tools/release.py b/scripts/nvx_tools/release.py deleted file mode 100644 index d9a4c38a9..000000000 --- a/scripts/nvx_tools/release.py +++ /dev/null @@ -1,1786 +0,0 @@ -"""Source collection, verification, and release packaging workflows.""" - -from __future__ import annotations - -import hashlib -import json -import os -import re -import shutil -import stat -import subprocess -import sys -import tarfile -import tempfile -import urllib.error -import urllib.parse -import urllib.request -import uuid -import zipfile -from collections.abc import Callable, Sequence -from dataclasses import dataclass -from pathlib import Path, PurePosixPath -from typing import cast - -from .archive import create_reproducible_release_archive, create_reproducible_tar_gz -from .azurelinux import input_sha256 as azurelinux_input_sha256 -from .azurelinux import package_lock_sha256 as azurelinux_package_lock_sha256 -from .build import ( - assert_required_kernel_config, - build_docker_linux_source, - initramfs_provenance_inputs, - kernel_provenance_inputs, -) -from .build_config import DockerBuildConfig -from .build_constants import ( - AlpineBuildConstants, - AzureLinuxBuildConstants, - BuildConstants, - InitramfsBuildConstants, - KernelBuildConstants, - OpenVMMBuildConstants, - ReleaseBuildConstants, - UbuntuBuildConstants, -) -from .collect_alpine_sources import collect_alpine_sources -from .collect_ubuntu_sources import collect_ubuntu_sources -from .common import ( - ScriptError, - VerifiedChecksumInventory, - artifact_path, - credential_safe_opener, - download, - git_output, - openvmm_binary_path, - openvmm_git_state, - require_file, - sha256_file, - verify_sha256_sums, - write_sha256_sums, -) -from .development_release import development_release_tag -from .ubuntu import converter_input_sha256, customization_files, package_lock_sha256 - -GITHUB_API_VERSION = "2022-11-28" -# HEAD often has no release, as with CI's [skip ci] baseline commits. Each -# commit searched costs one GitHub API request. -_RELEASE_SEARCH_COMMITS = 20 - - -@dataclass(frozen=True) -class _ReleaseAsset: - tag: str - name: str - url: str - size: int - - -class _GitHubReleaseQueryError(ScriptError): - def __init__(self, status: int, message: str) -> None: - self.status = status - super().__init__(message) - - -class _ReleaseRestoreError(ScriptError): - """Raised when release publication and restoration both fail.""" - - -def create_release_archive(source: Path, destination: Path) -> None: - if source.is_symlink(): - raise ScriptError(f"release package directory must not be a symlink: {source}") - source = source.resolve() - destination = destination.resolve() - if destination == source or source in destination.parents: - raise ScriptError("release archive destination must be outside its source") - accepted_inventory = verify_sha256_sums(source) - destination.parent.mkdir(parents=True, exist_ok=True) - staging = destination.with_name(f".staging-{uuid.uuid4().hex}-{destination.name}") - snapshot_parent = Path( - tempfile.mkdtemp( - prefix=f".snapshot-{uuid.uuid4().hex}-", - dir=destination.parent, - ) - ) - snapshot_parent.chmod(0o700) - snapshot = snapshot_parent / source.name - try: - _capture_release_snapshot(source, snapshot, accepted_inventory) - create_reproducible_release_archive(snapshot, staging) - _verify_release_archive(staging, source.name, accepted_inventory) - staging.replace(destination) - finally: - staging.unlink(missing_ok=True) - shutil.rmtree(snapshot_parent, ignore_errors=True) - print(f">> archived {source} as {destination}") - - -def _github_headers(token: str | None, accept: str) -> dict[str, str]: - headers = { - "Accept": accept, - "X-GitHub-Api-Version": GITHUB_API_VERSION, - "User-Agent": "nvx-release-downloader", - } - if token is not None: - headers["Authorization"] = f"Bearer {token}" - return headers - - -def _github_error_message(error: urllib.error.HTTPError) -> str: - try: - payload = error.read().decode("utf-8", "replace").strip() - except (AttributeError, OSError, ValueError): - return "" - if not payload: - return "" - try: - document: object = json.loads(payload) - except json.JSONDecodeError: - return " ".join(payload.split())[:200] - if isinstance(document, dict): - message = cast(dict[str, object], document).get("message") - if isinstance(message, str): - return message - return "" - - -def _github_error_hint(error: urllib.error.HTTPError, token: str | None) -> str: - if token is None and error.code in (401, 403, 404): - return "set GH_TOKEN to a token that can read the repository" - if error.code == 401: - return "the configured GitHub token was rejected; refresh or replace it" - if error.code == 403: - if error.headers.get("x-ratelimit-remaining") == "0": - return "the GitHub API rate limit is exhausted; retry later" - return ( - "the configured GitHub token lacks access; grant it read access to " - "the repository contents and authorize it for organization single sign-on" - ) - if error.code == 404: - return ( - "verify --repository and that the configured GitHub token can read " - "that repository" - ) - return "" - - -def _github_json( - url: str, - token: str | None, - *, - missing_ok: bool = False, -) -> object | None: - request = urllib.request.Request( - url, - headers=_github_headers(token, "application/vnd.github+json"), - ) - try: - with credential_safe_opener().open(request) as response: - return json.load(response) - except urllib.error.HTTPError as error: - if missing_ok and error.code == 404: - return None - message = _github_error_message(error) - hint = _github_error_hint(error, token) - detail = f": {message.rstrip('.')}" if message else "" - advice = f"; {hint}" if hint else "" - raise _GitHubReleaseQueryError( - error.code, - f"GitHub release query failed with HTTP {error.code}{detail}{advice}", - ) from error - except (OSError, urllib.error.URLError) as error: - raise ScriptError(f"GitHub release query failed: {error}") from error - - -def _first_parent_commits(limit: int) -> list[str]: - try: - output = git_output( - "rev-list", "--first-parent", f"--max-count={limit}", "HEAD" - ) - except subprocess.CalledProcessError as error: - detail = str(error.stderr or "").strip() or str(error) - raise ScriptError( - f"download needs a Git checkout to find its release: {detail}" - ) from error - except (OSError, subprocess.SubprocessError) as error: - raise ScriptError( - f"download needs a Git checkout to find its release: {error}" - ) from error - return output.split() - - -def _version_at(commit: str) -> str | None: - try: - return git_output("show", f"{commit}:VERSION") or None - except subprocess.CalledProcessError: - return None - except (OSError, subprocess.SubprocessError) as error: - raise ScriptError( - f"download failed to read VERSION at {commit}: {error}" - ) from error - - -def _checkout_release_asset( - repository: str, - platform: str, - token: str | None, -) -> _ReleaseAsset: - repository_parts = repository.split("/") - if len(repository_parts) != 2 or not all(repository_parts): - raise ScriptError("GitHub repository must be OWNER/REPOSITORY") - encoded_repository = "/".join( - urllib.parse.quote(part, safe="") for part in repository_parts - ) - repository_url = f"https://api.github.com/repos/{encoded_repository}" - - extension = ".zip" if platform.startswith("windows-") else ".tar.gz" - asset_pattern = re.compile(rf"^nvx-.+-{re.escape(platform)}{re.escape(extension)}$") - commits = _first_parent_commits(_RELEASE_SEARCH_COMMITS) - for distance, commit in enumerate(commits): - version = _version_at(commit) - if version is None: - continue - tag = development_release_tag(version, commit) - release = _github_json( - f"{repository_url}/releases/tags/{urllib.parse.quote(tag, safe='')}", - token, - missing_ok=True, - ) - if release is None: - continue - if not isinstance(release, dict): - raise ScriptError("GitHub release query returned an invalid response") - release_document = cast(dict[str, object], release) - if release_document.get("draft") is True: - continue - assets = release_document.get("assets") - if not isinstance(assets, list): - continue - for asset_value in cast(list[object], assets): - if not isinstance(asset_value, dict): - continue - asset = cast(dict[str, object], asset_value) - name = asset.get("name") - asset_url = asset.get("url") - size = asset.get("size") - if ( - isinstance(name, str) - and asset_pattern.fullmatch(name) is not None - and isinstance(asset_url, str) - and isinstance(size, int) - and not isinstance(size, bool) - ): - if distance: - plural = "s" if distance > 1 else "" - print( - f">> no {platform} release for HEAD; using {tag}, " - f"{distance} first-parent commit{plural} earlier" - ) - return _ReleaseAsset(tag, name, asset_url, size) - # GitHub also answers 404 for a repository that the caller cannot read, so - # query it to report that case with its usual hint. - _github_json(repository_url, token) - raise ScriptError( - f"no {repository} release with an NVX package for {platform} was built " - f"from HEAD or its first-parent ancestors ({len(commits)} commits " - "searched); check out a released commit or build the artifacts" - ) - - -def _checkout_release_asset_with_fallback( - repository: str, - platform: str, - token: str | None, -) -> tuple[_ReleaseAsset, str | None]: - if token is None: - return _checkout_release_asset(repository, platform, None), None - - try: - return _checkout_release_asset(repository, platform, token), token - except _GitHubReleaseQueryError as authenticated_error: - if authenticated_error.status not in (401, 403): - raise - print( - f">> {authenticated_error}; retrying without credentials", - file=sys.stderr, - ) - try: - return _checkout_release_asset(repository, platform, None), None - except _GitHubReleaseQueryError as public_error: - raise authenticated_error from public_error - - -def _same_file_identity(first: os.stat_result, second: os.stat_result) -> bool: - return first.st_dev == second.st_dev and first.st_ino == second.st_ino - - -def _require_safe_source_parents(root: Path, relative: PurePosixPath) -> None: - root_metadata = root.lstat() - if stat.S_ISLNK(root_metadata.st_mode) or not stat.S_ISDIR(root_metadata.st_mode): - raise ScriptError(f"release snapshot source is not a directory: {root}") - current = root - for part in relative.parts[:-1]: - current /= part - metadata = current.lstat() - if stat.S_ISLNK(metadata.st_mode) or not stat.S_ISDIR(metadata.st_mode): - raise ScriptError( - f"release snapshot source has an unsafe directory: {relative}" - ) - - -def _copy_pinned_regular_file( - source_root: Path, - snapshot_root: Path, - relative_name: str, - expected_sha256: str, -) -> None: - relative = PurePosixPath(relative_name) - _require_safe_source_parents(source_root, relative) - source = source_root.joinpath(*relative.parts) - before = source.lstat() - if stat.S_ISLNK(before.st_mode) or not stat.S_ISREG(before.st_mode): - raise ScriptError( - f"release snapshot source is not a regular file: {relative_name}" - ) - - flags = os.O_RDONLY | getattr(os, "O_BINARY", 0) - flags |= getattr(os, "O_NOFOLLOW", 0) - descriptor = os.open(source, flags) - try: - opened = os.fstat(descriptor) - after_open = source.lstat() - if ( - not stat.S_ISREG(opened.st_mode) - or not stat.S_ISREG(after_open.st_mode) - or not _same_file_identity(before, opened) - or not _same_file_identity(opened, after_open) - ): - raise ScriptError( - f"release snapshot source changed while opening: {relative_name}" - ) - - destination = snapshot_root.joinpath(*relative.parts) - destination.parent.mkdir(parents=True, exist_ok=True) - digest = hashlib.sha256() - with os.fdopen(descriptor, "rb", closefd=False) as source_file: - with destination.open("xb") as destination_file: - while chunk := source_file.read(1024 * 1024): - destination_file.write(chunk) - digest.update(chunk) - after_read = os.fstat(descriptor) - if not _same_file_identity(opened, after_read): - raise ScriptError( - f"release snapshot source changed while reading: {relative_name}" - ) - actual_sha256 = digest.hexdigest() - if actual_sha256 != expected_sha256: - raise ScriptError( - f"release snapshot source changed for {relative_name}: " - f"{actual_sha256}, expected {expected_sha256}" - ) - destination.chmod( - 0o755 if relative.parts[0] == "bin" or before.st_mode & 0o111 else 0o644 - ) - finally: - os.close(descriptor) - - -def _capture_release_snapshot( - source: Path, - snapshot: Path, - accepted: VerifiedChecksumInventory, -) -> None: - snapshot.mkdir(mode=0o700) - expected_files = dict(accepted.files) - expected_files["SHA256SUMS"] = accepted.checksum_sha256 - for relative_name, expected_sha256 in sorted(expected_files.items()): - _copy_pinned_regular_file( - source, - snapshot, - relative_name, - expected_sha256, - ) - captured = verify_sha256_sums(snapshot) - if captured != accepted: - raise ScriptError( - "captured release snapshot does not match the accepted source inventory" - ) - - -def _canonical_archive_member(name: str, *, is_directory: bool) -> str: - normalized_name = name[:-1] if is_directory and name.endswith("/") else name - path = PurePosixPath(normalized_name) - if ( - not normalized_name - or "\\" in normalized_name - or path.is_absolute() - or "." in path.parts - or ".." in path.parts - or any(":" in part for part in path.parts) - or path.as_posix() != normalized_name - or (not is_directory and name.endswith("/")) - ): - raise ScriptError(f"unsafe path in release archive: {name}") - return normalized_name - - -def _validate_archive_layout( - entries: Sequence[tuple[str, bool]], - *, - expected_root: str | None = None, -) -> tuple[str, dict[str, bool]]: - layout: dict[str, bool] = {} - for name, is_directory in entries: - canonical = _canonical_archive_member(name, is_directory=is_directory) - if canonical in layout: - raise ScriptError(f"duplicate path in release archive: {canonical}") - layout[canonical] = is_directory - roots = {PurePosixPath(name).parts[0] for name in layout} - if len(roots) != 1: - raise ScriptError("release archive must contain exactly one package root") - root = roots.pop() - if expected_root is not None and root != expected_root: - raise ScriptError(f"release archive root is {root}, expected {expected_root}") - if root in layout and layout[root] is not True: - raise ScriptError("release archive package root must be a directory") - if expected_root is not None: - if layout.get(root) is not True: - raise ScriptError("release archive omits its package root directory") - for name in layout: - parent = PurePosixPath(name).parent - while parent != PurePosixPath("."): - parent_name = parent.as_posix() - if layout.get(parent_name) is not True: - raise ScriptError( - f"release archive omits directory entry {parent_name}" - ) - parent = parent.parent - return root, layout - - -def _release_archive_layout( - archive_path: Path, - *, - expected_root: str | None = None, -) -> tuple[str, dict[str, bool]]: - try: - if archive_path.name.endswith(".tar.gz"): - with tarfile.open(archive_path, "r:gz") as archive: - entries: list[tuple[str, bool]] = [] - for member in archive.getmembers(): - if not (member.isfile() or member.isdir()): - raise ScriptError( - f"unsupported entry in release archive: {member.name}" - ) - entries.append((member.name, member.isdir())) - return _validate_archive_layout(entries, expected_root=expected_root) - if archive_path.suffix == ".zip": - with zipfile.ZipFile(archive_path) as archive: - entries = [] - for member in archive.infolist(): - is_directory = member.is_dir() - file_type = (member.external_attr >> 16) & 0o170000 - allowed_types = ( - (0, stat.S_IFDIR) if is_directory else (0, stat.S_IFREG) - ) - if file_type not in allowed_types: - raise ScriptError( - f"unsupported entry in release archive: {member.filename}" - ) - entries.append((member.filename, is_directory)) - return _validate_archive_layout(entries, expected_root=expected_root) - except (tarfile.TarError, zipfile.BadZipFile) as error: - raise ScriptError( - f"invalid release archive {archive_path.name}: {error}" - ) from error - raise ScriptError(f"unsupported release archive: {archive_path.name}") - - -def _extract_release_archive(archive_path: Path, destination: Path) -> None: - _release_archive_layout(archive_path) - try: - if archive_path.name.endswith(".tar.gz"): - with tarfile.open(archive_path, "r:gz") as archive: - for member in archive.getmembers(): - canonical = _canonical_archive_member( - member.name, - is_directory=member.isdir(), - ) - target = destination.joinpath(*PurePosixPath(canonical).parts) - if member.isdir(): - target.mkdir(parents=True, exist_ok=True) - target.chmod(member.mode & 0o777) - continue - target.parent.mkdir(parents=True, exist_ok=True) - source = archive.extractfile(member) - if source is None: - raise ScriptError( - f"could not read release archive entry: {member.name}" - ) - with source, target.open("xb") as output: - shutil.copyfileobj(source, output, length=1024 * 1024) - target.chmod(member.mode & 0o777) - return - if archive_path.suffix == ".zip": - with zipfile.ZipFile(archive_path) as archive: - for member in archive.infolist(): - is_directory = member.is_dir() - canonical = _canonical_archive_member( - member.filename, - is_directory=is_directory, - ) - target = destination.joinpath(*PurePosixPath(canonical).parts) - mode = (member.external_attr >> 16) & 0o777 - if is_directory: - target.mkdir(parents=True, exist_ok=True) - if mode: - target.chmod(mode) - continue - target.parent.mkdir(parents=True, exist_ok=True) - with archive.open(member) as source, target.open("xb") as output: - shutil.copyfileobj(source, output, length=1024 * 1024) - if mode: - target.chmod(mode) - return - except (tarfile.TarError, zipfile.BadZipFile) as error: - raise ScriptError( - f"invalid release archive {archive_path.name}: {error}" - ) from error - raise ScriptError(f"unsupported release archive: {archive_path.name}") - - -def _filesystem_archive_layout(destination: Path) -> dict[str, bool]: - layout: dict[str, bool] = {} - for path in destination.rglob("*"): - metadata = path.lstat() - relative = path.relative_to(destination).as_posix() - if stat.S_ISLNK(metadata.st_mode): - raise ScriptError(f"release archive extracted a symlink: {relative}") - if stat.S_ISDIR(metadata.st_mode): - layout[relative] = True - elif stat.S_ISREG(metadata.st_mode): - layout[relative] = False - else: - raise ScriptError(f"release archive extracted a special file: {relative}") - return layout - - -def _verify_release_archive( - archive_path: Path, - expected_root: str, - accepted: VerifiedChecksumInventory, -) -> None: - root, archived_layout = _release_archive_layout( - archive_path, - expected_root=expected_root, - ) - extraction = Path( - tempfile.mkdtemp( - prefix=f".{archive_path.name}.verify-", - dir=archive_path.parent, - ) - ) - try: - _extract_release_archive(archive_path, extraction) - extracted_layout = _filesystem_archive_layout(extraction) - if extracted_layout != archived_layout: - raise ScriptError( - "release archive extraction did not preserve its exact member layout" - ) - archived = verify_sha256_sums(extraction / root) - if archived != accepted: - raise ScriptError( - "release archive does not match the accepted source inventory" - ) - finally: - shutil.rmtree(extraction, ignore_errors=True) - - -def _replace_runtime_file(source: Path, destination: Path) -> None: - destination.parent.mkdir(parents=True, exist_ok=True) - temporary = destination.with_name(f"{destination.name}.part") - temporary.unlink(missing_ok=True) - try: - shutil.copy2(source, temporary) - temporary.replace(destination) - finally: - temporary.unlink(missing_ok=True) - - -def _packaged_guest_artifact_names(package_root: Path) -> list[str]: - source_manifest = package_root / "SOURCE-MANIFEST.json" - if not source_manifest.exists(): - return list(ReleaseBuildConstants.GUEST_ARTIFACT_NAMES) - manifest = _read_json_object(source_manifest, "packaged source manifest") - names: list[str] = [] - if isinstance(manifest.get("linux"), dict): - names.extend( - ( - KernelBuildConstants.BINARY_NAME, - KernelBuildConstants.CONFIG_NAME, - ) - ) - if isinstance(manifest.get("alpine"), dict): - names.extend( - ( - AlpineBuildConstants.INITRAMFS_NAME, - AlpineBuildConstants.PACKAGE_MANIFEST_NAME, - ) - ) - if isinstance(manifest.get("ubuntu"), dict): - names.extend( - ( - UbuntuBuildConstants.INITRAMFS_NAME, - UbuntuBuildConstants.PACKAGE_MANIFEST_NAME, - UbuntuBuildConstants.DISTRO_NAME, - UbuntuBuildConstants.DISTRO_MANIFEST_NAME, - ) - ) - if isinstance(manifest.get("azurelinux"), dict): - names.extend(ReleaseBuildConstants.AZURELINUX_ARTIFACT_NAMES) - if not names: - raise ScriptError("packaged source manifest does not declare guest artifacts") - return names - - -def _install_release_archive(archive_path: Path) -> None: - with tempfile.TemporaryDirectory(prefix="nvx-release-") as temporary: - extraction_root = Path(temporary) - _extract_release_archive(archive_path, extraction_root) - checksum_files = list(extraction_root.glob("*/SHA256SUMS")) - if len(checksum_files) != 1: - raise ScriptError( - "release archive must contain one package root with SHA256SUMS" - ) - package_root = checksum_files[0].parent - verify_sha256_sums(package_root) - - binary_destination = openvmm_binary_path() - binary_source = require_file( - package_root / "bin" / binary_destination.name, - "packaged OpenVMM binary", - ) - guest_sources = { - name: require_file( - package_root / "guest" / name, - f"packaged guest artifact {name}", - ) - for name in _packaged_guest_artifact_names(package_root) - } - provenance_sources = { - name: require_file( - package_root / "provenance" / name, - f"packaged provenance artifact {name}", - ) - for name in ( - OpenVMMBuildConstants.PROVENANCE_NAME, - KernelBuildConstants.PROVENANCE_NAME, - ) - } - _replace_runtime_file(binary_source, binary_destination) - for name, source in guest_sources.items(): - _replace_runtime_file(source, artifact_path(name)) - for name in ReleaseBuildConstants.GUEST_ARTIFACT_NAMES: - if name not in guest_sources: - artifact_path(name).unlink(missing_ok=True) - for name, source in provenance_sources.items(): - _replace_runtime_file(source, artifact_path(name)) - - -def download_checkout_release(repository: str, platform: str) -> None: - token = os.environ.get("GH_TOKEN") or os.environ.get("GITHUB_TOKEN") - asset, download_token = _checkout_release_asset_with_fallback( - repository, - platform, - token, - ) - print(f">> downloading {asset.name} from {asset.tag}") - with tempfile.TemporaryDirectory(prefix="nvx-download-") as temporary: - archive_path = Path(temporary) / asset.name - download( - asset.url, - archive_path, - headers=_github_headers(download_token, "application/octet-stream"), - opener=credential_safe_opener(), - ) - actual_size = archive_path.stat().st_size - if actual_size != asset.size: - raise ScriptError( - f"downloaded {asset.name} is {actual_size} bytes, expected {asset.size}" - ) - _install_release_archive(archive_path) - print(f">> installed {asset.tag} for {platform}") - - -def _copy_release_file(source: Path, destination: Path) -> None: - require_file(source, source.name) - destination.parent.mkdir(parents=True, exist_ok=True) - shutil.copy2(source, destination) - - -def _validate_alpine_sources(package_manifests: list[Path]) -> frozenset[str]: - source_root = BuildConstants.SOURCE_DIR / AlpineBuildConstants.GUEST_NAME - source_manifest_path = require_file( - source_root / "manifest.json", - "collected Alpine source manifest", - ) - source_manifest = _read_json_object( - source_manifest_path, - "collected Alpine source manifest", - ) - if source_manifest.get("format") != AlpineBuildConstants.SOURCE_MANIFEST_FORMAT: - raise ScriptError( - "collected Alpine sources use an unsupported manifest format; " - "run collect-sources again" - ) - recorded_executables = source_manifest.get("executables") - if not isinstance(recorded_executables, list) or not all( - isinstance(path, str) for path in cast(list[object], recorded_executables) - ): - raise ScriptError( - "collected Alpine source manifest has an invalid executables list" - ) - executables = frozenset(cast(list[str], recorded_executables)) - source_packages = cast(list[dict[str, str]], source_manifest["packages"]) - collected = { - (package["package"], package["version"], package["commit"]): package - for package in source_packages - } - missing: list[str] = [] - for manifest_path in package_manifests: - package_manifest = json.loads(manifest_path.read_text(encoding="utf-8")) - for package in package_manifest["packages"]: - key = ( - package["name"], - package["version"], - package["aports_commit"], - ) - if key not in collected: - missing.append(f"{key[0]}-{key[1]} ({key[2]})") - if missing: - raise ScriptError( - "collected Alpine sources do not cover packaged APKs: " - + ", ".join(sorted(missing)) - ) - for package in collected.values(): - for field in ("recipe", "source_directory"): - path = source_root / package[field] - if not path.exists(): - raise ScriptError( - f"collected Alpine {field} is missing for " - f"{package['package']}: {path}" - ) - checksummed = dict(verify_sha256_sums(source_root).files) - for path in sorted(executables): - if path not in checksummed or not path.startswith("recipes/"): - raise ScriptError( - f"collected Alpine executable is not a checksummed recipe file: {path}" - ) - return executables - - -def _validate_ubuntu_sources(package_manifests: list[Path]) -> None: - source_root = BuildConstants.SOURCE_DIR / UbuntuBuildConstants.GUEST_NAME - source_manifest_path = require_file( - source_root / "manifest.json", - "collected Ubuntu source manifest", - ) - source_manifest = json.loads(source_manifest_path.read_text(encoding="utf-8")) - collected = { - (package["source_name"], package["source_version"]): package - for package in source_manifest["packages"] - } - missing: list[str] = [] - for manifest_path in package_manifests: - package_manifest = json.loads(manifest_path.read_text(encoding="utf-8")) - for package in package_manifest["packages"]: - key = (package["source_name"], package["source_version"]) - if key not in collected: - missing.append(f"{key[0]}={key[1]}") - if missing: - raise ScriptError( - "collected Ubuntu sources do not cover packaged binaries: " - + ", ".join(sorted(set(missing))) - ) - for package in collected.values(): - dsc = source_root / package["dsc"] - require_file(dsc, f"Ubuntu source metadata for {package['source_name']}") - for source_file in package["files"]: - require_file( - source_root / source_file["path"], - f"Ubuntu source member {source_file['name']}", - ) - verify_sha256_sums(source_root) - - -def _create_source_archive( - output: Path, - inputs: list[tuple[Path, str]], - *, - mode: Callable[[tarfile.TarInfo], int] | None = None, -) -> None: - print(f">> creating source archive {output}") - create_reproducible_tar_gz(output, inputs, mode=mode) - - -def _project_source_archive( - output: Path, - version: str, - package_manifests: list[Path], -) -> None: - root = f"nvx-project-source-{version}" - inputs = [ - (BuildConstants.REPO_ROOT / relative, f"{root}/{relative}") - for relative in ReleaseBuildConstants.PROJECT_SOURCE_PATHS - ] - inputs.extend( - ( - manifest, - f"{root}/build/{manifest.name}", - ) - for manifest in package_manifests - ) - inputs.append( - ( - artifact_path(KernelBuildConstants.CONFIG_NAME), - f"{root}/{BuildConstants.BUILD_DIRECTORY_NAME}/{KernelBuildConstants.CONFIG_NAME}", - ) - ) - _create_source_archive(output, inputs) - - -def _alpine_source_archive( - output: Path, - version: str, - package_manifests: list[Path], - executables: frozenset[str], -) -> None: - root = f"nvx-alpine-source-{version}" - inputs = [ - (BuildConstants.SOURCE_DIR / AlpineBuildConstants.GUEST_NAME, f"{root}/sources") - ] - inputs.extend( - ( - manifest, - f"{root}/manifests/{manifest.name}", - ) - for manifest in package_manifests - ) - executable_members = {f"{root}/sources/{path}" for path in executables} - - def mode(member: tarfile.TarInfo) -> int: - # Apply the recorded modes, which hosts such as Windows cannot store. - if member.isdir() or member.name in executable_members: - return 0o755 - return 0o644 - - _create_source_archive(output, inputs, mode=mode) - - -def _ubuntu_source_archive( - output: Path, - version: str, - package_manifests: list[Path], -) -> None: - root = f"nvx-ubuntu-source-{version}" - inputs = [ - (BuildConstants.SOURCE_DIR / UbuntuBuildConstants.GUEST_NAME, f"{root}/sources") - ] - inputs.extend( - ( - manifest, - f"{root}/manifests/{manifest.name}", - ) - for manifest in package_manifests - ) - _create_source_archive(output, inputs) - - -def _validate_linux_source_archive(path: Path) -> None: - expected_members = { - KernelBuildConstants.CONFIG_NAME: artifact_path( - KernelBuildConstants.CONFIG_NAME - ).read_bytes(), - "SOURCE-MANIFEST.json": ( - BuildConstants.REPO_ROOT / "SOURCE-MANIFEST.json" - ).read_bytes(), - } - manifest = json.loads(expected_members["SOURCE-MANIFEST.json"]) - expected_members.update( - { - patch: (BuildConstants.REPO_ROOT / patch).read_bytes() - for patch in manifest["linux"]["patches"] - } - ) - with tarfile.open(path, "r:gz") as archive: - members = archive.getmembers() - names = [member.name for member in members] - if not any( - name.endswith( - f"{KernelBuildConstants.SOURCE_NAME}/drivers/tty/hvc/hvc_xe9.c" - ) - for name in names - ): - raise ScriptError(f"{path} does not contain the patched xe9 HVC driver") - for suffix, expected in expected_members.items(): - matches = [member for member in members if member.name.endswith(suffix)] - if len(matches) != 1: - raise ScriptError( - f"{path} contains {len(matches)} members ending in {suffix}" - ) - extracted = archive.extractfile(matches[0]) - if extracted is None or extracted.read() != expected: - raise ScriptError(f"{path} has stale contents for {suffix}") - - -def _validate_azurelinux_manifest(manifest_path: Path) -> None: - manifest = _read_json_object(manifest_path, "Azure Linux initramfs manifest") - initramfs = artifact_path(AzureLinuxBuildConstants.INITRAMFS_NAME) - if ( - manifest.get("format") != AzureLinuxBuildConstants.PACKAGE_MANIFEST_VERSION - or manifest.get("guest") != AzureLinuxBuildConstants.GUEST_NAME - or manifest.get("artifact") != initramfs.name - or manifest.get("artifact_sha256") != sha256_file(initramfs) - or manifest.get("package_manifest_format") - != AzureLinuxBuildConstants.PACKAGE_MANIFEST_FORMAT - or manifest.get("image") != AzureLinuxBuildConstants.IMAGE - ): - raise ScriptError("Azure Linux initramfs manifest is invalid") - if manifest.get("input_sha256") != azurelinux_input_sha256(): - raise ScriptError( - "Azure Linux initramfs manifest does not match the current build inputs" - ) - - -def _guest_release_inputs( - *, - include_azurelinux: bool, -) -> tuple[list[str], list[Path], list[Path], list[Path]]: - guest_names = [ - name - for name in ReleaseBuildConstants.GUEST_ARTIFACT_NAMES - if include_azurelinux - or name not in ReleaseBuildConstants.AZURELINUX_ARTIFACT_NAMES - ] - for name in guest_names: - require_file(artifact_path(name), f"required guest artifact {name}") - alpine_manifests = [artifact_path(AlpineBuildConstants.PACKAGE_MANIFEST_NAME)] - ubuntu_manifests = [ - artifact_path(UbuntuBuildConstants.PACKAGE_MANIFEST_NAME), - artifact_path(UbuntuBuildConstants.DISTRO_MANIFEST_NAME), - ] - azurelinux_manifests: list[Path] = [] - if include_azurelinux: - azurelinux_manifests.append( - artifact_path(AzureLinuxBuildConstants.PACKAGE_MANIFEST_NAME) - ) - _validate_azurelinux_manifest(azurelinux_manifests[0]) - expected_input_sha256 = converter_input_sha256(customization_files()) - for artifact_name, manifest in zip( - (UbuntuBuildConstants.INITRAMFS_NAME, UbuntuBuildConstants.DISTRO_NAME), - ubuntu_manifests, - strict=True, - ): - artifact = artifact_path(artifact_name) - document = _read_json_object(manifest, f"{artifact_name} manifest") - if ( - document.get("format") != UbuntuBuildConstants.PACKAGE_MANIFEST_VERSION - or document.get("guest") != UbuntuBuildConstants.GUEST_NAME - or document.get("artifact") != artifact.name - or document.get("artifact_sha256") != sha256_file(artifact) - or document.get("input_sha256") != expected_input_sha256 - ): - raise ScriptError( - f"Ubuntu artifact manifest does not match {artifact_name}" - ) - return guest_names, alpine_manifests, ubuntu_manifests, azurelinux_manifests - - -def _read_json_object(path: Path, description: str) -> dict[str, object]: - try: - value: object = json.loads(path.read_text(encoding="utf-8")) - except (json.JSONDecodeError, OSError) as error: - raise ScriptError(f"invalid {description}: {error}") from error - if not isinstance(value, dict): - raise ScriptError(f"invalid {description}: expected a JSON object") - return cast(dict[str, object], value) - - -def _validate_openvmm_provenance( - binary: Path, - provenance_path: Path, -) -> dict[str, object]: - provenance = _read_json_object(provenance_path, "OpenVMM build provenance") - revision, source_clean = openvmm_git_state(OpenVMMBuildConstants.DIRECTORY) - if ( - provenance.get("format") != OpenVMMBuildConstants.PROVENANCE_FORMAT - or provenance.get("source_revision") != revision - or provenance.get("source_clean") is not True - or not source_clean - or provenance.get("executable_sha256") != sha256_file(binary) - ): - raise ScriptError( - "OpenVMM build provenance does not match the current clean pinned " - "source and executable" - ) - return provenance - - -def _validate_kernel_provenance( - kernel: Path, - kernel_config: Path, - provenance_path: Path, -) -> dict[str, object]: - provenance = _read_json_object(provenance_path, "kernel build provenance") - expected_inputs = kernel_provenance_inputs() - if ( - provenance.get("format") != KernelBuildConstants.PROVENANCE_FORMAT - or provenance.get("source") != expected_inputs["source"] - or provenance.get("input_config") != expected_inputs["input_config"] - or provenance.get("kernel_sha256") != sha256_file(kernel) - or provenance.get("config_sha256") != sha256_file(kernel_config) - ): - raise ScriptError( - "kernel build provenance does not match the current source, config, " - "and vmlinux" - ) - assert_required_kernel_config(kernel_config) - return provenance - - -def _validate_initramfs_provenance( - initramfs: Path, - package_manifest: Path, - provenance_path: Path, -) -> dict[str, object]: - provenance = _read_json_object( - provenance_path, - "initramfs build provenance", - ) - if ( - provenance.get("format") != InitramfsBuildConstants.PROVENANCE_FORMAT - or provenance.get("inputs") != initramfs_provenance_inputs() - or provenance.get("initramfs_sha256") != sha256_file(initramfs) - or provenance.get("package_manifest_sha256") != sha256_file(package_manifest) - ): - raise ScriptError( - "initramfs build provenance does not match the current source, " - "package manifest, and initramfs" - ) - return provenance - - -def _require_runtime_provenance_paths() -> tuple[Path, Path, Path]: - return ( - require_file( - artifact_path(OpenVMMBuildConstants.PROVENANCE_NAME), - "OpenVMM build provenance", - ), - require_file( - artifact_path(KernelBuildConstants.PROVENANCE_NAME), - "kernel build provenance", - ), - require_file( - artifact_path(InitramfsBuildConstants.PROVENANCE_NAME), - "initramfs build provenance", - ), - ) - - -def validate_runtime_artifact_provenance() -> None: - binary = require_file(openvmm_binary_path(), "OpenVMM release binary") - kernel = require_file( - artifact_path(KernelBuildConstants.BINARY_NAME), - "required guest artifact vmlinux", - ) - initramfs = require_file( - artifact_path(AlpineBuildConstants.INITRAMFS_NAME), - "required guest artifact initramfs.cpio.gz", - ) - package_manifest = require_file( - artifact_path(AlpineBuildConstants.PACKAGE_MANIFEST_NAME), - "initramfs package manifest", - ) - kernel_config = require_file( - artifact_path(KernelBuildConstants.CONFIG_NAME), - "required guest artifact vmlinux.config", - ) - ( - openvmm_provenance_path, - kernel_provenance_path, - initramfs_provenance_path, - ) = _require_runtime_provenance_paths() - _validate_openvmm_provenance(binary, openvmm_provenance_path) - _validate_kernel_provenance(kernel, kernel_config, kernel_provenance_path) - _validate_initramfs_provenance( - initramfs, - package_manifest, - initramfs_provenance_path, - ) - - -def _packaged_source_manifest( - root_manifest: dict[str, object], - release_version: str, - release_root: Path, - openvmm_provenance: dict[str, object], - binary_name: str, - *, - include_azurelinux: bool, -) -> bytes: - distribution = root_manifest.get("distribution") - openvmm = root_manifest.get("openvmm") - linux = root_manifest.get("linux") - alpine = root_manifest.get("alpine") - ubuntu = root_manifest.get("ubuntu") - azurelinux = root_manifest.get("azurelinux") - if not all( - isinstance(section, dict) - for section in (distribution, openvmm, linux, alpine, ubuntu, azurelinux) - ): - raise ScriptError("SOURCE-MANIFEST.json is missing a required object") - distribution_section = cast(dict[str, object], distribution) - openvmm_section = cast(dict[str, object], openvmm) - linux_section = cast(dict[str, object], linux) - alpine_section = cast(dict[str, object], alpine) - ubuntu_section = cast(dict[str, object], ubuntu) - distribution_section["version"] = release_version - openvmm_section["source_revision"] = openvmm_provenance["source_revision"] - openvmm_section["executable_sha256"] = sha256_file( - release_root / "bin" / binary_name - ) - linux_section["kernel_sha256"] = sha256_file( - release_root / "guest" / KernelBuildConstants.BINARY_NAME - ) - linux_section["config_sha256"] = sha256_file( - release_root / "guest" / KernelBuildConstants.CONFIG_NAME - ) - alpine_section["initramfs_sha256"] = sha256_file( - release_root / "guest" / AlpineBuildConstants.INITRAMFS_NAME - ) - alpine_section["initramfs_package_manifest_sha256"] = sha256_file( - release_root / "guest" / AlpineBuildConstants.PACKAGE_MANIFEST_NAME - ) - ubuntu_section["initramfs_sha256"] = sha256_file( - release_root / "guest" / UbuntuBuildConstants.INITRAMFS_NAME - ) - ubuntu_section["initramfs_package_manifest_sha256"] = sha256_file( - release_root / "guest" / UbuntuBuildConstants.PACKAGE_MANIFEST_NAME - ) - ubuntu_section["distro_layer_sha256"] = sha256_file( - release_root / "guest" / UbuntuBuildConstants.DISTRO_NAME - ) - ubuntu_section["distro_layer_manifest_sha256"] = sha256_file( - release_root / "guest" / UbuntuBuildConstants.DISTRO_MANIFEST_NAME - ) - if include_azurelinux: - azurelinux_section = cast(dict[str, object], azurelinux) - azurelinux_section["initramfs_sha256"] = sha256_file( - release_root / "guest" / AzureLinuxBuildConstants.INITRAMFS_NAME - ) - azurelinux_section["initramfs_package_manifest_sha256"] = sha256_file( - release_root / "guest" / AzureLinuxBuildConstants.PACKAGE_MANIFEST_NAME - ) - else: - del root_manifest["azurelinux"] - return (json.dumps(root_manifest, indent=2, sort_keys=True) + "\n").encode("utf-8") - - -def _validate_root_manifest_contract(manifest: dict[str, object]) -> None: - distribution = manifest.get("distribution") - openvmm = manifest.get("openvmm") - if not isinstance(distribution, dict): - raise ScriptError("SOURCE-MANIFEST.json distribution must identify nvx") - distribution_section = cast(dict[str, object], distribution) - if distribution_section.get("name") != "nvx": - raise ScriptError("SOURCE-MANIFEST.json distribution must identify nvx") - expected_openvmm = { - "microvm_abi_version": OpenVMMBuildConstants.MICROVM_ABI_VERSION, - "control_session_protocol_version": OpenVMMBuildConstants.CONTROL_SESSION_PROTOCOL_VERSION, - "control_contract_revision": OpenVMMBuildConstants.CONTROL_CONTRACT_REVISION, - } - if not isinstance(openvmm, dict): - raise ScriptError( - "SOURCE-MANIFEST.json OpenVMM contract does not match the build contract" - ) - openvmm_section = cast(dict[str, object], openvmm) - for field, expected in expected_openvmm.items(): - if openvmm_section.get(field) != expected: - raise ScriptError( - "SOURCE-MANIFEST.json OpenVMM contract does not match the " - "build contract" - ) - if "source_revision" in openvmm_section: - raise ScriptError( - "root SOURCE-MANIFEST.json must not duplicate the OpenVMM gitlink revision" - ) - if "guest_agent" in manifest: - raise ScriptError( - "root SOURCE-MANIFEST.json must not contain product guest-agent metadata" - ) - - -def _validate_source_manifest_metadata( - manifest: dict[str, object], - kernel_inputs: dict[str, object], -) -> list[str]: - _validate_root_manifest_contract(manifest) - if manifest.get("format") != ReleaseBuildConstants.SOURCE_MANIFEST_FORMAT: - raise ScriptError("SOURCE-MANIFEST.json format must be 1") - linux_value = manifest.get("linux") - alpine_value = manifest.get("alpine") - ubuntu_value = manifest.get("ubuntu") - azurelinux_value = manifest.get("azurelinux") - source_value = kernel_inputs.get("source") - config_value = kernel_inputs.get("input_config") - if not all( - isinstance(section, dict) - for section in ( - linux_value, - alpine_value, - ubuntu_value, - azurelinux_value, - source_value, - config_value, - ) - ): - raise ScriptError("SOURCE-MANIFEST.json is missing source metadata") - linux = cast(dict[str, object], linux_value) - alpine = cast(dict[str, object], alpine_value) - ubuntu = cast(dict[str, object], ubuntu_value) - azurelinux = cast(dict[str, object], azurelinux_value) - source = cast(dict[str, object], source_value) - input_config = cast(dict[str, object], config_value) - source_patches = source.get("patches") - if not isinstance(source_patches, list): - raise ScriptError("kernel provenance source patches must be a list") - patch_paths: list[str] = [] - for patch_value in cast(list[object], source_patches): - if not isinstance(patch_value, dict): - raise ScriptError("kernel provenance source patch must be an object") - patch = cast(dict[str, object], patch_value) - path = patch.get("path") - if not isinstance(path, str): - raise ScriptError("kernel provenance source patch path must be a string") - patch_paths.append(path) - expected_linux: dict[str, object] = { - "version": KernelBuildConstants.VERSION, - "upstream_url": KernelBuildConstants.URL, - "upstream_archive_sha256": KernelBuildConstants.SHA256, - "source_cache": ( - Path(BuildConstants.CACHE_DIRECTORY_NAME) - / KernelBuildConstants.SOURCE_DIRECTORY_NAME - / KernelBuildConstants.SOURCE_NAME - ).as_posix(), - "source_archive": ( - BuildConstants.SOURCE_RELATIVE_DIRECTORY - / KernelBuildConstants.SOURCE_DIRECTORY_NAME - / KernelBuildConstants.SOURCE_ARCHIVE_NAME - ).as_posix(), - "generated_final_config": ( - Path(BuildConstants.BUILD_DIRECTORY_NAME) / KernelBuildConstants.CONFIG_NAME - ).as_posix(), - "input_config": input_config.get("path"), - "patches": patch_paths, - } - for field, expected in expected_linux.items(): - if linux.get(field) != expected: - raise ScriptError( - f"SOURCE-MANIFEST.json Linux {field} does not match the build pin" - ) - expected_alpine: dict[str, object] = { - "version": AlpineBuildConstants.VERSION, - "branch": AlpineBuildConstants.BRANCH, - "architecture": AlpineBuildConstants.ARCHITECTURE, - "minirootfs_url": AlpineBuildConstants.MINIROOTFS_URL, - "minirootfs_sha256": AlpineBuildConstants.MINIROOTFS_SHA256, - "guest_sources": [ - path.as_posix() for path in AlpineBuildConstants.GUEST_SOURCE_DIRECTORIES - ], - "package_manifests": [ - ( - Path(BuildConstants.BUILD_DIRECTORY_NAME) - / AlpineBuildConstants.PACKAGE_MANIFEST_NAME - ).as_posix() - ], - "source_output": ( - BuildConstants.SOURCE_RELATIVE_DIRECTORY / AlpineBuildConstants.GUEST_NAME - ).as_posix(), - } - for field, expected in expected_alpine.items(): - if alpine.get(field) != expected: - raise ScriptError( - f"SOURCE-MANIFEST.json Alpine {field} does not match the build pin" - ) - expected_ubuntu: dict[str, object] = { - "distribution": UbuntuBuildConstants.DISTRIBUTION, - "version": UbuntuBuildConstants.VERSION, - "codename": UbuntuBuildConstants.CODENAME, - "architecture": UbuntuBuildConstants.ARCHITECTURE, - "base_url": UbuntuBuildConstants.BASE_URL, - "base_sha256": UbuntuBuildConstants.BASE_SHA256, - "archive_keyring_url": UbuntuBuildConstants.ARCHIVE_KEYRING_URL, - "archive_keyring_sha256": UbuntuBuildConstants.ARCHIVE_KEYRING_SHA256, - "package_lock": UbuntuBuildConstants.PACKAGE_LOCK_RELATIVE_PATH.as_posix(), - "package_lock_sha256": package_lock_sha256(), - "guest_sources": [ - *( - path.as_posix() - for path in UbuntuBuildConstants.GUEST_SOURCE_DIRECTORIES - ), - UbuntuBuildConstants.PACKAGE_LOCK_RELATIVE_PATH.as_posix(), - ], - "package_manifests": [ - ( - Path(BuildConstants.BUILD_DIRECTORY_NAME) - / UbuntuBuildConstants.PACKAGE_MANIFEST_NAME - ).as_posix(), - ( - Path(BuildConstants.BUILD_DIRECTORY_NAME) - / UbuntuBuildConstants.DISTRO_MANIFEST_NAME - ).as_posix(), - ], - "source_output": ( - BuildConstants.SOURCE_RELATIVE_DIRECTORY / UbuntuBuildConstants.GUEST_NAME - ).as_posix(), - "erofs_converter_format": UbuntuBuildConstants.EROFS_FORMAT, - } - for field, expected in expected_ubuntu.items(): - if ubuntu.get(field) != expected: - raise ScriptError( - f"SOURCE-MANIFEST.json Ubuntu {field} does not match the build pin" - ) - expected_azurelinux: dict[str, object] = { - "distribution": AzureLinuxBuildConstants.DISTRIBUTION, - "version": AzureLinuxBuildConstants.VERSION, - "architecture": AzureLinuxBuildConstants.ARCHITECTURE, - "image": AzureLinuxBuildConstants.IMAGE, - "package_lock": AzureLinuxBuildConstants.PACKAGE_LOCK_RELATIVE_PATH.as_posix(), - "package_lock_sha256": azurelinux_package_lock_sha256(), - "guest_sources": [ - *( - path.as_posix() - for path in AzureLinuxBuildConstants.GUEST_SOURCE_DIRECTORIES - ), - AzureLinuxBuildConstants.PACKAGE_LOCK_RELATIVE_PATH.as_posix(), - ], - "package_manifests": [ - ( - Path(BuildConstants.BUILD_DIRECTORY_NAME) - / AzureLinuxBuildConstants.PACKAGE_MANIFEST_NAME - ).as_posix() - ], - } - for field, expected in expected_azurelinux.items(): - if azurelinux.get(field) != expected: - raise ScriptError( - f"SOURCE-MANIFEST.json Azure Linux {field} does not match the build pin" - ) - return patch_paths - - -def _validate_release_replacement(destination: Path, force: bool) -> None: - if not destination.exists(): - return - if not force: - raise ScriptError( - f"release directory already exists: {destination}; " - "pass --force to replace it" - ) - release_root = ( - BuildConstants.REPO_ROOT / ReleaseBuildConstants.DIRECTORY_NAME - ).resolve() - if destination == release_root or release_root not in destination.parents: - raise ScriptError("--force may only replace a version directory below dist/") - - -def _publish_release_directory( - staging: Path, - destination: Path, - *, - force: bool, -) -> None: - backup = destination.with_name(f".{destination.name}.backup-{uuid.uuid4().hex}") - moved_prior = False - try: - _validate_release_replacement(destination, force) - if destination.exists(): - destination.replace(backup) - moved_prior = True - staging.replace(destination) - except ScriptError: - shutil.rmtree(staging) - raise - except OSError as publish_error: - if moved_prior: - try: - backup.replace(destination) - except OSError as restore_error: - raise _ReleaseRestoreError( - "failed to publish the staged release and restore the prior " - f"release; prior release remains at {backup} and staged " - f"release remains at {staging}: {restore_error}" - ) from publish_error - shutil.rmtree(staging, ignore_errors=True) - raise ScriptError(f"failed to publish staged release: {publish_error}") from ( - publish_error - ) - if moved_prior: - shutil.rmtree(backup) - - -def collect_release_sources(config: DockerBuildConfig) -> None: - _guest_names, alpine_manifests, ubuntu_manifests, _azurelinux_manifests = ( - _guest_release_inputs(include_azurelinux=False) - ) - collect_alpine_sources( - alpine_manifests, - BuildConstants.SOURCE_DIR / AlpineBuildConstants.GUEST_NAME, - BuildConstants.REPO_ROOT - / BuildConstants.CACHE_DIRECTORY_NAME - / AlpineBuildConstants.APORTS_CACHE_DIRECTORY_NAME, - ) - collect_ubuntu_sources( - ubuntu_manifests, - BuildConstants.SOURCE_DIR / UbuntuBuildConstants.GUEST_NAME, - BuildConstants.REPO_ROOT - / BuildConstants.CACHE_DIRECTORY_NAME - / UbuntuBuildConstants.SOURCE_CACHE_DIRECTORY_NAME, - ) - build_docker_linux_source(config) - print( - "!! Azure Linux corresponding source is not collected: source-inclusive " - "packages omit the Azure Linux guest", - file=sys.stderr, - ) - print(f">> collected release sources under {BuildConstants.SOURCE_DIR}") - - -def package_release( - *, - version: str | None, - destination: Path | None, - include_source: bool, - force: bool, -) -> None: - guest_names, alpine_manifests, ubuntu_manifests, azurelinux_manifests = ( - _guest_release_inputs(include_azurelinux=not include_source) - ) - linux_source_archive = ( - BuildConstants.SOURCE_DIR - / KernelBuildConstants.SOURCE_DIRECTORY_NAME - / KernelBuildConstants.SOURCE_ARCHIVE_NAME - ) - alpine_executables: frozenset[str] = frozenset() - if include_source: - alpine_executables = _validate_alpine_sources(alpine_manifests) - _validate_ubuntu_sources(ubuntu_manifests) - require_file(linux_source_archive, "Linux corresponding-source archive") - _validate_linux_source_archive(linux_source_archive) - print( - "!! source-inclusive package omits the Azure Linux guest: " - "collect-sources does not collect its corresponding source", - file=sys.stderr, - ) - else: - print( - "!! binary-only package: publish matching Linux, Alpine, Ubuntu, and " - "Azure Linux corresponding source separately", - file=sys.stderr, - ) - package_manifests = [ - *alpine_manifests, - *ubuntu_manifests, - *azurelinux_manifests, - ] - - release_version = ( - version - or (BuildConstants.REPO_ROOT / "VERSION").read_text(encoding="ascii").strip() - ) - release_destination = ( - destination - or BuildConstants.REPO_ROOT - / ReleaseBuildConstants.DIRECTORY_NAME - / release_version - ).resolve() - _validate_release_replacement(release_destination, force) - binary = require_file(openvmm_binary_path(), "OpenVMM release binary") - kernel = require_file( - artifact_path(KernelBuildConstants.BINARY_NAME), - "required guest artifact vmlinux", - ) - kernel_config = require_file( - artifact_path(KernelBuildConstants.CONFIG_NAME), - "required guest artifact vmlinux.config", - ) - initramfs = require_file( - artifact_path(AlpineBuildConstants.INITRAMFS_NAME), - "required guest artifact initramfs.cpio.gz", - ) - package_manifest = require_file( - artifact_path(AlpineBuildConstants.PACKAGE_MANIFEST_NAME), - "initramfs package manifest", - ) - ( - openvmm_provenance_path, - kernel_provenance_path, - initramfs_provenance_path, - ) = _require_runtime_provenance_paths() - openvmm_provenance = _validate_openvmm_provenance( - binary, - openvmm_provenance_path, - ) - kernel_provenance = _validate_kernel_provenance( - kernel, - kernel_config, - kernel_provenance_path, - ) - initramfs_provenance = _validate_initramfs_provenance( - initramfs, - package_manifest, - initramfs_provenance_path, - ) - root_manifest_path = require_file( - BuildConstants.REPO_ROOT / "SOURCE-MANIFEST.json", - "source manifest", - ) - root_manifest = _read_json_object(root_manifest_path, "source manifest") - _validate_source_manifest_metadata(root_manifest, kernel_provenance) - for name in ("LICENSE", "README.md", "THIRD_PARTY_NOTICES.md"): - require_file(BuildConstants.REPO_ROOT / name, name) - require_file(OpenVMMBuildConstants.DIRECTORY / "LICENSE", "OpenVMM license") - require_file( - BuildConstants.REPO_ROOT / "kernel" / "COPYING-LINUX", "Linux copyright notice" - ) - - release_destination.parent.mkdir(parents=True, exist_ok=True) - staging = Path( - tempfile.mkdtemp( - prefix=f".{release_destination.name}.staging-", - dir=release_destination.parent, - ) - ) - preserve_staging = False - try: - _copy_release_file(binary, staging / "bin" / binary.name) - for name in guest_names: - _copy_release_file(artifact_path(name), staging / "guest" / name) - _copy_release_file( - openvmm_provenance_path, - staging / "provenance" / OpenVMMBuildConstants.PROVENANCE_NAME, - ) - _copy_release_file( - kernel_provenance_path, - staging / "provenance" / KernelBuildConstants.PROVENANCE_NAME, - ) - _copy_release_file( - initramfs_provenance_path, - staging / "provenance" / InitramfsBuildConstants.PROVENANCE_NAME, - ) - for name in ("LICENSE", "README.md", "THIRD_PARTY_NOTICES.md"): - _copy_release_file(BuildConstants.REPO_ROOT / name, staging / name) - _copy_release_file( - OpenVMMBuildConstants.DIRECTORY / "LICENSE", - staging / "licenses" / "LICENSE-OPENVMM", - ) - _copy_release_file( - BuildConstants.REPO_ROOT / "kernel" / "COPYING-LINUX", - staging / "licenses" / "COPYING-LINUX", - ) - if include_source: - source_destination = staging / "source" - _copy_release_file( - linux_source_archive, - source_destination / linux_source_archive.name, - ) - _project_source_archive( - source_destination / f"nvx-project-source-{release_version}.tar.gz", - release_version, - package_manifests, - ) - _alpine_source_archive( - source_destination / f"nvx-alpine-source-{release_version}.tar.gz", - release_version, - alpine_manifests, - alpine_executables, - ) - _ubuntu_source_archive( - source_destination / f"nvx-ubuntu-source-{release_version}.tar.gz", - release_version, - ubuntu_manifests, - ) - packaged_binary = staging / "bin" / binary.name - packaged_kernel = staging / "guest" / KernelBuildConstants.BINARY_NAME - packaged_config = staging / "guest" / KernelBuildConstants.CONFIG_NAME - packaged_initramfs = staging / "guest" / AlpineBuildConstants.INITRAMFS_NAME - packaged_package_manifest = ( - staging / "guest" / AlpineBuildConstants.PACKAGE_MANIFEST_NAME - ) - packaged_openvmm_provenance = ( - staging / "provenance" / OpenVMMBuildConstants.PROVENANCE_NAME - ) - packaged_kernel_provenance = ( - staging / "provenance" / KernelBuildConstants.PROVENANCE_NAME - ) - packaged_initramfs_provenance = ( - staging / "provenance" / InitramfsBuildConstants.PROVENANCE_NAME - ) - if sha256_file(packaged_binary) != openvmm_provenance["executable_sha256"]: - raise ScriptError( - "packaged OpenVMM executable does not match its build provenance" - ) - if sha256_file(packaged_kernel) != kernel_provenance["kernel_sha256"]: - raise ScriptError("packaged kernel does not match its build provenance") - if sha256_file(packaged_config) != kernel_provenance["config_sha256"]: - raise ScriptError( - "packaged kernel config does not match its build provenance" - ) - if ( - sha256_file(packaged_initramfs) != initramfs_provenance["initramfs_sha256"] - or sha256_file(packaged_package_manifest) - != initramfs_provenance["package_manifest_sha256"] - ): - raise ScriptError( - "packaged initramfs artifacts do not match their build provenance" - ) - if ( - _read_json_object( - packaged_openvmm_provenance, - "packaged OpenVMM build provenance", - ) - != openvmm_provenance - ): - raise ScriptError("packaged OpenVMM provenance changed while staging") - if ( - _read_json_object( - packaged_kernel_provenance, - "packaged kernel build provenance", - ) - != kernel_provenance - ): - raise ScriptError("packaged kernel provenance changed while staging") - if ( - _read_json_object( - packaged_initramfs_provenance, - "packaged initramfs build provenance", - ) - != initramfs_provenance - ): - raise ScriptError("packaged initramfs provenance changed while staging") - (staging / "SOURCE-MANIFEST.json").write_bytes( - _packaged_source_manifest( - root_manifest, - release_version, - staging, - openvmm_provenance, - binary.name, - include_azurelinux=not include_source, - ) - ) - write_sha256_sums(staging) - verify_sha256_sums(staging) - _publish_release_directory(staging, release_destination, force=force) - except _ReleaseRestoreError: - preserve_staging = True - raise - finally: - if staging.exists() and not preserve_staging: - shutil.rmtree(staging) - print(f">> packaged {release_destination}") - - -def verify_source_tree() -> None: - manifest = _read_json_object( - BuildConstants.REPO_ROOT / "SOURCE-MANIFEST.json", - "source manifest", - ) - patch_paths = _validate_source_manifest_metadata( - manifest, - kernel_provenance_inputs(), - ) - required = { - BuildConstants.REPO_ROOT - / "kernel" - / "config-microvm": "kernel build configuration", - BuildConstants.REPO_ROOT - / "kernel" - / "patches" - / "0001-microvm-xe9-earlycon.patch": "xe9 early console patch", - BuildConstants.REPO_ROOT - / "kernel" - / "patches" - / "0002-microvm-hvc-xe9.patch": "xe9 HVC patch", - BuildConstants.REPO_ROOT / "kernel" / "COPYING-LINUX": "Linux copyright notice", - BuildConstants.REPO_ROOT / "guest" / "common" / "init": "common guest init", - BuildConstants.REPO_ROOT - / "guest" - / "alpine" - / "nvx-container-enter": "Alpine sandbox container helper", - UbuntuBuildConstants.PACKAGE_LOCK: "Ubuntu supplemental package lock", - OpenVMMBuildConstants.DIRECTORY / "Cargo.toml": "initialized OpenVMM submodule", - } - required.update( - { - BuildConstants.REPO_ROOT / patch: f"kernel patch {patch}" - for patch in patch_paths - } - ) - for path, description in required.items(): - require_file(path, description) - forbidden = ( - BuildConstants.REPO_ROOT / "third_party" / "linux", - BuildConstants.REPO_ROOT / "third_party" / "alpine-sources", - BuildConstants.REPO_ROOT / "third_party" / "ubuntu-sources", - ) - present = [str(path) for path in forbidden if path.exists()] - if present: - raise ScriptError( - "generated third-party sources must not be checked out here: " - + ", ".join(present) - ) - config_path = BuildConstants.REPO_ROOT / "kernel" / "config-microvm" - config = config_path.read_text(encoding="utf-8") - for setting in ( - *KernelBuildConstants.REQUIRED_DIRECT_BOOT_CONFIG, - "CONFIG_HVC_XE9=y", - "CONFIG_VIRTIO_FS=y", - "CONFIG_FUSE_FS=y", - *KernelBuildConstants.REQUIRED_SANDBOX_CONFIG, - *KernelBuildConstants.REQUIRED_NETWORK_CONFIG, - *KernelBuildConstants.REQUIRED_TIME_ABI_CONFIG, - *KernelBuildConstants.REQUIRED_HARDENING_CONFIG, - ): - if setting not in config.splitlines(): - raise ScriptError(f"{config_path} is missing {setting}") - hvc_patch_text = ( - BuildConstants.REPO_ROOT / "kernel" / "patches" / "0002-microvm-hvc-xe9.patch" - ).read_text(encoding="utf-8") - for marker in ("config HVC_XE9", "hvc_xe9.o", "hvc_xe9.c"): - if marker not in hvc_patch_text: - raise ScriptError(f"xe9 HVC patch is missing {marker}") - generated_config = artifact_path(KernelBuildConstants.CONFIG_NAME) - if generated_config.is_file(): - generated = generated_config.read_text(encoding="utf-8").splitlines() - for setting in ( - *KernelBuildConstants.REQUIRED_DIRECT_BOOT_CONFIG, - "CONFIG_HVC_XE9=y", - *KernelBuildConstants.REQUIRED_SANDBOX_CONFIG, - *KernelBuildConstants.REQUIRED_NETWORK_CONFIG, - ): - if setting not in generated: - raise ScriptError(f"{generated_config} is missing {setting}") - head = subprocess.run( - ["git", "-C", OpenVMMBuildConstants.DIRECTORY, "rev-parse", "HEAD"], - check=True, - capture_output=True, - text=True, - ).stdout.strip() - expected = subprocess.run( - ["git", "-C", BuildConstants.REPO_ROOT, "rev-parse", ":openvmm"], - check=True, - capture_output=True, - text=True, - ).stdout.strip() - if head != expected: - raise ScriptError(f"OpenVMM submodule is at {head}, expected {expected}") - print(">> source tree and submodule metadata are consistent") From 4dbae53fbe2069e8dd270f1a4b0c7da8d41cef17 Mon Sep 17 00:00:00 2001 From: Akhil Gupta Date: Sun, 11 Oct 2026 16:10:58 -0700 Subject: [PATCH 3/5] Delete scripts/test_nvx_tools.py --- scripts/test_nvx_tools.py | 21500 ------------------------------------ 1 file changed, 21500 deletions(-) delete mode 100644 scripts/test_nvx_tools.py diff --git a/scripts/test_nvx_tools.py b/scripts/test_nvx_tools.py deleted file mode 100644 index c1f13d052..000000000 --- a/scripts/test_nvx_tools.py +++ /dev/null @@ -1,21500 +0,0 @@ -#!/usr/bin/env python3 -# pyright: reportPrivateUsage=false - -import argparse -import ast -import contextlib -import errno -import hashlib -import http.client -import http.server -import io -import json -import lzma -import os -import queue -import re -import select -import shlex -import shutil -import signal -import struct -import subprocess -import sys -import tarfile -import tempfile -import threading -import time -import unittest -import urllib.error -import urllib.request -import uuid -import zipfile -from collections.abc import Callable -from pathlib import Path, PurePosixPath -from typing import Any, cast -from unittest.mock import MagicMock, call, patch - -sys.path.insert(0, str(Path(__file__).parent)) -import nvx # noqa: E402 -from nvx_tools import ( # noqa: E402 - aci_edge_sandboxes_tests, - archive, - azurelinux, - benchmark, - build, - build_config, - ci, - collect_alpine_sources, - collect_ubuntu_sources, - common, - control_session, - guests, - release, - sandbox, - sandbox_lifecycle, - ubuntu, -) -from nvx_tools.build_constants import ( # noqa: E402 - AlpineBuildConstants, - AzureLinuxBuildConstants, - BuildConstants, - DockerBuildConstants, - InitramfsBuildConstants, - KernelBuildConstants, - OpenVMMBuildConstants, - ReleaseBuildConstants, - UbuntuBuildConstants, - ZstdBuildConstants, -) -from nvx_tools.time_abi import ( # noqa: E402 - CPU_GENERATIONS, - OPENVMM_CPU_PROFILES, - CpuGeneration, - CpuModel, - HostCpu, - cpu_profile_request_url, -) - - -def _workflow_job(workflow: str, job_name: str) -> str: - lines = workflow.splitlines() - start = lines.index(f" {job_name}:") - end = next( - ( - index - for index, line in enumerate(lines[start + 1 :], start + 1) - if len(line) > 2 and line[:2] == " " and not line[2].isspace() - ), - len(lines), - ) - return "\n".join(lines[start:end]) - - -def _composite_action_step(action: str, step_name: str) -> str: - lines = action.splitlines() - start = lines.index(f" - name: {step_name}") - end = next( - ( - index - for index, line in enumerate(lines[start + 1 :], start + 1) - if line.startswith(" - ") - ), - len(lines), - ) - return "\n".join(lines[start:end]) - - -def _composite_action_steps(action: str) -> list[str]: - lines = action.splitlines() - starts = [index for index, line in enumerate(lines) if line.startswith(" - ")] - return [ - "\n".join(lines[start:end]) - for start, end in zip(starts, starts[1:] + [len(lines)], strict=True) - ] - - -def _composite_action_script(action: str, step_name: str) -> str: - lines = _composite_action_step(action, step_name).splitlines() - start = lines.index(" run: |") + 1 - end = next( - ( - index - for index, line in enumerate(lines[start:], start) - if line and not line.startswith(" ") - ), - len(lines), - ) - return "\n".join(line[8:] for line in lines[start:end]) - - -def _workflow_step(workflow: str, job_name: str, step_name: str) -> str: - lines = _workflow_job(workflow, job_name).splitlines() - start = lines.index(f" - name: {step_name}") - end = next( - ( - index - for index, line in enumerate(lines[start + 1 :], start + 1) - if line.startswith(" - ") - ), - len(lines), - ) - return "\n".join(lines[start:end]) - - -def _workflow_steps(workflow: str, job_name: str) -> list[str]: - lines = _workflow_job(workflow, job_name).splitlines() - starts = [index for index, line in enumerate(lines) if line.startswith(" - ")] - return [ - "\n".join(lines[start:end]) - for start, end in zip(starts, starts[1:] + [len(lines)], strict=True) - ] - - -def _yaml_field(configuration: str, field: str) -> list[str]: - """Return the first value of a field, with one entry per literal block line.""" - lines = configuration.splitlines() - for index, line in enumerate(lines): - stripped = line.lstrip() - if not stripped.startswith(f"{field}: "): - continue - value = stripped[len(field) + 2 :].rstrip() - if value != "|": - return [value] - indent = len(line) - len(stripped) - values: list[str] = [] - for block_line in lines[index + 1 :]: - block_value = block_line.strip() - if block_value and len(block_line) - len(block_line.lstrip()) <= indent: - break - if block_value: - values.append(block_value) - return values - raise AssertionError(f"configuration has no {field} field") - - -def _yaml_scalar(configuration: str, field: str) -> str: - lines = configuration.splitlines() - for index, line in enumerate(lines): - stripped = line.lstrip() - if stripped.startswith("- "): - stripped = stripped[2:] - if not stripped.startswith(f"{field}: "): - continue - value = stripped[len(field) + 2 :].rstrip() - indent = len(line) - len(line.lstrip()) - if line.lstrip().startswith("- "): - indent += 2 - values: list[str] = [] if value in {">", ">-", "|", "|-"} else [value] - for block_line in lines[index + 1 :]: - block_value = block_line.strip() - if block_value and len(block_line) - len(block_line.lstrip()) <= indent: - break - if block_value: - values.append(block_value) - return " ".join(values) - raise AssertionError(f"configuration has no {field} field") - - -def _outer_parentheses_enclose(expression: str) -> bool: - expression = expression.strip() - if not expression.startswith("("): - return False - depth = 0 - quote: str | None = None - for index, character in enumerate(expression): - if quote is not None: - if character == quote and (index == 0 or expression[index - 1] != "\\"): - quote = None - continue - if character in {"'", '"'}: - quote = character - elif character == "(": - depth += 1 - elif character == ")": - depth -= 1 - if depth == 0 and index != len(expression) - 1: - return False - if depth < 0: - return False - return depth == 0 and quote is None - - -def _write_release_fixture( - root: Path, -) -> tuple[dict[str, Path], dict[str, object], str]: - build_dir = root / "build" - source_dir = build_dir / "sources" - openvmm_dir = root / "openvmm" - binary_name = "openvmm.exe" if os.name == "nt" else "openvmm" - binary = openvmm_dir / "target" / "release" / binary_name - revision = "0bc357bbcf3a654b63dfb51f1103c5751bf3d31f" - guest_names = ReleaseBuildConstants.GUEST_ARTIFACT_NAMES - source_files = ( - set(build._initramfs_source_files()) - | set(ubuntu.customization_files()) - | set(azurelinux.input_files()) - ) - for source in source_files: - destination = root / source.relative_to(BuildConstants.REPO_ROOT) - destination.parent.mkdir(parents=True, exist_ok=True) - shutil.copyfile(source, destination) - for path in ( - binary, - *(build_dir / name for name in guest_names), - root / "LICENSE", - root / "README.md", - root / "THIRD_PARTY_NOTICES.md", - openvmm_dir / "LICENSE", - root / "kernel" / "COPYING-LINUX", - ): - path.parent.mkdir(parents=True, exist_ok=True) - path.write_bytes(path.name.encode("ascii")) - for artifact_name, manifest_name in ( - ( - "initramfs-ubuntu.cpio.gz", - "initramfs-ubuntu.cpio.gz.packages.json", - ), - ("ubuntu-distro.erofs", "ubuntu-distro.erofs.manifest.json"), - ): - artifact = build_dir / artifact_name - input_sha256 = ubuntu.converter_input_sha256(ubuntu.customization_files()) - (build_dir / manifest_name).write_text( - json.dumps( - { - "format": 1, - "guest": "ubuntu", - "artifact": artifact.name, - "artifact_sha256": common.sha256_file(artifact), - "input_sha256": input_sha256, - "packages": [], - } - ), - encoding="utf-8", - ) - (build_dir / AzureLinuxBuildConstants.PACKAGE_MANIFEST_NAME).write_text( - json.dumps( - { - "format": AzureLinuxBuildConstants.PACKAGE_MANIFEST_VERSION, - "guest": AzureLinuxBuildConstants.GUEST_NAME, - "artifact": AzureLinuxBuildConstants.INITRAMFS_NAME, - "artifact_sha256": common.sha256_file( - build_dir / AzureLinuxBuildConstants.INITRAMFS_NAME - ), - "package_manifest_format": ( - AzureLinuxBuildConstants.PACKAGE_MANIFEST_FORMAT - ), - "image": AzureLinuxBuildConstants.IMAGE, - "input_sha256": azurelinux.input_sha256(), - } - ), - encoding="utf-8", - ) - generated_config = build_dir / "vmlinux.config" - generated_config.write_text( - "\n".join( - ( - *KernelBuildConstants.REQUIRED_DIRECT_BOOT_CONFIG, - *KernelBuildConstants.REQUIRED_VIRTIO_CONSOLE_CONFIG, - *KernelBuildConstants.REQUIRED_SHARED_STATUS_CONFIG, - *KernelBuildConstants.REQUIRED_SANDBOX_CONFIG, - *KernelBuildConstants.REQUIRED_NETWORK_CONFIG, - *KernelBuildConstants.REQUIRED_TIME_ABI_CONFIG, - *KernelBuildConstants.REQUIRED_HARDENING_CONFIG, - ) - ) - + "\n", - encoding="utf-8", - ) - manifest = { - "format": 1, - "distribution": {"name": "nvx", "version": "0.1.0"}, - "openvmm": { - "microvm_abi_version": 2, - "control_session_protocol_version": 1, - "control_contract_revision": "nvx-microvm-v2-control-v2", - }, - "linux": { - "version": "6.18.38", - "upstream_url": ( - "https://cdn.kernel.org/pub/linux/kernel/v6.x/linux-6.18.38.tar.xz" - ), - "upstream_archive_sha256": KernelBuildConstants.SHA256, - "source_cache": ".cache/linux/linux-6.18.38", - "source_archive": ("build/sources/linux/nvx-linux-source-6.18.38.tar.gz"), - "generated_final_config": "build/vmlinux.config", - "input_config": "kernel/config-microvm", - "patches": ["kernel/patches/example.patch"], - }, - "alpine": { - "version": "3.24.1", - "branch": "v3.24", - "architecture": "x86_64", - "minirootfs_url": ( - "https://dl-cdn.alpinelinux.org/alpine/v3.24/releases/" - "x86_64/alpine-minirootfs-3.24.1-x86_64.tar.gz" - ), - "minirootfs_sha256": AlpineBuildConstants.MINIROOTFS_SHA256, - "guest_sources": ["guest/common", "guest/alpine"], - "package_manifests": ["build/initramfs.cpio.gz.packages.json"], - "source_output": "build/sources/alpine", - }, - "ubuntu": { - "distribution": "Ubuntu Base", - "version": UbuntuBuildConstants.VERSION, - "codename": UbuntuBuildConstants.CODENAME, - "architecture": UbuntuBuildConstants.ARCHITECTURE, - "base_url": UbuntuBuildConstants.BASE_URL, - "base_sha256": UbuntuBuildConstants.BASE_SHA256, - "archive_keyring_url": (UbuntuBuildConstants.ARCHIVE_KEYRING_URL), - "archive_keyring_sha256": (UbuntuBuildConstants.ARCHIVE_KEYRING_SHA256), - "package_lock": "ubuntu/packages.lock.json", - "package_lock_sha256": ubuntu.package_lock_sha256(), - "guest_sources": [ - "guest/common", - "guest/ubuntu", - "ubuntu/packages.lock.json", - ], - "package_manifests": [ - "build/initramfs-ubuntu.cpio.gz.packages.json", - "build/ubuntu-distro.erofs.manifest.json", - ], - "source_output": "build/sources/ubuntu", - "erofs_converter_format": UbuntuBuildConstants.EROFS_FORMAT, - }, - "azurelinux": { - "distribution": "Azure Linux", - "version": AzureLinuxBuildConstants.VERSION, - "architecture": AzureLinuxBuildConstants.ARCHITECTURE, - "image": AzureLinuxBuildConstants.IMAGE, - "package_lock": "azurelinux/packages.lock.json", - "package_lock_sha256": azurelinux.package_lock_sha256(), - "guest_sources": [ - *( - path.as_posix() - for path in AzureLinuxBuildConstants.GUEST_SOURCE_DIRECTORIES - ), - "azurelinux/packages.lock.json", - ], - "package_manifests": [ - ( - Path(BuildConstants.BUILD_DIRECTORY_NAME) - / AzureLinuxBuildConstants.PACKAGE_MANIFEST_NAME - ).as_posix() - ], - }, - } - (root / "SOURCE-MANIFEST.json").write_text( - json.dumps(manifest), - encoding="utf-8", - ) - kernel_inputs: dict[str, object] = { - "source": { - "version": "6.18.38", - "patches": [ - { - "path": "kernel/patches/example.patch", - "sha256": "2" * 64, - } - ], - }, - "input_config": { - "path": "kernel/config-microvm", - "sha256": "1" * 64, - }, - } - (build_dir / OpenVMMBuildConstants.PROVENANCE_NAME).write_text( - json.dumps( - { - "format": 1, - "source_revision": revision, - "source_clean": True, - "executable_sha256": common.sha256_file(binary), - } - ), - encoding="utf-8", - ) - (build_dir / KernelBuildConstants.PROVENANCE_NAME).write_text( - json.dumps( - { - "format": 1, - **kernel_inputs, - "kernel_sha256": common.sha256_file(build_dir / "vmlinux"), - "config_sha256": common.sha256_file(generated_config), - } - ), - encoding="utf-8", - ) - (build_dir / InitramfsBuildConstants.PROVENANCE_NAME).write_text( - json.dumps( - { - "format": 1, - "inputs": build.initramfs_provenance_inputs(), - "initramfs_sha256": common.sha256_file(build_dir / "initramfs.cpio.gz"), - "package_manifest_sha256": common.sha256_file( - build_dir / "initramfs.cpio.gz.packages.json" - ), - } - ), - encoding="utf-8", - ) - paths = { - "build": build_dir, - "source": source_dir, - "openvmm": openvmm_dir, - "binary": binary, - } - return paths, kernel_inputs, revision - - -class CliTests(unittest.TestCase): - def test_usage_documents_every_command_and_option(self): - # doc/usage.md is the CLI reference: every command has a row in its - # command table and a section that names exactly the options of the - # command's --help usage. - option = re.compile(r"(? str: - stdout = io.StringIO() - with contextlib.redirect_stdout(stdout), self.assertRaises(SystemExit): - nvx.parse_args([*command, "--help"]) - return stdout.getvalue().split("\n\n", 1)[0] - - def subcommands(usage: str) -> list[str]: - # A subcommand list reads "{a,b} ...", which argparse may wrap - # before the dots; an option's repeated choices read - # "{a,b} [{a,b} ...]". - match = re.search(r"(? str: - match = re.search( - rf"^{'#' * level} `{re.escape(title)}`\n(.*?)(?=^#{{2,{level}}} |\Z)", - document, - re.MULTILINE | re.DOTALL, - ) - self.assertIsNotNone(match, f"doc/usage.md has no section for {title}") - return match.group(1) if match else "" - - table = document.split("## Commands\n", 1)[1].split("\n## ", 1)[0] - commands = subcommands(usage_of()) - self.assertEqual( - set(re.findall(r"^\| `([a-z0-9-]+)` \|", table, re.MULTILINE)), - set(commands), - ) - for name in commands: - usage = usage_of(name) - nested = subcommands(usage) - section(name, 3) - targets = ( - [((name, child), 4) for child in nested] if nested else [((name,), 3)] - ) - for command, level in targets: - title = " ".join(command) - with self.subTest(command=title): - self.assertEqual( - set(option.findall(section(title, level))), - set(option.findall(usage_of(*command))), - ) - - def test_guest_selection_includes_azure_linux(self): - args = nvx.parse_args(["build-initramfs", "--guest", "azurelinux"]) - self.assertEqual(args.guest, "azurelinux") - self.assertEqual( - guests.guest_descriptor(args.guest).initramfs_name, - "initramfs-azurelinux.cpio.gz", - ) - - def test_run_help_and_usage_state_each_guest_memory_default(self): - stdout = io.StringIO() - with contextlib.redirect_stdout(stdout), self.assertRaises(SystemExit): - nvx.parse_args(["run", "--help"]) - help_text = " ".join(stdout.getvalue().split()) - usage = (BuildConstants.REPO_ROOT / "doc" / "usage.md").read_text( - encoding="utf-8" - ) - row = next( - line - for line in usage.splitlines() - if line.startswith("| `--memory-mib MIB`") - ) - labels = {"alpine": "Alpine", "ubuntu": "Ubuntu", "azurelinux": "Azure Linux"} - for name in guests.GUEST_NAMES: - default = guests.guest_descriptor(name).default_memory_mib - with self.subTest(guest=name): - self.assertIn(f"{name} {default}", help_text) - self.assertIn(f"{default} for {labels[name]}", row) - - def test_benchmark_exposes_device_restore_profile(self): - args = nvx.parse_args( - [ - "benchmark", - "--suite", - "device-restore-profile", - "--restore-devices", - "console", - "virtiofs", - "--restore-modes", - "deferred", - "--output-dir", - "results", - ] - ) - - benchmark.apply_benchmark_suite_defaults(args) - self.assertEqual(args.restore_devices, ["console", "virtiofs"]) - self.assertEqual(args.restore_modes, ["deferred"]) - self.assertEqual(args.warmups, 1) - self.assertEqual(args.runs, 5) - self.assertEqual(args.output_dir, Path("results")) - self.assertIs(args.handler, nvx.command_benchmark) - - def test_benchmark_exposes_non_python_performance_suite(self): - args = nvx.parse_args( - [ - "benchmark", - "--suite", - "performance", - "--shell-memories", - "64", - "256", - "--output-dir", - "results", - ] - ) - - self.assertEqual(args.suite, "performance") - self.assertEqual(args.shell_memories, [64, 256]) - self.assertEqual(args.virtfs_runs, 3) - self.assertEqual(args.payload_mib, 64) - self.assertEqual(args.network_memory_mib, 256) - self.assertEqual(args.host_cpu_reserve, 2) - self.assertEqual(args.output_dir, Path("results")) - self.assertIs(args.handler, nvx.command_benchmark) - - def test_device_io_uses_canonical_sampling_defaults(self): - args = nvx.parse_args(["benchmark", "--suite", "device-io"]) - - benchmark.apply_benchmark_suite_defaults(args) - - self.assertEqual(args.warmups, 5) - self.assertEqual(args.runs, 30) - self.assertEqual(args.device_io_duration_seconds, 10.0) - self.assertEqual(args.device_io_size_mib, 512) - self.assertEqual(args.device_io_port, 5201) - self.assertFalse(hasattr(args, "device_io_abi")) - - with self.assertRaises(SystemExit): - nvx.parse_args( - ["benchmark", "--suite", "device-io", "--device-io-abi", "1"] - ) - - def test_benchmark_exposes_restore_only_shell_suite(self): - args = nvx.parse_args( - [ - "benchmark", - "--suite", - "shell-snapshot-restore", - "--shell-memories", - "512", - "--processors", - "8", - "--output-dir", - "results", - ] - ) - - self.assertEqual(args.suite, "shell-snapshot-restore") - self.assertEqual(args.shell_memories, [512]) - self.assertEqual(args.processors, 8) - self.assertIs(args.handler, nvx.command_benchmark) - - def test_benchmark_exposes_restore_vcpu_matrix(self): - args = nvx.parse_args( - [ - "benchmark", - "--suite", - "snapshot-restore-vcpu", - "--processors", - "8", - "--memory-mib", - "512", - "--output-dir", - "results", - ] - ) - - self.assertEqual(args.suite, "snapshot-restore-vcpu") - self.assertEqual(args.processors, 8) - self.assertEqual(args.memory_mib, 512) - self.assertIs(args.handler, nvx.command_benchmark) - - def test_benchmark_exposes_snapshot_profile_matrix(self): - args = nvx.parse_args( - [ - "benchmark", - "--suite", - "snapshot-profile", - "--backend", - "mshv", - "--cache-state", - "cold", - ] - ) - - self.assertEqual(args.suite, "snapshot-profile") - self.assertIsNone(args.shell_memories) - benchmark.apply_benchmark_suite_defaults(args) - self.assertEqual(args.shell_memories, [128, 256, 512, 1024]) - self.assertEqual(args.cache_state, "cold") - self.assertFalse(args.snapshot_profile) - self.assertIs(args.handler, nvx.command_benchmark) - - def test_benchmark_preserves_canonical_shell_memory_defaults(self): - args = nvx.parse_args(["benchmark", "--suite", "performance"]) - - benchmark.apply_benchmark_suite_defaults(args) - - self.assertEqual(args.shell_memories, [128, 256, 512]) - - def test_benchmark_positive_integer_diagnostics_are_stable(self): - for value, message in ( - ("not-an-int", "argument --runs: invalid positive_int value: 'not-an-int'"), - ("0", "argument --runs: must be at least 1"), - ): - with self.subTest(value=value): - with ( - patch("sys.stderr", new_callable=io.StringIO) as errors, - self.assertRaises(SystemExit) as exit_context, - ): - nvx.parse_args(["benchmark", "--runs", value]) - - self.assertEqual(exit_context.exception.code, 2) - self.assertIn(message, errors.getvalue()) - - def test_release_commands_keep_their_cli_contract(self): - download = nvx.parse_args( - ["download", "--repository", "example/nvx", "--hypervisor", "auto"] - ) - self.assertEqual(download.command, "download") - self.assertEqual(download.repository, "example/nvx") - self.assertEqual(download.hypervisor, "auto") - self.assertIs(download.handler, nvx.command_download) - - collect = nvx.parse_args(["collect-sources"]) - self.assertEqual(collect.command, "collect-sources") - self.assertIs(collect.handler, nvx.command_collect_sources) - - package = nvx.parse_args( - [ - "package", - "--version", - "1.2.3", - "--destination", - "output", - "--include-source", - "--force", - ] - ) - self.assertEqual(package.command, "package") - self.assertEqual(package.version, "1.2.3") - self.assertEqual(package.destination, Path("output")) - self.assertTrue(package.include_source) - self.assertFalse(package.binary_only) - self.assertTrue(package.force) - self.assertIs(package.handler, nvx.command_package) - - verify = nvx.parse_args(["verify"]) - self.assertEqual(verify.command, "verify") - self.assertIs(verify.handler, nvx.command_verify) - - openvmm_unit_tests = nvx.parse_args(["test-openvmm-unit"]) - self.assertEqual(openvmm_unit_tests.command, "test-openvmm-unit") - self.assertIs(openvmm_unit_tests.handler, nvx.command_test_openvmm_unit) - - openvmm_tests = nvx.parse_args(["test-openvmm", "--backend", "mshv"]) - self.assertEqual(openvmm_tests.backend, "mshv") - self.assertIs(openvmm_tests.handler, nvx.command_test_openvmm) - - def test_openvmm_help_describes_petri_vmm_tests(self): - with ( - patch("sys.stdout", new_callable=io.StringIO) as output, - self.assertRaises(SystemExit) as exit_context, - ): - nvx.parse_args(["--help"]) - - self.assertEqual(exit_context.exception.code, 0) - help_text = output.getvalue() - self.assertIn("run OpenVMM Petri VMM tests", help_text) - self.assertNotIn("run OpenVMM microVM integration tests", help_text) - - def test_guest_selection_cli_contract(self): - default_build = nvx.parse_args(["build-guest"]) - self.assertEqual(default_build.guest, "alpine") - self.assertFalse(default_build.debug_kernel) - - debug_guest = nvx.parse_args(["build-guest", "--debug-kernel"]) - self.assertTrue(debug_guest.debug_kernel) - self.assertTrue(nvx._build_config(debug_guest).debug_kernel) - self.assertTrue(nvx.parse_args(["build", "--debug-kernel"]).debug_kernel) - - self.assertFalse(nvx.parse_args(["build-kernel"]).debug) - self.assertTrue(nvx.parse_args(["build-kernel", "--debug"]).debug) - - all_guests = nvx.parse_args(["build-guest", "--guest", "all"]) - self.assertEqual(all_guests.guest, "all") - - ubuntu_initramfs = nvx.parse_args(["build-initramfs", "--guest", "ubuntu"]) - self.assertEqual(ubuntu_initramfs.guest, "ubuntu") - - azurelinux_initramfs = nvx.parse_args( - ["build-initramfs", "--guest", "azurelinux"] - ) - with patch("nvx.build_docker_initramfs") as build_docker_initramfs: - nvx.command_build_initramfs(azurelinux_initramfs) - build_docker_initramfs.assert_called_once_with( - build.DockerBuildConfig(artifact_destination=BuildConstants.BUILD_DIR), - "azurelinux", - ) - - distro = nvx.parse_args( - [ - "build-distro-layer", - "--guest", - "ubuntu", - "--output", - "ubuntu.erofs", - "--replace", - ] - ) - self.assertEqual(distro.guest, "ubuntu") - self.assertEqual(distro.output, Path("ubuntu.erofs")) - self.assertTrue(distro.replace) - - with self.assertRaises(SystemExit): - nvx.parse_args(["run", "--guest", "all"]) - - def test_docker_initramfs_rejects_native_guest(self): - with self.assertRaisesRegex( - common.ScriptError, "Alpine Linux.*do not require Docker" - ): - build.build_docker_initramfs(build.DockerBuildConfig(), "alpine") - - def test_docker_initramfs_requires_expected_artifacts(self): - with tempfile.TemporaryDirectory() as temporary: - config = build.DockerBuildConfig(artifact_destination=Path(temporary)) - with ( - patch.object(build, "require_tool"), - patch.object(build, "run_checked") as run_checked, - self.assertRaisesRegex( - common.ScriptError, - "initramfs-azurelinux.cpio.gz", - ), - ): - build.build_docker_initramfs(config, "azurelinux") - command = run_checked.call_args.args[0] - self.assertEqual( - command[command.index("--target") + 1], - "azurelinux-initramfs-artifacts", - ) - - def test_docker_artifacts_require_expected_files(self): - with tempfile.TemporaryDirectory() as temporary: - config = build.DockerBuildConfig(artifact_destination=Path(temporary)) - with ( - patch.object(build, "require_tool"), - patch.object(build, "run_checked"), - self.assertRaisesRegex(common.ScriptError, "vmlinux"), - ): - build.build_docker_artifacts(config, "azurelinux") - - def test_ubuntu_run_selects_artifact_and_default_memory(self): - args = nvx.parse_args(["run", "--guest", "ubuntu", "--dry-run"]) - - def require(path: Path, _description: str) -> Path: - return path - - with ( - patch.object(nvx, "require_file", side_effect=require), - patch.object( - nvx, - "_format_command", - return_value="formatted", - ) as format_command, - ): - nvx.command_run(args) - - command = format_command.call_args.args[0] - self.assertEqual(command[command.index("--memory") + 1], "512M") - self.assertEqual( - Path(command[command.index("--initrd") + 1]).name, - "initramfs-ubuntu.cpio.gz", - ) - - def test_azurelinux_run_selects_artifact_and_default_memory(self): - args = nvx.parse_args(["run", "--guest", "azurelinux", "--dry-run"]) - - def require(path: Path, _description: str) -> Path: - return path - - with ( - patch.object(nvx, "require_file", side_effect=require), - patch.object( - nvx, - "_format_command", - return_value="formatted", - ) as format_command, - ): - nvx.command_run(args) - - command = format_command.call_args.args[0] - self.assertEqual(command[command.index("--memory") + 1], "512M") - self.assertEqual( - Path(command[command.index("--initrd") + 1]).name, - "initramfs-azurelinux.cpio.gz", - ) - - def test_restore_rejects_ubuntu_guest_selection(self): - args = nvx.parse_args( - [ - "run", - "--guest", - "ubuntu", - "--restore-snapshot", - "snapshot", - "--dry-run", - ] - ) - with self.assertRaisesRegex(common.ScriptError, "snapshot already fixes"): - nvx.command_run(args) - - def test_sandbox_rejects_systemd_entrypoint(self): - for entrypoint in nvx.SYSTEMD_ENTRYPOINTS: - with self.subTest(entrypoint=entrypoint): - args = nvx.parse_args( - [ - "sandbox", - "--entrypoint", - entrypoint, - "--layer", - "distro,distro.erofs,11111111-1111-1111-1111-111111111111", - "--scratch", - "scratch.ext4", - ] - ) - with self.assertRaisesRegex( - common.ScriptError, - "systemd entrypoints", - ): - nvx.command_sandbox(args) - - def test_sandbox_allows_non_systemd_sbin_init(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - layer = root / "distro.erofs" - scratch = root / "scratch.ext4" - layer.write_bytes(b"distro") - scratch.write_bytes(b"scratch") - args = nvx.parse_args( - [ - "sandbox", - "--entrypoint", - "/sbin/init", - "--layer", - f"distro,{layer},11111111-1111-1111-1111-111111111111", - "--scratch", - str(scratch), - "--dry-run", - ] - ) - - def require(path: Path, _description: str) -> Path: - return path - - with ( - patch.object(nvx, "require_file", side_effect=require), - patch.object(nvx, "_format_command", return_value="formatted"), - ): - nvx.command_sandbox(args) - - def test_sandbox_rejects_systemd_from_bound_distro_metadata(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - layer = root / "distro.erofs" - scratch = root / "scratch.ext4" - layer.write_bytes(b"distro") - scratch.write_bytes(b"scratch") - layer.with_name(f"{layer.name}.manifest.json").write_text( - json.dumps( - { - "format": 1, - "artifact": layer.name, - "artifact_sha256": common.sha256_file(layer), - "packages": [{"name": "systemd"}], - } - ), - encoding="utf-8", - ) - args = nvx.parse_args( - [ - "sandbox", - "--entrypoint", - "/sbin/init", - "--layer", - f"distro,{layer},11111111-1111-1111-1111-111111111111", - "--scratch", - str(scratch), - ] - ) - with self.assertRaisesRegex(common.ScriptError, "systemd images"): - nvx.command_sandbox(args) - - def test_sandbox_command_parses_typed_launch_contract(self): - args = nvx.parse_args( - [ - "sandbox", - "--layer", - "distro,distro.erofs,11111111-1111-1111-1111-111111111111", - "--scratch", - "scratch.ext4", - "--entrypoint", - "/bin/workload", - "--arg=--serve", - "--memory-max", - "268435456", - "--pids-max", - "64", - "--workload-user", - "1000:1001", - ] - ) - - self.assertEqual(args.layer[0].role, "distro") - self.assertEqual(args.scratch, Path("scratch.ext4")) - self.assertEqual(args.sandbox_arg, ["--serve"]) - self.assertEqual(args.memory_max, 268435456) - self.assertEqual(args.pids_max, 64) - self.assertEqual(args.workload_user, (1000, 1001)) - self.assertIs(args.handler, nvx.command_sandbox) - - def test_sandbox_command_parses_state_aware_operations(self): - provision = nvx.parse_args( - [ - "sandbox", - "provision", - "--state-dir", - "state", - "--layer", - "distro,distro.erofs,11111111-1111-1111-1111-111111111111", - "--scratch", - "scratch.ext4", - ] - ) - self.assertEqual(provision.sandbox_operation, "provision") - self.assertEqual(provision.state_dir, Path("state")) - - execute = nvx.parse_args( - [ - "sandbox", - "exec", - "--state-dir", - "state", - "--entrypoint", - "/bin/sh", - "--arg=-c", - "--arg=echo managed", - "--exec-timeout-ms", - "5000", - ] - ) - self.assertEqual(execute.sandbox_operation, "exec") - self.assertEqual(execute.sandbox_arg, ["-c", "echo managed"]) - self.assertEqual(execute.exec_timeout_ms, 5000) - self.assertFalse(execute.inherit_default_environment) - - configured = nvx.parse_args( - [ - "sandbox", - "exec", - "--state-dir", - "state", - "--cwd", - "/work tree", - "--environment", - "EMPTY=", - "--environment", - "VALUE=space = value", - "--inherit-default-environment", - ] - ) - self.assertEqual(configured.cwd, "/work tree") - self.assertEqual( - configured.environment, - ["EMPTY=", "VALUE=space = value"], - ) - self.assertTrue(configured.inherit_default_environment) - - report = nvx.parse_args( - [ - "sandbox", - "exec", - "--state-dir", - "state", - "--outcome-report", - "exec-outcome.json", - ] - ) - self.assertEqual(report.outcome_report, Path("exec-outcome.json")) - - def test_sandbox_timeout_must_be_finite_and_positive(self): - for value in ("0", "-1", "nan", "inf"): - stderr = io.StringIO() - with ( - self.subTest(value=value), - contextlib.redirect_stderr(stderr), - self.assertRaises(SystemExit), - ): - nvx.parse_args( - ["sandbox", "start", "--state-dir", "state", "--timeout", value] - ) - self.assertIn("must be finite and greater than 0", stderr.getvalue()) - - def test_sandbox_managed_operations_reject_dry_run_before_state_access(self): - lifecycle_methods = ( - ("provision", "provision"), - ("start", "start"), - ("exec", "exec_workload"), - ("stop", "stop"), - ("deprovision", "deprovision"), - ) - with tempfile.TemporaryDirectory() as temporary: - state = Path(temporary) / "state" - state.mkdir() - marker = state / "marker" - marker.write_text("preserved", encoding="utf-8") - - for operation, lifecycle_method in lifecycle_methods: - with self.subTest(operation=operation): - args = nvx.parse_args( - [ - "sandbox", - operation, - "--state-dir", - str(state), - "--dry-run", - ] - ) - with ( - patch.object( - sandbox_lifecycle, - lifecycle_method, - ) as lifecycle, - self.assertRaisesRegex( - common.ScriptError, - "--dry-run is only valid for sandbox run", - ), - ): - nvx.command_sandbox(args) - lifecycle.assert_not_called() - self.assertEqual(marker.read_text(encoding="utf-8"), "preserved") - - def test_sandbox_exec_forwards_explicit_empty_environment_and_cwd(self): - with tempfile.TemporaryDirectory() as temporary: - environment_file = Path(temporary) / "environment.json" - environment_file.write_text("[]", encoding="utf-8") - args = nvx.parse_args( - [ - "sandbox", - "exec", - "--state-dir", - "state", - "--cwd", - "/work", - "--environment-file", - str(environment_file), - ] - ) - result = sandbox_lifecycle.ManagedExecResult(0, "exit", b"", b"") - with ( - patch.object( - sandbox_lifecycle, - "exec_workload", - return_value=result, - ) as execute, - self.assertRaises(SystemExit) as exit_context, - ): - nvx.command_sandbox(args) - - self.assertEqual(exit_context.exception.code, 0) - execute.assert_called_once_with( - Path("state"), - ("/bin/sh",), - timeout_ms=0, - response_timeout=60.0, - cwd="/work", - environment=(), - inherit_default_environment=False, - ) - - def test_sandbox_exec_forwards_environment_inheritance(self): - cases: tuple[tuple[list[str], tuple[str, ...] | None], ...] = ( - (["--environment", "A=1"], ("A=1",)), - ( - ["--environment", "A=1", "--environment", "TERM=dumb"], - ("A=1", "TERM=dumb"), - ), - # Without entries the workload gets the defaults whatever the flag says. - ([], None), - ) - for environment_args, environment in cases: - with self.subTest(environment=environment): - args = nvx.parse_args( - [ - "sandbox", - "exec", - "--state-dir", - "state", - *environment_args, - "--inherit-default-environment", - ] - ) - result = sandbox_lifecycle.ManagedExecResult(0, "exit", b"", b"") - with ( - patch.object( - sandbox_lifecycle, - "exec_workload", - return_value=result, - ) as execute, - self.assertRaises(SystemExit), - ): - nvx.command_sandbox(args) - execute.assert_called_once_with( - Path("state"), - ("/bin/sh",), - timeout_ms=0, - response_timeout=60.0, - cwd=None, - environment=environment, - inherit_default_environment=True, - ) - - def test_sandbox_exec_rejects_ambiguous_or_one_shot_environment(self): - both = nvx.parse_args( - [ - "sandbox", - "exec", - "--state-dir", - "state", - "--environment", - "A=1", - "--environment-file", - "environment.json", - ] - ) - with self.assertRaisesRegex(common.ScriptError, "mutually exclusive"): - nvx.command_sandbox(both) - - one_shot = nvx.parse_args(["sandbox", "run", "--environment", "A=1"]) - with self.assertRaisesRegex(common.ScriptError, "require.*exec"): - nvx.command_sandbox(one_shot) - - for operation in ("run", "start"): - inheriting = nvx.parse_args( - ["sandbox", operation, "--inherit-default-environment"] - ) - with ( - self.subTest(operation=operation), - self.assertRaisesRegex(common.ScriptError, "require.*exec"), - ): - nvx.command_sandbox(inheriting) - - def test_sandbox_environment_file_is_bounded_before_state_access(self): - with tempfile.TemporaryDirectory() as temporary: - path = Path(temporary) / "environment.json" - path.write_bytes(b" " * (1024 * 1024 + 1)) - args = nvx.parse_args( - [ - "sandbox", - "exec", - "--state-dir", - "absent-state", - "--environment-file", - str(path), - ] - ) - with ( - patch.object(sandbox_lifecycle, "exec_workload") as execute, - self.assertRaisesRegex(common.ScriptError, "exceeds.*byte limit"), - ): - nvx.command_sandbox(args) - execute.assert_not_called() - - def test_sandbox_environment_is_validated_before_state_access(self): - too_many = [ - value - for index in range(257) - for value in ("--environment", f"KEY{index}=value") - ] - cases = ( - (too_many, "exceeds 256 entries"), - (["--environment", "NO_EQUALS"], "non-empty KEY=VALUE"), - (["--environment", "=empty-key"], "non-empty KEY=VALUE"), - (["--environment", "KEY=embedded\0nul"], "non-empty KEY=VALUE"), - ( - ["--environment", "KEY=" + "\N{SNOWMAN}" * 1365], - "at most 4096 bytes", - ), - ( - ["--environment", "DUPLICATE=one", "--environment", "DUPLICATE=two"], - "duplicate key", - ), - ) - for environment_args, message in cases: - with self.subTest(message=message): - args = nvx.parse_args( - [ - "sandbox", - "exec", - "--state-dir", - "absent-state", - *environment_args, - ] - ) - with ( - patch.object(sandbox_lifecycle, "exec_workload") as execute, - self.assertRaisesRegex(common.ScriptError, message), - ): - nvx.command_sandbox(args) - execute.assert_not_called() - - def test_sandbox_environment_file_entries_are_validated_before_state_access(self): - with tempfile.TemporaryDirectory() as temporary: - path = Path(temporary) / "environment.json" - path.write_text( - json.dumps(["DUPLICATE=one", "DUPLICATE=two"]), - encoding="utf-8", - ) - args = nvx.parse_args( - [ - "sandbox", - "exec", - "--state-dir", - "absent-state", - "--environment-file", - str(path), - ] - ) - with ( - patch.object(sandbox_lifecycle, "exec_workload") as execute, - self.assertRaisesRegex(common.ScriptError, "duplicate key"), - ): - nvx.command_sandbox(args) - execute.assert_not_called() - - def test_network_requires_explicit_portable_profile(self): - args = nvx.parse_args( - [ - "run", - "--net", - "10.0.0.2/24", - "--network-profile", - "portable", - ] - ) - self.assertEqual(args.network_profile, "portable") - - missing_profile = nvx.parse_args(["run", "--net", "10.0.0.2/24"]) - with self.assertRaisesRegex(common.ScriptError, "--net and --network-profile"): - nvx.command_run(missing_profile) - - missing_network = nvx.parse_args(["run", "--network-profile", "portable"]) - with self.assertRaisesRegex(common.ScriptError, "--net and --network-profile"): - nvx.command_run(missing_network) - - def test_run_and_sandbox_forward_network_arguments(self): - network_arguments = ( - "--net 10.0.0.2/24 --network-profile portable " - "--network-egress deny --network-ingress deny " - "--network-egress-allow 140.82.112.0/20:tcp:443 " - "--network-egress-deny 10.0.0.0/8 --host-loopback allow " - "--network-proxy 10.0.0.1:3128 --host-loopback-forward tcp:8080:80" - ).split() - commands = [ - ["run", "--dry-run", *network_arguments], - [ - "sandbox", - "--dry-run", - "--layer", - "distro,distro.erofs,11111111-1111-1111-1111-111111111111", - "--scratch", - "scratch.ext4", - *network_arguments, - ], - ] - - def require(path: Path, _description: str) -> Path: - return path - - for arguments in commands: - with self.subTest(command=arguments[0]): - args = nvx.parse_args(arguments) - with ( - patch.object(nvx, "require_file", side_effect=require), - patch.object(sandbox, "require_file", side_effect=require), - patch.object( - nvx, "_format_command", return_value="formatted" - ) as format_command, - ): - args.handler(args) - - command = format_command.call_args.args[0] - self.assertEqual( - command[command.index("--net") :], - network_arguments, - ) - - def test_run_lowers_structured_egress_policy_before_launch(self): - with tempfile.TemporaryDirectory() as temporary: - policy = Path(temporary) / "policy.json" - policy.write_text( - json.dumps( - { - "allow": [ - { - "to": [ - { - "cidr": "192.0.2.0/24", - "except": ["192.0.2.128/25"], - } - ], - "ports": [ - { - "protocol": "tcp", - "port": 8000, - "endPort": 8001, - } - ], - }, - {"to": [{"cidr": "192.0.2.200/32"}]}, - ], - "deny": [ - { - "to": [{"cidr": "192.0.2.0/24"}], - "ports": [{"protocol": "tcp", "port": 8001}], - } - ], - } - ), - encoding="utf-8", - ) - args = nvx.parse_args( - [ - "run", - "--dry-run", - "--network-egress", - "deny", - "--network-egress-policy-file", - str(policy), - ] - ) - - def require(path: Path, _description: str) -> Path: - return path - - with ( - patch.object(nvx, "require_file", side_effect=require), - patch.object( - nvx, "_format_command", return_value="formatted" - ) as format_command, - ): - nvx.command_run(args) - - command = format_command.call_args.args[0] - self.assertNotIn("--network-egress-policy-file", command) - self.assertEqual(command.count("--network-egress-allow"), 2) - self.assertIn("192.0.2.0/25:tcp:8000-8001", command) - self.assertIn("192.0.2.200/32", command) - self.assertEqual(command.count("--network-egress-deny"), 1) - self.assertIn("192.0.2.0/24:tcp:8001", command) - - def test_policy_file_rejects_ambiguous_flags_before_artifact_access(self): - cases = ( - [ - "--network-egress", - "deny", - "--network-egress-policy-file", - "policy.json", - "--network-egress-allow", - "192.0.2.1", - ], - ["--network-egress-policy-file", "policy.json"], - ) - for extra in cases: - with self.subTest(extra=extra): - args = nvx.parse_args(["run", "--dry-run", *extra]) - with ( - patch.object(nvx, "require_file") as require, - self.assertRaises(common.ScriptError), - ): - nvx.command_run(args) - require.assert_not_called() - - def test_sandbox_rejects_policy_mixing_before_validating_layer_files(self): - for operation in ("run", "provision"): - with self.subTest(operation=operation): - args = nvx.parse_args( - [ - "sandbox", - operation, - "--layer", - "distro,absent.erofs,00000000-0000-4000-8000-000000000001", - "--scratch", - "absent.ext4", - "--network-egress", - "deny", - "--network-egress-policy-file", - "absent.json", - "--network-egress-allow", - "192.0.2.1", - ] - ) - with ( - patch.object(nvx.SandboxLaunch, "validated") as validated, - self.assertRaisesRegex(common.ScriptError, "cannot be combined"), - ): - nvx.command_sandbox(args) - validated.assert_not_called() - - def test_managed_provision_persists_lowered_policy_not_source_path(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - layer = root / "distro.erofs" - scratch = root / "scratch.ext4" - policy = root / "policy.json" - state = root / "state" - layer.write_bytes(b"layer") - scratch.write_bytes(b"scratch") - policy.write_text( - json.dumps( - { - "allow": [ - { - "to": [ - { - "cidr": "192.0.2.0/24", - "except": ["192.0.2.128/25"], - } - ], - } - ] - } - ), - encoding="utf-8", - ) - args = nvx.parse_args( - [ - "sandbox", - "provision", - "--state-dir", - str(state), - "--layer", - f"distro,{layer},11111111-1111-1111-1111-111111111111", - "--scratch", - str(scratch), - "--network-egress", - "deny", - "--network-egress-policy-file", - str(policy), - ] - ) - - nvx.command_sandbox(args) - config = json.loads( - (state / sandbox_lifecycle.CONFIG_NAME).read_text(encoding="utf-8") - ) - policy.unlink() - - self.assertEqual(config["network_egress"], "deny") - self.assertEqual(config["network_egress_allow"], ["192.0.2.0/25"]) - self.assertEqual(config["network_egress_deny"], []) - self.assertNotIn("network_egress_policy_file", config) - - def test_public_cli_provision_persists_policy_after_source_removal(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - layer = root / "distro.erofs" - scratch = root / "scratch.ext4" - policy = root / "policy.json" - state = root / "state" - layer.write_bytes(b"layer") - scratch.write_bytes(b"scratch") - policy.write_text( - json.dumps( - { - "allow": [ - { - "to": [ - { - "cidr": "192.0.2.0/24", - "except": ["192.0.2.128/25"], - } - ], - "ports": [ - { - "protocol": "tcp", - "port": 8000, - "endPort": 8001, - } - ], - } - ], - "deny": [ - { - "to": [{"cidr": "192.0.2.0/24"}], - "ports": [{"protocol": "tcp", "port": 8001}], - } - ], - } - ), - encoding="utf-8", - ) - result = subprocess.run( - [ - sys.executable, - str(Path(nvx.__file__).resolve()), - "sandbox", - "provision", - "--state-dir", - str(state), - "--layer", - f"distro,{layer},11111111-1111-1111-1111-111111111111", - "--scratch", - str(scratch), - "--network-egress", - "deny", - "--network-egress-policy-file", - str(policy), - ], - cwd=BuildConstants.REPO_ROOT, - capture_output=True, - timeout=10, - ) - self.assertEqual( - result.returncode, - 0, - result.stderr.decode("utf-8", "replace"), - ) - policy.unlink() - config = json.loads( - (state / sandbox_lifecycle.CONFIG_NAME).read_text(encoding="utf-8") - ) - - self.assertEqual( - config["network_egress_allow"], - ["192.0.2.0/25:tcp:8000-8001"], - ) - self.assertEqual( - config["network_egress_deny"], - ["192.0.2.0/24:tcp:8001"], - ) - self.assertNotIn(str(policy), json.dumps(config)) - - def test_invalid_policy_has_no_managed_state_side_effect(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - layer = root / "distro.erofs" - scratch = root / "scratch.ext4" - policy = root / "policy.json" - state = root / "state" - layer.write_bytes(b"layer") - scratch.write_bytes(b"scratch") - policy.write_text( - '{"allow":[{"to":[{"cidr":"192.0.2.0/24"}],' - '"ports":[{"protocol":"icmp","port":8}]}]}', - encoding="utf-8", - ) - args = nvx.parse_args( - [ - "sandbox", - "provision", - "--state-dir", - str(state), - "--layer", - f"distro,{layer},11111111-1111-1111-1111-111111111111", - "--scratch", - str(scratch), - "--network-egress", - "deny", - "--network-egress-policy-file", - str(policy), - ] - ) - - with self.assertRaises(common.ScriptError): - nvx.command_sandbox(args) - self.assertFalse(state.exists()) - - def test_managed_non_launch_operations_reject_network_policy_options(self): - args = nvx.parse_args( - [ - "sandbox", - "start", - "--state-dir", - "state", - "--network-egress", - "deny", - "--network-egress-policy-file", - "policy.json", - ] - ) - with ( - patch.object(sandbox_lifecycle, "start") as start, - self.assertRaisesRegex( - common.ScriptError, "only valid for sandbox run or provision" - ), - ): - nvx.command_sandbox(args) - start.assert_not_called() - - def test_run_parses_denied_filesystem_paths(self): - args = nvx.parse_args( - [ - "run", - "--mount", - "/mnt/share,share,rw", - "--mount-deny", - "share/secrets", - "--mount-deny", - "share/private", - ] - ) - self.assertEqual( - args.mount_deny, - [Path("share/secrets"), Path("share/private")], - ) - - def test_run_forwards_the_share_access_policy(self): - args = nvx.parse_args( - [ - "run", - "--mount", - "/mnt/share,share,rw", - "--mount-write", - "share/out", - "--mount-allow", - "share/logs/payloads", - "--mount-deny", - "share/logs", - "--dry-run", - ] - ) - with ( - patch.object(nvx, "require_file", return_value=Path("artifact")), - patch.object( - nvx, "_format_command", return_value="formatted" - ) as format_command, - ): - nvx.command_run(args) - - command = format_command.call_args.args[0] - self.assertEqual( - [ - (value, command[index + 1]) - for index, value in enumerate(command) - if value.startswith("--mount-") - ], - [ - ("--mount-deny", str(Path("share/logs"))), - ("--mount-allow", str(Path("share/logs/payloads"))), - ("--mount-write", str(Path("share/out"))), - ], - ) - - def test_run_forwards_mount_owner_and_requires_mount(self): - args = nvx.parse_args( - ["run", "--mount", "/mnt/share,share,rw", "--mount-owner", "caller"] - + ["--dry-run"] - ) - with ( - patch.object(nvx, "require_file", return_value=Path("artifact")), - patch.object( - nvx, "_format_command", return_value="formatted" - ) as format_command, - ): - if sys.platform == "win32": - with self.assertRaisesRegex(common.ScriptError, "Linux host"): - nvx.command_run(args) - format_command.assert_not_called() - else: - nvx.command_run(args) - command = format_command.call_args.args[0] - self.assertEqual(command[command.index("--mount-owner") + 1], "caller") - - default = nvx.parse_args(["run", "--mount", "/mnt/share,share,rw"]) - self.assertIsNone(default.mount_owner) - unmounted = nvx.parse_args(["run", "--mount-owner", "vmm", "--dry-run"]) - with self.assertRaisesRegex(common.ScriptError, "requires --mount"): - nvx.command_run(unmounted) - with self.assertRaises(SystemExit), contextlib.redirect_stderr(io.StringIO()): - nvx.parse_args( - ["run", "--mount", "/mnt/share,share", "--mount-owner", "root"] - ) - - def test_run_forwards_bounded_outcome_report(self): - args = nvx.parse_args(["run", "--outcome-report", "outcome.json", "--dry-run"]) - with ( - patch.object(nvx, "require_file", return_value=Path("artifact")), - patch.object( - nvx, "_format_command", return_value="formatted" - ) as format_command, - ): - nvx.command_run(args) - - command = format_command.call_args.args[0] - self.assertEqual( - command[command.index("--microvm-report") + 1], - "outcome.json", - ) - - def test_run_forwards_the_cpu_profile(self): - def launched(*extra: str) -> list[str]: - args = nvx.parse_args(["run", *extra, "--dry-run"]) - with ( - patch.object(nvx, "require_file", return_value=Path("artifact")), - patch.object( - nvx, "_format_command", return_value="formatted" - ) as format_command, - ): - nvx.command_run(args) - return format_command.call_args.args[0] - - # Without the option, OpenVMM selects auto itself. - self.assertNotIn("--cpu-profile", launched()) - for profile in ("host", "auto", "intel.alderlake.v1"): - command = launched("--cpu-profile", profile) - self.assertEqual(command[command.index("--cpu-profile") + 1], profile) - restored = launched("--restore-snapshot", "snapshot", "--cpu-profile", "host") - self.assertEqual(restored[restored.index("--cpu-profile") + 1], "host") - - def test_run_explains_a_host_cpu_that_no_profile_serves(self): - tiger_lake = HostCpu("GenuineIntel", 6, 140, 1, "Intel(R) Core(TM) i7-1185G7") - zen3 = HostCpu("AuthenticAMD", 25, 33, 0) - openvmm = "/nvx/openvmm/target/release/openvmm" - - def run( - returncode: int, - cpu_profile: str | None, - cold_boot: bool = True, - host: HostCpu | None = tiger_lake, - ) -> tuple[int, str]: - stderr = io.StringIO() - completed = subprocess.CompletedProcess[bytes]([openvmm], returncode) - with ( - patch.object(nvx.sys, "stderr", stderr), - patch.object(nvx.subprocess, "run", return_value=completed) as launch, - patch.object(nvx, "host_cpu_signature", return_value=host), - ): - status = nvx._run_openvmm( # pyright: ignore[reportPrivateUsage] - [openvmm, "--hypervisor", "kvm"], cpu_profile, cold_boot, "kvm" - ) - # OpenVMM keeps the terminal, its standard error included. - launch.assert_called_once_with([openvmm, "--hypervisor", "kvm"]) - return status, stderr.getvalue() - - status, output = run(1, None) - self.assertEqual(status, 1) - # auto falls back on the CPU, so run says so, and asks for the CPU's - # fingerprint, which this OpenVMM writes for this backend, through the - # CPU profile request form. - self.assertIn( - "nvx: no built-in CPU profile serves this host's CPU, " - "GenuineIntel 6/140/1 (Intel(R) Core(TM) i7-1185G7), so --cpu-profile " - "auto, the default, falls back on it to a CPU profile derived from " - "this host", - output, - ) - self.assertIn("alderlake 6/151 and 6/154", output) - self.assertIn( - f"nvx: {openvmm} --hypervisor kvm --cpu-fingerprint fingerprint.json\n", - output, - ) - self.assertIn(cpu_profile_request_url(tiger_lake), output) - self.assertIn("https://github.com/microsoft/nvx/issues/408", output) - self.assertNotIn("rerun with --cpu-profile host", output) - self.assertEqual(run(1, "auto"), (1, output)) - # Host profiles serve AMD CPUs too, so an AMD host that no built-in - # profile serves falls back too, with the issue that tracks profiles - # for more AMD CPUs, and a host profile that fails on it fails for - # another reason, which OpenVMM explains. - status, output = run(1, None, host=zen3) - self.assertIn("AuthenticAMD 25/33/0, so --cpu-profile auto", output) - self.assertIn(cpu_profile_request_url(zen3), output) - self.assertIn("https://github.com/microsoft/nvx/issues/409", output) - self.assertNotIn("issues/408", output) - self.assertEqual(run(1, "host", host=zen3), (1, "")) - # Host profiles serve only Intel and AMD CPUs, so another vendor's - # host still fails, with either request. - hygon = HostCpu("HygonGenuine", 24, 0, 1) - status, output = run(1, None, host=hygon) - self.assertIn( - "HygonGenuine 24/0/1, so a cold boot with --cpu-profile auto, the " - "default, fails on it with E_PROFILE_HOST_UNKNOWN", - output, - ) - self.assertIn("host CPU profiles serve only Intel and AMD CPUs", output) - self.assertNotIn("cpu-profile.yml", output) - self.assertEqual(run(1, "host", host=hygon), (1, output)) - # Successful runs, restores, other requests and CPUs, and an unknown - # CPU get no guidance. - for returncode, cpu_profile, cold_boot, host in ( - (0, None, True, tiger_lake), - (1, None, False, tiger_lake), - (1, "host", True, tiger_lake), - (1, "intel.alderlake.v1", True, tiger_lake), - (1, None, True, HostCpu("GenuineIntel", 6, 154, 3)), - (1, None, True, HostCpu("AuthenticAMD", 25, 1, 1)), - (1, None, True, None), - ): - with self.subTest( - returncode=returncode, - cpu_profile=cpu_profile, - cold_boot=cold_boot, - host=host, - ): - self.assertEqual( - run(returncode, cpu_profile, cold_boot, host), (returncode, "") - ) - - def test_run_quotes_the_openvmm_path_in_the_cpu_profile_guidance(self): - # The guidance's fingerprint command is for copying, so a path with a - # space is quoted as the shell of this platform reads it. - openvmm = str(Path("/nvx checkout") / "openvmm") - stderr = io.StringIO() - with ( - patch.object(nvx.sys, "stderr", stderr), - patch.object( - nvx.subprocess, - "run", - return_value=subprocess.CompletedProcess[bytes]([openvmm], 1), - ), - patch.object( - nvx, - "host_cpu_signature", - return_value=HostCpu("GenuineIntel", 6, 140, 1), - ), - ): - nvx._run_openvmm( # pyright: ignore[reportPrivateUsage] - [openvmm, "--hypervisor", "kvm"], None, True, "kvm" - ) - quoted = ( - subprocess.list2cmdline([openvmm]) - if os.name == "nt" - else shlex.quote(openvmm) - ) - self.assertNotEqual(quoted, openvmm) - self.assertIn( - f"nvx: {quoted} --hypervisor kvm --cpu-fingerprint fingerprint.json\n", - stderr.getvalue(), - ) - - def test_run_passes_its_backend_to_the_cpu_profile_guidance(self): - args = nvx.parse_args(["run", "--hypervisor", "kvm"]) - with ( - patch.object(nvx, "require_file", return_value=Path("artifact")), - patch.object(nvx, "_run_openvmm", return_value=0) as run_openvmm, - contextlib.redirect_stdout(io.StringIO()), - self.assertRaises(SystemExit), - ): - nvx.command_run(args) - command, cpu_profile, cold_boot, hypervisor = run_openvmm.call_args.args - self.assertEqual(command[command.index("--hypervisor") + 1], "kvm") - self.assertEqual((cpu_profile, cold_boot, hypervisor), (None, True, "kvm")) - - def test_benchmark_refuses_a_cpu_that_no_built_in_profile_serves(self): - # The command guards the benchmark coordinator, which it runs after. - self.assertIs(nvx.run_benchmark, benchmark.run) - granite_rapids = HostCpu("GenuineIntel", 6, 173, 1, "Intel(R) Xeon(R) 6973P-C") - icelake = HostCpu("GenuineIntel", 6, 106, 6) - temporary = tempfile.TemporaryDirectory() - self.addCleanup(temporary.cleanup) - - def checkout(name: str, *generations: CpuGeneration) -> str: - # An OpenVMM checkout that pins profiles of these generations, with - # the fields that NVX reads. - directory = Path(temporary.name) / name - profiles = directory / OPENVMM_CPU_PROFILES - profiles.mkdir(parents=True) - for generation in generations: - document = { - "vendor": generation.vendor, - "generation": { - "name": generation.name, - "cpus": [ - { - "family": cpu.family, - "model": cpu.model, - "steppings": [ - cpu.steppings.start, - cpu.steppings.stop - 1, - ], - } - for cpu in generation.cpus - ], - }, - } - (profiles / f"{generation.name}.json").write_text( - json.dumps(document), encoding="utf-8" - ) - return str(directory) - - pinned = checkout("pinned", *CPU_GENERATIONS) - - def run_command( - host: HostCpu | None, *arguments: str, openvmm_dir: str = pinned - ) -> tuple[int, str, list[argparse.Namespace]]: - stderr = io.StringIO() - launched: list[argparse.Namespace] = [] - - def run(args: argparse.Namespace) -> int: - launched.append(args) - return 0 - - with ( - patch.object(nvx, "host_cpu_signature", return_value=host), - patch.object(nvx, "run_benchmark", side_effect=run), - contextlib.redirect_stderr(stderr), - ): - status = nvx.main( - ["benchmark", "--openvmm-dir", openvmm_dir, *arguments] - ) - return status, stderr.getvalue(), launched - - # The refusal comes before the coordinator builds or measures anything. - status, output, launched = run_command( - granite_rapids, "--suite", "e2e", "--backend", "kvm" - ) - self.assertEqual((status, launched), (1, [])) - self.assertTrue( - output.startswith( - "error: benchmark refuses this host: no built-in CPU profile serves " - "its CPU, GenuineIntel 6/173/1 (Intel(R) Xeon(R) 6973P-C)" - ), - output, - ) - self.assertIn("openvmm --hypervisor kvm --cpu-fingerprint", output) - self.assertIn(cpu_profile_request_url(granite_rapids), output) - # On Windows, both backends name WHP's fingerprint. - status, output, launched = run_command(granite_rapids, "--backend", "both") - self.assertEqual((status, launched), (1, [])) - self.assertIn("openvmm --hypervisor whp --cpu-fingerprint", output) - # A CPU that cannot be identified is refused too. - status, output, launched = run_command(None) - self.assertEqual((status, launched), (1, [])) - self.assertIn("it cannot identify the host's CPU", output) - # The phase 2 suite boots no microVM, so it runs on any CPU, and a CPU - # that a built-in profile serves runs every suite. - for host, arguments in ( - (granite_rapids, ("--suite", "phase2")), - (None, ("--suite", "phase2")), - (icelake, ("--suite", "e2e")), - (icelake, ()), - ): - with self.subTest(host=host, arguments=arguments): - status, output, launched = run_command(host, *arguments) - self.assertEqual((status, output), (0, "")) - self.assertEqual(len(launched), 1) - # The profiles that decide are those of the OpenVMM checkout that - # benchmark builds: one that adds a profile for a CPU benchmarks it, and - # one that lacks a CPU's profile refuses it. - graniterapids = CpuGeneration( - "graniterapids", "GenuineIntel", (CpuModel(6, 173),) - ) - candidate = checkout("candidate", *CPU_GENERATIONS, graniterapids) - status, output, launched = run_command( - granite_rapids, "--suite", "e2e", openvmm_dir=candidate - ) - self.assertEqual((status, output), (0, "")) - self.assertEqual([args.openvmm_dir for args in launched], [Path(candidate)]) - lacking = checkout( - "lacking", - *( - generation - for generation in CPU_GENERATIONS - if generation.name != "icelake-sp" - ), - ) - status, output, launched = run_command(icelake, openvmm_dir=lacking) - self.assertEqual((status, launched), (1, [])) - self.assertIn( - "no built-in CPU profile serves its CPU, GenuineIntel 6/106/6", output - ) - # A checkout whose generations overlap on a CPU is refused there, since - # every cold boot fails on it. - ambiguous = checkout( - "ambiguous", - *CPU_GENERATIONS, - CpuGeneration("icelake-x", "GenuineIntel", (CpuModel(6, 106),)), - ) - status, output, launched = run_command(icelake, openvmm_dir=ambiguous) - self.assertEqual((status, launched), (1, [])) - self.assertIn( - "is in more than one generation of the built-in CPU profiles " - "(icelake-sp, icelake-x)", - output, - ) - # A checkout whose profiles cannot be read is refused, except for the - # phase 2 suite. - missing = str(Path(temporary.name) / "missing") - status, output, launched = run_command(icelake, openvmm_dir=missing) - self.assertEqual((status, launched), (1, [])) - self.assertTrue( - output.startswith( - "error: benchmark cannot confirm that a built-in CPU profile serves " - f"this host's CPU: the OpenVMM checkout {missing} pins no CPU profile" - ), - output, - ) - status, output, launched = run_command( - icelake, "--suite", "phase2", openvmm_dir=missing - ) - self.assertEqual((status, output, len(launched)), (0, "", 1)) - # A KVM worker, which the Windows coordinator reinvokes inside WSL - # without its --openvmm-dir, runs the binaries that the coordinator - # staged after the check, so it skips the check, whatever its checkout. - status, output, launched = run_command( - granite_rapids, "--_kvm-worker", openvmm_dir=missing - ) - self.assertEqual((status, output, len(launched)), (0, "", 1)) - - def test_run_leaves_openvmm_the_terminal_to_restore(self): - """A guest-triggered shutdown can end OpenVMM while its console holds - the terminal in raw mode, and OpenVMM restores the terminal settings - only when its standard error is a terminal, so run must leave it - one.""" - if sys.platform != "linux": - self.skipTest("needs a Linux pseudo-terminal") - import pty - import termios - - openvmm = ( - "import os, termios, tty\n" - # OpenVMM saves the settings through standard error, if it is a - # terminal, and its console makes standard input raw... - "saved = termios.tcgetattr(2) if os.isatty(2) else None\n" - "tty.setraw(0)\n" - # ...until the guest shuts down. - "if saved is not None:\n" - " termios.tcsetattr(2, termios.TCSAFLUSH, saved)\n" - ) - runner = ( - "import sys\n" - f"sys.path.insert(0, {str(Path(__file__).parent)!r})\n" - "import nvx\n" - "sys.exit(nvx._run_openvmm(" - f"[sys.executable, '-c', {openvmm!r}], None, cold_boot=False, " - "hypervisor='kvm'))\n" - ) - primary, secondary = pty.openpty() - try: - before = termios.tcgetattr(secondary) - completed = subprocess.run( - [sys.executable, "-c", runner], - stdin=secondary, - stdout=secondary, - stderr=secondary, - timeout=60, - check=False, - ) - after = termios.tcgetattr(secondary) - output = b"" - while select.select([primary], [], [], 0)[0]: - chunk = os.read(primary, 65536) - if not chunk: - break - output += chunk - finally: - os.close(secondary) - os.close(primary) - self.assertEqual(completed.returncode, 0, output.decode(errors="replace")) - self.assertEqual(after, before, "the terminal was left in raw mode") - - def test_run_exposes_restore_readiness(self): - args = nvx.parse_args( - [ - "run", - "--hypervisor", - "mshv", - "--restore-snapshot", - "snapshot", - "--processors", - "4", - "--restore-processors", - "2", - "--restore-ready-path", - "ready.sock", - "--dry-run", - ] - ) - - with ( - patch.object(nvx, "require_file", return_value=Path("openvmm")) as require, - patch.object( - nvx, "_format_command", return_value="formatted" - ) as format_command, - ): - nvx.command_run(args) - - require.assert_called_once() - command = format_command.call_args.args[0] - self.assertEqual( - command, - [ - "openvmm", - "--single-process", - "--machine", - "microvm", - "--processors", - "4", - "--hypervisor", - "mshv", - "--restore-snapshot", - "snapshot", - "--restore-processors", - "2", - "--restore-ready-path", - "ready.sock", - ], - ) - - missing_snapshot = nvx.parse_args( - ["run", "--restore-ready-path", "ready.sock", "--dry-run"] - ) - with self.assertRaisesRegex( - common.ScriptError, "--restore-ready-path requires --restore-snapshot" - ): - nvx.command_run(missing_snapshot) - - missing_processor_snapshot = nvx.parse_args( - ["run", "--restore-processors", "1", "--dry-run"] - ) - with self.assertRaisesRegex( - common.ScriptError, "--restore-processors requires --restore-snapshot" - ): - nvx.command_run(missing_processor_snapshot) - - invalid_restore_capacity = nvx.parse_args( - [ - "run", - "--processors", - "2", - "--restore-snapshot", - "snapshot", - "--restore-processors", - "4", - "--dry-run", - ] - ) - with self.assertRaisesRegex(common.ScriptError, "cannot exceed"): - nvx.command_run(invalid_restore_capacity) - - legacy = nvx.parse_args(["run", "--restore-snapshot", "snapshot", "--dry-run"]) - with ( - patch.object(nvx, "require_file", return_value=Path("openvmm")), - patch.object( - nvx, "_format_command", return_value="formatted" - ) as format_command, - ): - nvx.command_run(legacy) - self.assertIn("microvm", format_command.call_args.args[0]) - self.assertNotIn("microvm-v2", format_command.call_args.args[0]) - - capture_memory = nvx.parse_args( - [ - "run", - "--memory-mib", - "512", - "--memory-capacity-mib", - "2048", - "--dry-run", - ] - ) - with ( - patch.object(nvx, "require_file", return_value=Path("artifact")), - patch.object( - nvx, "_format_command", return_value="formatted" - ) as format_command, - ): - nvx.command_run(capture_memory) - command = format_command.call_args.args[0] - self.assertEqual(command[command.index("--memory") + 1], "512M") - self.assertEqual(command[command.index("--memory-capacity") + 1], "2048M") - - restore_memory = nvx.parse_args( - [ - "run", - "--restore-snapshot", - "snapshot", - "--restore-memory-mib", - "1024", - "--dry-run", - ] - ) - with ( - patch.object(nvx, "require_file", return_value=Path("openvmm")), - patch.object( - nvx, "_format_command", return_value="formatted" - ) as format_command, - ): - nvx.command_run(restore_memory) - command = format_command.call_args.args[0] - self.assertEqual(command[command.index("--restore-memory") + 1], "1024M") - self.assertNotIn("--memory-capacity", command) - - smp = nvx.parse_args( - ["run", "--machine", "microvm", "--processors", "2", "--dry-run"] - ) - self.assertEqual(smp.machine, "microvm") - self.assertEqual(smp.processors, 2) - - with self.assertRaises(SystemExit): - nvx.parse_args( - ["run", "--machine", "microvm-v2", "--processors", "4", "--dry-run"] - ) - - with self.assertRaises(SystemExit): - nvx.parse_args(["run", "--machine", "microvm-v3", "--dry-run"]) - - def test_openvmm_command_passes_restore_choice_in_build_config(self): - with patch.object(nvx, "build_openvmm") as build_openvmm: - nvx.command_build_openvmm(argparse.Namespace(skip_restore=True)) - - config = build_openvmm.call_args.args[0] - self.assertIsInstance(config, build_config.OpenVmmBuildConfig) - self.assertTrue(config.skip_restore) - self.assertIsNone(config.backend) - - def test_build_commands_pass_backend_choice_in_config(self): - for command in ("build-openvmm", "build"): - for backend in (None, "kvm", "mshv", "whp"): - with self.subTest(command=command, backend=backend): - arguments = [command, "--skip-restore"] - if backend is not None: - arguments.extend(["--backend", backend]) - if command == "build": - arguments.append("--native") - args = nvx.parse_args(arguments) - with ( - patch.object(nvx, "build_openvmm") as openvmm, - patch.object(nvx, "build_all") as combined, - ): - args.handler(args) - - if command == "build": - aggregate = combined.call_args.args[0] - self.assertIsInstance(aggregate, build_config.BuildConfig) - self.assertTrue(aggregate.native_guest) - config = aggregate.openvmm - openvmm.assert_not_called() - else: - config = openvmm.call_args.args[0] - combined.assert_not_called() - self.assertIsInstance(config, build_config.OpenVmmBuildConfig) - self.assertEqual(config.backend, backend) - self.assertTrue(config.skip_restore) - - def test_build_commands_reject_unknown_backends(self): - for command in ("build-openvmm", "build"): - with self.subTest(command=command), self.assertRaises(SystemExit): - nvx.parse_args([command, "--backend", "unknown"]) - - def test_build_commands_pass_specialized_configs_to_consumers(self): - with ( - patch.object(nvx, "build_kernel") as build_kernel, - patch.object(nvx, "build_initramfs") as build_initramfs, - patch.object(nvx, "record_openvmm_provenance") as provenance, - patch.object(nvx, "collect_release_sources") as collect_sources, - ): - nvx.command_build_kernel(argparse.Namespace()) - nvx.command_build_initramfs(argparse.Namespace(guest="alpine")) - nvx.command_record_openvmm_provenance(argparse.Namespace()) - nvx.command_collect_sources(argparse.Namespace()) - - self.assertIsInstance( - build_kernel.call_args.args[0], - build_config.KernelBuildConfig, - ) - self.assertFalse(build_kernel.call_args.args[0].debug) - with patch.object(nvx, "build_kernel") as build_debug_kernel: - nvx.command_build_kernel(argparse.Namespace(debug=True)) - self.assertEqual( - build_debug_kernel.call_args.args[0], - build_config.KernelBuildConfig.debug_variant(), - ) - self.assertIsInstance( - build_initramfs.call_args.args[0], - build_config.InitramfsBuildConfig, - ) - self.assertIsInstance( - provenance.call_args.args[0], - build_config.OpenVmmBuildConfig, - ) - self.assertIsInstance( - collect_sources.call_args.args[0], - build_config.DockerBuildConfig, - ) - - def test_record_openvmm_provenance_command_is_exposed(self): - args = nvx.parse_args(["record-openvmm-provenance"]) - self.assertIs(args.handler, nvx.command_record_openvmm_provenance) - - -class CiTests(unittest.TestCase): - def test_openvmm_unit_tests_run_nextest_and_doctests(self): - with tempfile.TemporaryDirectory() as temporary: - openvmm = Path(temporary) / "openvmm" - openvmm.mkdir() - (openvmm / "Cargo.toml").touch() - fuzz_crates = common.CommandResult( - args=("cargo", "xtask", "fuzz", "list", "--crates"), - returncode=0, - stdout=b"fuzz_alpha\nfuzz_beta\n", - stderr=b"", - ) - - with ( - patch.object(OpenVMMBuildConstants, "DIRECTORY", openvmm), - patch.object(ci, "require_tool", return_value="cargo"), - patch.object(ci, "run_capture", return_value=fuzz_crates) as capture, - patch.object(ci, "run_checked") as run_checked, - ): - ci.run_openvmm_unit_tests() - - capture.assert_called_once_with( - ["cargo", "xtask", "fuzz", "list", "--crates"], - cwd=openvmm, - ) - self.assertEqual(run_checked.call_count, 3) - self.assertEqual( - run_checked.call_args_list[0], - call( - [ - "cargo", - "xflowey", - "restore-packages", - "--no-compat-igvm", - ], - cwd=openvmm, - ), - ) - command = run_checked.call_args_list[1].args[0] - self.assertEqual( - command[:10], - [ - "cargo", - "nextest", - "run", - "--profile", - "agent", - "--workspace", - "--tests", - "--bins", - "--features", - "ci", - ], - ) - excluded_packages = [ - command[index + 1] - for index, argument in enumerate(command) - if argument == "--exclude" - ] - self.assertEqual( - excluded_packages, - [ - *ci.OPENVMM_UNIT_TEST_EXCLUDED_PACKAGES, - "fuzz_alpha", - "fuzz_beta", - ], - ) - self.assertEqual(run_checked.call_args_list[1].kwargs["cwd"], openvmm) - self.assertEqual( - run_checked.call_args_list[2], - call( - [ - "cargo", - "test", - "--locked", - "--doc", - "--workspace", - "--no-fail-fast", - ], - cwd=openvmm, - ), - ) - - def test_openvmm_unit_tests_stop_when_fuzz_crate_query_fails(self): - with tempfile.TemporaryDirectory() as temporary: - openvmm = Path(temporary) / "openvmm" - openvmm.mkdir() - (openvmm / "Cargo.toml").touch() - failed_query = common.CommandResult( - args=("cargo", "xtask", "fuzz", "list", "--crates"), - returncode=2, - stdout=b"", - stderr=b"query failed", - ) - - with ( - patch.object(OpenVMMBuildConstants, "DIRECTORY", openvmm), - patch.object(ci, "require_tool", return_value="cargo"), - patch.object(ci, "run_capture", return_value=failed_query), - patch.object(ci, "run_checked") as run_checked, - self.assertRaisesRegex( - common.ScriptError, - "OpenVMM fuzz crate query exited 2", - ), - ): - ci.run_openvmm_unit_tests() - - run_checked.assert_not_called() - - def test_openvmm_tests_use_nvx_guest_artifacts(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - openvmm = root / "openvmm" - openvmm.mkdir() - (openvmm / "Cargo.toml").touch() - kernel = root / "vmlinux" - initrd = root / "initramfs.cpio.gz" - kernel.touch() - initrd.touch() - backend = "whp" if os.name == "nt" else "kvm" - installed_targets = common.CommandResult( - args=("rustup", "target", "list"), - returncode=0, - stdout=( - "\n".join(OpenVMMBuildConstants.TEST_RUST_TARGETS[backend]) + "\n" - ).encode(), - stderr=b"", - ) - - with ( - patch.object(OpenVMMBuildConstants, "DIRECTORY", openvmm), - patch.object( - ci, - "artifact_path", - side_effect=[kernel, initrd], - ), - patch.object(ci.os, "access", return_value=True), - patch.object(ci.Path, "exists", return_value=False), - patch.object(ci, "require_tool", side_effect=["cargo", "rustup"]), - patch.object( - ci, - "run_capture", - return_value=installed_targets, - ) as run_capture, - patch.object(ci, "run_checked") as run_checked, - patch.dict( - os.environ, - { - "PETRI_CAPABILITIES": "vpci", - "RUNNER_TEMP": os.fspath(root), - }, - ), - ): - ci.run_openvmm_tests(backend) - - run_capture.assert_called_once_with( - [ - "rustup", - "target", - "list", - "--installed", - "--toolchain", - OpenVMMBuildConstants.RUST_TOOLCHAIN, - ] - ) - self.assertEqual(run_checked.call_count, 2) - restore, tests = run_checked.call_args_list - self.assertEqual( - restore.args[0], - ["cargo", "xflowey", "restore-packages", "--no-compat-igvm"], - ) - environment = restore.kwargs["env"] - self.assertEqual( - environment["RUSTUP_TOOLCHAIN"], - OpenVMMBuildConstants.RUST_TOOLCHAIN, - ) - if os.name == "nt": - self.assertNotIn("XDG_CACHE_HOME", environment) - else: - self.assertEqual( - environment["XDG_CACHE_HOME"], - os.fspath(root / "openvmm-cache"), - ) - command = tests.args[0] - self.assertEqual(command[:3], ["cargo", "xflowey", "vmm-tests-run"]) - filter_index = command.index("--filter") - self.assertEqual( - command[filter_index + 1], - ci.OPENVMM_TEST_FILTERS[backend], - ) - self.assertEqual(tests.kwargs["cwd"], openvmm) - self.assertIs(tests.kwargs["env"], environment) - self.assertEqual( - environment["OPENVMM_MICROVM_TEST_KERNEL"], - os.fspath(kernel.resolve()), - ) - self.assertEqual( - environment["OPENVMM_MICROVM_TEST_INITRD"], - os.fspath(initrd.resolve()), - ) - self.assertEqual(environment["PETRI_CAPABILITIES"], "vpci") - if os.name == "nt": - self.assertEqual( - command[command.index("--dir") + 1], - os.fspath(root / backend), - ) - - def test_openvmm_tests_pin_toolchain_without_runner_temp(self): - installed_targets = common.CommandResult( - args=("rustup", "target", "list"), - returncode=0, - stdout=( - "\n".join(OpenVMMBuildConstants.TEST_RUST_TARGETS["whp"]) + "\n" - ).encode(), - stderr=b"", - ) - - with ( - patch.object(ci, "run_capture", return_value=installed_targets), - patch.object(ci, "run_checked") as run_checked, - patch.dict(os.environ, {}, clear=True), - ): - environment = ci._prepare_openvmm_test_environment("whp", "rustup") - - self.assertEqual( - environment, - {"RUSTUP_TOOLCHAIN": OpenVMMBuildConstants.RUST_TOOLCHAIN}, - ) - run_checked.assert_not_called() - - def test_openvmm_tests_prepare_job_local_toolchain_for_missing_targets(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - openvmm = root / "openvmm" - openvmm.mkdir() - (openvmm / "Cargo.toml").touch() - kernel = root / "vmlinux" - initrd = root / "initramfs.cpio.gz" - kernel.touch() - initrd.touch() - backend = "whp" if os.name == "nt" else "kvm" - installed_targets = common.CommandResult( - args=("rustup", "target", "list"), - returncode=0, - stdout=f"{OpenVMMBuildConstants.GUEST_RUST_TARGET}\n".encode(), - stderr=b"", - ) - - with ( - patch.object(OpenVMMBuildConstants, "DIRECTORY", openvmm), - patch.object( - ci, - "artifact_path", - side_effect=[kernel, initrd], - ), - patch.object(ci.os, "access", return_value=True), - patch.object(ci.Path, "exists", return_value=False), - patch.object(ci, "require_tool", side_effect=["cargo", "rustup"]), - patch.object(ci, "run_capture", return_value=installed_targets), - patch.object(ci, "run_checked") as run_checked, - patch.dict(os.environ, {"RUNNER_TEMP": os.fspath(root)}), - ): - ci.run_openvmm_tests(backend) - - self.assertEqual(run_checked.call_count, 4) - install_toolchain, install_targets, restore, tests = ( - run_checked.call_args_list - ) - environment = install_toolchain.kwargs["env"] - self.assertEqual( - environment["RUSTUP_HOME"], - os.fspath(root / "openvmm-rustup"), - ) - self.assertEqual( - environment["RUSTUP_TOOLCHAIN"], - OpenVMMBuildConstants.RUST_TOOLCHAIN, - ) - if os.name != "nt": - self.assertEqual( - environment["XDG_CACHE_HOME"], - os.fspath(root / "openvmm-cache"), - ) - self.assertEqual( - install_toolchain.args[0], - [ - "rustup", - "toolchain", - "install", - OpenVMMBuildConstants.RUST_TOOLCHAIN, - "--profile", - "minimal", - ], - ) - self.assertEqual( - install_targets.args[0], - [ - "rustup", - "target", - "add", - *OpenVMMBuildConstants.TEST_RUST_TARGETS[backend], - "--toolchain", - OpenVMMBuildConstants.RUST_TOOLCHAIN, - ], - ) - for command in (install_targets, restore, tests): - self.assertIs(command.kwargs["env"], environment) - - def test_openvmm_tests_install_missing_pinned_toolchain(self): - missing_toolchain = common.CommandResult( - args=("rustup", "target", "list"), - returncode=1, - stdout=b"", - stderr=( - f"error: toolchain '{OpenVMMBuildConstants.RUST_TOOLCHAIN}' " - "is not installed\n" - ).encode(), - ) - install_toolchain = [ - "rustup", - "toolchain", - "install", - OpenVMMBuildConstants.RUST_TOOLCHAIN, - "--profile", - "minimal", - ] - add_targets = [ - "rustup", - "target", - "add", - *OpenVMMBuildConstants.TEST_RUST_TARGETS["kvm"], - "--toolchain", - OpenVMMBuildConstants.RUST_TOOLCHAIN, - ] - - with tempfile.TemporaryDirectory() as temporary: - for runner_temp in (None, temporary): - with ( - self.subTest(runner_temp=runner_temp), - patch.object(ci, "run_capture", return_value=missing_toolchain), - patch.object(ci, "run_checked") as run_checked, - patch.dict( - os.environ, - {} if runner_temp is None else {"RUNNER_TEMP": runner_temp}, - clear=True, - ), - ): - environment = ci._prepare_openvmm_test_environment("kvm", "rustup") - - self.assertEqual( - [command.args[0] for command in run_checked.call_args_list], - [install_toolchain, add_targets], - ) - for command in run_checked.call_args_list: - self.assertIs(command.kwargs["env"], environment) - self.assertEqual( - environment["RUSTUP_TOOLCHAIN"], - OpenVMMBuildConstants.RUST_TOOLCHAIN, - ) - if runner_temp is None: - self.assertNotIn("RUSTUP_HOME", environment) - else: - self.assertEqual( - environment["RUSTUP_HOME"], - os.fspath( - Path(runner_temp) - / OpenVMMBuildConstants.RUSTUP_DIRECTORY_NAME - ), - ) - - def test_openvmm_tests_define_each_backend_filter(self): - self.assertEqual( - set(ci.OPENVMM_TEST_FILTERS), - set(ci.OPENVMM_TEST_BACKENDS), - ) - self.assertEqual( - set(OpenVMMBuildConstants.TEST_RUST_TARGETS), - set(ci.OPENVMM_TEST_BACKENDS), - ) - self.assertEqual( - ci.OPENVMM_TEST_FILTERS["kvm"], - ci.OPENVMM_KVM_TEST_FILTER, - ) - self.assertIn( - "!test(no_vmbus_prepped_boot_no_vmbus_windows)", - ci.OPENVMM_KVM_TEST_FILTER, - ) - self.assertIn( - "!test(windows_datacenter_core_2022_x64)", - ci.OPENVMM_KVM_TEST_FILTER, - ) - self.assertIn("!test(virtio_net_windows)", ci.OPENVMM_KVM_TEST_FILTER) - self.assertEqual( - set(ci.OPENVMM_KVM_EXCLUDED_TESTS), - { - "multiarch::openvmm_pcat_x64_freebsd_13_2_x64_boot_no_agent", - "multiarch::openvmm_pcat_x64_freebsd_13_2_x64_iso_boot_no_agent", - "multiarch::openvmm_pcat_x64_ubuntu_2404_server_x64_boot", - "multiarch::openvmm_pcat_x64_ubuntu_2504_server_x64_boot", - "multiarch::openvmm_pcat_x64_ubuntu_2504_server_x64_boot_heavy", - }, - ) - for excluded_test in ci.OPENVMM_KVM_EXCLUDED_TESTS: - self.assertIn( - f"!{ci._exact_openvmm_test(excluded_test)}", - ci.OPENVMM_KVM_TEST_FILTER, - ) - self.assertNotIn("!test(openvmm_pcat_x64)", ci.OPENVMM_KVM_TEST_FILTER) - self.assertNotIn( - "openvmm_linux_x64_apicid_offset", - ci.OPENVMM_KVM_TEST_FILTER, - ) - self.assertNotIn( - "openvmm_linux_x64_legacy_xapic", - ci.OPENVMM_KVM_TEST_FILTER, - ) - self.assertEqual( - ci.OPENVMM_TEST_FILTERS["mshv"], - ci.OPENVMM_MSHV_TEST_FILTER, - ) - self.assertIn( - "!test(windows_datacenter_core_2022_x64)", - ci.OPENVMM_MSHV_TEST_FILTER, - ) - self.assertEqual( - set(ci.OPENVMM_MSHV_EXCLUDED_TESTS), - { - "multiarch::openvmm_pcat_x64_freebsd_13_2_x64_boot_no_agent", - "multiarch::openvmm_pcat_x64_freebsd_13_2_x64_iso_boot_no_agent", - "multiarch::openvmm_pcat_x64_ubuntu_2404_server_x64_boot", - "multiarch::openvmm_pcat_x64_ubuntu_2504_server_x64_boot", - "multiarch::openvmm_pcat_x64_ubuntu_2504_server_x64_boot_heavy", - "multiarch::pcie::openvmm_linux_x64_pcie_save_restore", - }, - ) - self.assertNotIn( - "x86_64::openvmm_linux_x64_virtio_blk_device", - ci.OPENVMM_MSHV_EXCLUDED_TESTS, - ) - for excluded_test in ci.OPENVMM_MSHV_EXCLUDED_TESTS: - self.assertIn( - f"!{ci._exact_openvmm_test(excluded_test)}", - ci.OPENVMM_MSHV_TEST_FILTER, - ) - self.assertNotIn("!test(openvmm_pcat_x64)", ci.OPENVMM_MSHV_TEST_FILTER) - for backend, test_filter in ci.OPENVMM_TEST_FILTERS.items(): - for required_test in ci.OPENVMM_REQUIRED_MICROVM_TESTS: - with self.subTest(backend=backend, required_test=required_test): - self.assertIn( - ci._exact_openvmm_test(required_test), - test_filter, - ) - self.assertEqual(len(ci.OPENVMM_WHP_TESTS), 29) - self.assertEqual( - len(set(ci.OPENVMM_WHP_TESTS)), - len(ci.OPENVMM_WHP_TESTS), - ) - for required_test in ci.OPENVMM_REQUIRED_MICROVM_TESTS: - self.assertIn(required_test, ci.OPENVMM_WHP_TESTS) - # Only Linux hosts can build the Linux pipette this TTRPC test boots. - self.assertNotIn("test_ttrpc_interface", ci.OPENVMM_TEST_FILTERS["whp"]) - for backend in ("kvm", "mshv"): - with self.subTest(backend=backend): - self.assertIn("test(ttrpc)", ci.OPENVMM_TEST_FILTERS[backend]) - self.assertNotIn( - "test_ttrpc_interface", - ci.OPENVMM_TEST_FILTERS[backend], - ) - self.assertEqual(ci.OPENVMM_WHP_EXCLUDED_TESTS, ()) - self.assertNotIn( - "multiarch::openvmm_pcat_x64_windows_datacenter_core_2022_x64_boot_heavy", - ci.OPENVMM_WHP_EXCLUDED_TESTS, - ) - self.assertNotIn( - "multiarch::openvmm_pcat_x64_windows_datacenter_core_2022_x64_boot", - ci.OPENVMM_WHP_EXCLUDED_TESTS, - ) - self.assertNotIn( - "multiarch::openvmm_pcat_x64_freebsd_13_2_x64_iso_boot_no_agent", - ci.OPENVMM_WHP_EXCLUDED_TESTS, - ) - self.assertNotIn( - "multiarch::openvmm_pcat_x64_freebsd_13_2_x64_boot_no_agent", - ci.OPENVMM_WHP_EXCLUDED_TESTS, - ) - self.assertNotIn( - "multiarch::pcie::openvmm_uefi_x64_windows_datacenter_core_2022_x64_pcie_nvme_boot", - ci.OPENVMM_WHP_EXCLUDED_TESTS, - ) - self.assertTrue( - set(ci.OPENVMM_WHP_EXCLUDED_TESTS).issubset(ci.OPENVMM_WHP_TESTS) - ) - self.assertEqual( - ci.OPENVMM_TEST_FILTERS["whp"], - ci._join_openvmm_tests( - ci.OPENVMM_WHP_TESTS, - ci.OPENVMM_WHP_EXCLUDED_TESTS, - ), - ) - self.assertEqual( - ci._join_openvmm_tests( - ("suite::boot", "suite::boot_heavy"), - ("suite::boot_heavy",), - ), - ( - "(test(/^suite::boot$/) | test(/^suite::boot_heavy$/))" - " & !test(/^suite::boot_heavy$/)" - ), - ) - - def test_openvmm_tests_reject_unknown_backend(self): - with self.assertRaisesRegex(common.ScriptError, "unsupported.*backend"): - ci.run_openvmm_tests("unknown") - - -class CiConfigurationTests(unittest.TestCase): - def test_ci_needs_reference_declared_jobs(self): - workflow = ( - BuildConstants.REPO_ROOT / ".github" / "workflows" / "ci.yml" - ).read_text(encoding="utf-8") - jobs = set( - re.findall( - r"^ ([A-Za-z0-9_-]+):$", - workflow.split("jobs:\n", 1)[1], - re.MULTILINE, - ) - ) - for job_name in sorted(jobs): - job = _workflow_job(workflow, job_name) - match = re.search(r"^ needs:(.*)$", job, re.MULTILINE) - if match is None: - continue - inline = match.group(1).strip() - if inline: - dependencies = {name.strip() for name in inline.strip("[]").split(",")} - else: - dependencies = set( - re.findall( - r"^ - ([A-Za-z0-9_-]+)$", - job.split(" if:", 1)[0], - re.MULTILINE, - ) - ) - with self.subTest(job=job_name): - self.assertLessEqual( - dependencies, - jobs, - f"undefined job dependencies: {dependencies - jobs}", - ) - required = _workflow_job(workflow, "required-status-check") - self.assertIn(" - aci-edge-sandboxes\n", required) - - def test_development_release_requires_successful_crate_checks(self): - workflow = ( - BuildConstants.REPO_ROOT / ".github" / "workflows" / "ci.yml" - ).read_text(encoding="utf-8") - release_job = _workflow_job(workflow, "release") - self.assertIn(" - aci-edge-sandboxes\n", release_job) - predicate = release_job.split(" if:", 1)[1].split(" runs-on:", 1)[0] - self.assertIn("needs.aci-edge-sandboxes.result == 'success' &&", predicate) - self.assertNotIn("needs.aci-edge-sandboxes.result == 'skipped'", predicate) - - def test_required_ci_result_policy(self): - always_successful = {"quality", "aci-edge-sandboxes", "openvmm-changes"} - # Jobs that succeed whenever the repository's jobs run. - repository = always_successful | set(ci.REQUIRED_CI_CLI_JOBS) - builds = set(ci.REQUIRED_CI_BUILD_JOBS) - openvmm_tests = set(ci.REQUIRED_CI_OPENVMM_TEST_JOBS) - openvmm_artifact_tests = set(ci.REQUIRED_CI_OPENVMM_ARTIFACT_TEST_JOBS) - microvm_tests = set(ci.REQUIRED_CI_MICROVM_TEST_JOBS) - debug_kernel = set(ci.REQUIRED_CI_MICROVM_DEBUG_JOBS) - debug_kernel_push = set(ci.REQUIRED_CI_MICROVM_DEBUG_PUSH_JOBS) - artifacts = {ci.REQUIRED_CI_ARTIFACT_JOB, ci.REQUIRED_CI_DEBUG_KERNEL_JOB} - platforms = set(ci.REQUIRED_CI_PLATFORM_JOBS) - workloads = artifacts | builds | platforms - microvm = microvm_tests | debug_kernel - # Event, same repository, the change detector's outputs (tests, - # workloads, OpenVMM unit tests, OpenVMM VMM tests), and the jobs that - # must succeed; every other required job must be skipped. - cases = ( - ("pull_request", True, (False, False, False, False), repository), - ( - "pull_request", - True, - (False, True, False, False), - repository | workloads | {"performance-gate"}, - ), - ( - "pull_request", - True, - (True, False, True, True), - repository | builds | openvmm_tests, - ), - ( - "pull_request", - True, - (True, True, True, True), - repository - | workloads - | openvmm_tests - | openvmm_artifact_tests - | microvm - | {"performance-gate"}, - ), - # A pull request that changes no input of either OpenVMM suite. - ( - "pull_request", - True, - (True, True, False, False), - repository | workloads | microvm | {"performance-gate"}, - ), - # Guest inputs reach the VMM tests but not the unit tests. - ( - "pull_request", - True, - (True, True, False, True), - repository - | workloads - | openvmm_artifact_tests - | microvm - | {"performance-gate"}, - ), - ("pull_request", False, (False, False, False, False), always_successful), - ( - "pull_request", - False, - (False, True, False, False), - always_successful | artifacts, - ), - ("pull_request", False, (True, False, True, True), always_successful), - ( - "pull_request", - False, - (True, True, True, True), - always_successful | artifacts, - ), - ("push", True, (False, False, False, False), repository), - ("push", True, (False, True, False, False), repository | workloads), - ( - "push", - True, - (True, False, True, True), - repository | builds | openvmm_tests, - ), - ( - "push", - True, - (True, True, True, True), - repository - | workloads - | openvmm_tests - | openvmm_artifact_tests - | microvm - | debug_kernel_push, - ), - ) - - for event_name, same_repository, flags, successful_jobs in cases: - changes = ci.CiChanges(*flags) - with self.subTest( - event_name=event_name, - same_repository=same_repository, - changes=changes, - ): - expected = ci.required_ci_expected_results( - event_name, - same_repository=same_repository, - changes=changes, - ) - self.assertEqual(set(expected), set(ci.REQUIRED_CI_RESULT_ENVIRONMENTS)) - self.assertEqual( - {job for job, result in expected.items() if result == "success"}, - successful_jobs, - ) - self.assertEqual( - ci.required_ci_failures( - event_name, - same_repository=same_repository, - changes=changes, - results=expected, - ), - [], - ) - - for job, expected_result in expected.items(): - unexpected = expected.copy() - unexpected[job] = ( - "skipped" if expected_result == "success" else "success" - ) - with self.subTest(job=job, actual=unexpected[job]): - self.assertEqual( - ci.required_ci_failures( - event_name, - same_repository=same_repository, - changes=changes, - results=unexpected, - ), - [ - f"{job}: expected {expected_result}, " - f"got {unexpected[job]}" - ], - ) - - for job in expected: - failed = expected.copy() - failed[job] = "failure" - with self.subTest(job=job, actual="failure"): - self.assertEqual( - ci.required_ci_failures( - event_name, - same_repository=same_repository, - changes=changes, - results=failed, - ), - [f"{job}: expected {expected[job]}, got failure"], - ) - - def test_required_ci_job_uses_tested_policy(self): - workflow = ( - BuildConstants.REPO_ROOT / ".github" / "workflows" / "ci.yml" - ).read_text(encoding="utf-8") - job = _workflow_job(workflow, "required-status-check") - - self.assertIn("python3 scripts/nvx.py check-required-ci", job) - self.assertIn('--same-repository "${SAME_REPOSITORY}"', job) - self.assertIn( - "github.event.pull_request.head.repo.full_name == github.repository", - job, - ) - self.assertLess( - job.index("uses: actions/checkout@v5"), - job.index("python3 scripts/nvx.py check-required-ci"), - ) - self.assertIn("persist-credentials: false", job) - for environment in ci.REQUIRED_CI_RESULT_ENVIRONMENTS.values(): - with self.subTest(environment=environment): - self.assertIn(f" {environment}:", job) - # Every suite flag reaches the policy as the detector reported it. - detector = _workflow_job(workflow, "openvmm-changes") - for output in ci.CI_CHANGE_OUTPUTS: - environment = output.upper().replace("-", "_") - with self.subTest(output=output): - self.assertIn( - f" {output}: ${{{{ steps.changes.outputs.{output} }}}}", - detector, - ) - self.assertIn( - f" {environment}: " - f"${{{{ needs.openvmm-changes.outputs.{output} }}}}", - job, - ) - self.assertIn(f'--{output} "${{{environment}}}"', job) - - def test_ci_change_classification_selects_suites_by_input(self): - # The pull-request outputs (tests, workloads, OpenVMM unit tests, - # OpenVMM VMM tests) for each class of changed path. - cases = ( - ((), (False, True, False, False)), - (("doc/ci.md", "README.md", "DOC/Guide.MD"), (False, False, False, False)), - (("openvmm",), (True, True, True, True)), - (("rust-toolchain.toml",), (True, True, True, True)), - (("scripts/setup/setup-windows-whp.ps1",), (True, True, True, True)), - (("scripts/setup/README.md",), (True, False, True, True)), - ((".github/workflows/ci.yml",), (True, True, True, True)), - ((".github/actions/validate-runner/action.yml",), (True, True, True, True)), - ((".github/actions/sccache/action.yml",), (True, True, True, True)), - ((".github/actions/setup-curl/curl-shim.rs",), (True, True, True, True)), - (("scripts/nvx.py",), (True, True, True, True)), - (("scripts/nvx_tools/ci.py",), (True, True, True, True)), - (("scripts/nvx_tools/common.py",), (True, True, True, True)), - (("scripts/nvx_tools/build.py",), (True, True, False, True)), - (("scripts/nvx_tools/guests.py",), (True, True, False, True)), - (("kernel/config-microvm",), (True, True, False, True)), - (("kernel/patches/0001-example.patch",), (True, True, False, True)), - (("guest/common/init",), (True, True, False, True)), - (("guest/alpine/nvx-container-enter",), (True, True, False, True)), - (("SOURCE-MANIFEST.json",), (True, True, False, True)), - (("docker/Dockerfile",), (True, True, False, True)), - ( - (".github/actions/build-guest-artifacts/action.yml",), - (True, True, False, True), - ), - (("guest/ubuntu/nvx-bashrc",), (True, True, False, False)), - (("ubuntu/packages.lock.json",), (True, True, False, False)), - (("azurelinux/packages.lock.json",), (True, True, False, False)), - (("aci_edge_sandboxes/src/lib.rs",), (True, True, False, False)), - (("scripts/nvx_tools/microvm_tests.py",), (True, True, False, False)), - ( - (".github/workflows/run-nvx-microvm-tests.yml",), - (True, True, False, False), - ), - ((".github/actions/build-openvmm/action.yml",), (True, True, False, False)), - ((".github/actions/validate-nvx/action.yml",), (False, True, False, False)), - ( - (".github/actions/run-benchmark/action.yml",), - (False, True, False, False), - ), - (("scripts/test_nvx_tools.py",), (False, True, False, False)), - (("data/linux-kvm-virtual-machine.csv",), (False, True, False, False)), - ( - ("scripts/nvx_tools/microvm_tests.py", "kernel/config-microvm"), - (True, True, False, True), - ), - ) - for changed_files, flags in cases: - with self.subTest(changed_files=changed_files): - pull_request = ci.classify_ci_changes("pull_request", changed_files) - self.assertEqual(pull_request, ci.CiChanges(*flags)) - # Pushes keep running both OpenVMM suites with the tests. - run_tests, run_workloads = flags[:2] - self.assertEqual( - ci.classify_ci_changes("push", changed_files), - ci.CiChanges(run_tests, run_workloads, run_tests, run_tests), - ) - self.assertEqual( - ci.classify_ci_changes("pull_request", ["openvmm", ""]), - ci.classify_ci_changes("pull_request", ["openvmm"]), - ) - with self.assertRaisesRegex(ValueError, "unsupported CI event"): - ci.classify_ci_changes("workflow_dispatch", ["openvmm"]) - - def test_ci_change_classification_covers_each_suite_input(self): - github = BuildConstants.REPO_ROOT / ".github" - tools = BuildConstants.REPO_ROOT / "scripts" / "nvx_tools" - workflow = (github / "workflows" / "ci.yml").read_text(encoding="utf-8") - - def local_actions(configuration: str) -> set[str]: - names = set( - re.findall(r"uses: \./\.github/actions/([a-z0-9-]+)", configuration) - ) - pending = list(names) - while pending: - action = (github / "actions" / pending.pop() / "action.yml").read_text( - encoding="utf-8" - ) - for name in local_actions(action) - names: - names.add(name) - pending.append(name) - return names - - def imported_modules(module: str) -> set[str]: - modules = {module} - pending = [module] - while pending: - tree = ast.parse( - (tools / f"{pending.pop()}.py").read_text(encoding="utf-8") - ) - for node in ast.walk(tree): - if not isinstance(node, ast.ImportFrom) or node.level != 1: - continue - names = ( - [node.module.split(".")[0]] - if node.module - else [alias.name for alias in node.names] - ) - for name in set(names) - modules: - modules.add(name) - pending.append(name) - return modules - - command_inputs = { - "scripts/nvx.py", - "scripts/nvx_tools/__init__.py", - ".github/workflows/ci.yml", - *(f"scripts/nvx_tools/{name}.py" for name in imported_modules("ci")), - } - self.assertLessEqual( - {"ci", "common", "build_constants"}, imported_modules("ci") - ) - unit_inputs = command_inputs | { - f".github/actions/{name}/action.yml" - for name in local_actions(_workflow_job(workflow, "openvmm-unit-tests")) - } - vmm_inputs = command_inputs | { - f".github/actions/{name}/action.yml" - for job in ("openvmm-vmm-tests", "artifacts") - for name in local_actions(_workflow_job(workflow, job)) - } - # Every cache key input of the kernel and Alpine initramfs that the - # VMM tests boot, and of every guest that the microVM tests boot. - guest_action = ( - github / "actions" / "build-guest-artifacts" / "action.yml" - ).read_text(encoding="utf-8") - guest_inputs = { - match.group(1): { - path.replace("**", "example").replace("*", "example") - for path in re.findall(r"'([^']+)'", match.group(2)) - } - for match in re.finditer( - r"^\s+([A-Z_]+)_INPUT_HASH: \$\{\{ hashFiles\((.+)\) \}\}$", - guest_action, - re.MULTILINE, - ) - } - self.assertEqual( - set(guest_inputs), - {"KERNEL", "DEBUG_KERNEL", "ALPINE", "UBUNTU", "AZURELINUX"}, - ) - vmm_inputs |= guest_inputs["KERNEL"] | guest_inputs["ALPINE"] - test_inputs = vmm_inputs.union(*guest_inputs.values()) | { - ".github/workflows/run-nvx-microvm-tests.yml", - ".github/workflows/build-openvmm-binary.yml", - *( - f".github/actions/{name}/action.yml" - for configuration in ( - (github / "workflows" / "run-nvx-microvm-tests.yml").read_text( - encoding="utf-8" - ), - (github / "workflows" / "build-openvmm-binary.yml").read_text( - encoding="utf-8" - ), - _workflow_job(workflow, "debug-kernel"), - ) - for name in local_actions(configuration) - ), - } - self.assertNotIn(".github/actions/validate-nvx/action.yml", test_inputs) - for pattern, inputs in ( - (ci.OPENVMM_UNIT_TEST_INPUT_PATHS, unit_inputs), - (ci.OPENVMM_VMM_TEST_INPUT_PATHS, vmm_inputs), - (ci.CI_TEST_INPUT_PATHS, test_inputs), - ): - for path in sorted(inputs): - with self.subTest(pattern=pattern.pattern, path=path): - self.assertIsNotNone(pattern.match(path)) - - def test_ci_change_summary_explains_each_decision(self): - changed_files = [ - "scripts/nvx_tools/microvm_tests.py", - "guest/common/init", - "guest/common/nvx-hostmount", - "guest/common/nvx-snapshot", - "guest/common/nvx-identity-probe", - ] - changes = ci.classify_ci_changes("pull_request", changed_files) - summary = ci.describe_ci_changes("pull_request", changed_files, changes) - self.assertIn("### Artifact-backed tests and benchmarks: run\n", summary) - self.assertIn("### NVX microVM tests: run\n", summary) - self.assertIn( - "### OpenVMM unit tests: skip\nNo input of this suite changed.\n", - summary, - ) - self.assertIn( - "### OpenVMM VMM tests: run\nSuite inputs changed: " - "`guest/common/init`, `guest/common/nvx-hostmount`, " - "`guest/common/nvx-snapshot`, and 1 more.\n", - summary, - ) - push = ci.describe_ci_changes( - "push", changed_files, ci.classify_ci_changes("push", changed_files) - ) - self.assertIn( - "### OpenVMM unit tests: run\n" - "Pushes run every OpenVMM suite whenever the tests run.\n", - push, - ) - documentation = ci.describe_ci_changes( - "pull_request", - ["doc/ci.md"], - ci.classify_ci_changes("pull_request", ["doc/ci.md"]), - ) - self.assertIn( - "### Artifact-backed tests and benchmarks: skip\n" - "Only documentation files changed.\n", - documentation, - ) - self.assertIn( - "### OpenVMM VMM tests: skip\nNo test input changed.\n", documentation - ) - - def test_classify_ci_changes_command_prints_every_output(self): - with tempfile.TemporaryDirectory() as temporary: - changed_files = Path(temporary) / "changed-files" - changed_files.write_text("guest/common/init\n", encoding="utf-8") - summary = Path(temporary) / "summary.md" - summary.write_text("before\n", encoding="utf-8") - args = nvx.parse_args( - [ - "classify-ci-changes", - "--event-name", - "pull_request", - "--changed-files", - os.fspath(changed_files), - "--summary", - os.fspath(summary), - ] - ) - stdout = io.StringIO() - with contextlib.redirect_stdout(stdout): - args.handler(args) - self.assertEqual( - stdout.getvalue().splitlines(), - [ - "run-tests=true", - "run-workloads=true", - "run-openvmm-unit-tests=false", - "run-openvmm-vmm-tests=true", - ], - ) - text = summary.read_text(encoding="utf-8") - self.assertTrue(text.startswith("before\n### ")) - self.assertIn("### OpenVMM VMM tests: run\n", text) - - def test_classify_ci_changes_command_reads_null_delimited_paths(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - changed_files = root / "changed-files" - changed_files.write_bytes( - "doc/notes.md\0kernel/patches/é\r\n.patch\0".encode() - ) - args = nvx.parse_args( - [ - "classify-ci-changes", - "--event-name", - "pull_request", - "--changed-files", - os.fspath(changed_files), - "--null", - "--summary", - os.fspath(root / "summary"), - ] - ) - stdout = io.StringIO() - with contextlib.redirect_stdout(stdout): - args.handler(args) - self.assertEqual( - stdout.getvalue().splitlines(), - [ - "run-tests=true", - "run-workloads=true", - "run-openvmm-unit-tests=false", - "run-openvmm-vmm-tests=true", - ], - ) - summary = (root / "summary").read_bytes().decode("utf-8") - self.assertIn("kernel/patches/é\r\n.patch", summary) - - @unittest.skipUnless(os.name == "posix", "requires bash and POSIX filenames") - def test_ci_change_detector_classifies_real_git_paths(self): - workflow = ( - BuildConstants.REPO_ROOT / ".github" / "workflows" / "ci.yml" - ).read_text(encoding="utf-8") - step = _workflow_step(workflow, "openvmm-changes", "Detect relevant changes") - lines = step.splitlines() - start = lines.index(" run: |") + 1 - script = "\n".join(line[10:] for line in lines[start:]) - classifier = BuildConstants.REPO_ROOT / "scripts" / "nvx.py" - shim = ( - f"python3() {{ {shlex.quote(sys.executable)} " - f'{shlex.quote(os.fspath(classifier))} "${{@:2}}"; }}\n' - ) - cases = ( - ("rename", "doc/input.patch", True), - ("delete", None, True), - ("unicode", "kernel/patches/é.patch", True), - ("newline", "kernel/patches/line\nbreak.patch", True), - ("documentation", "doc/notes.md", False), - ) - for name, destination, test_input in cases: - with self.subTest(name=name), tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - - def git(*arguments: str, directory: Path = root) -> str: - result = subprocess.run( - ["git", *arguments], - cwd=directory, - capture_output=True, - text=True, - check=True, - ) - return result.stdout.strip() - - git("init", "--quiet") - git("config", "user.name", "Reproduction") - git("config", "user.email", "repro@example.invalid") - git("config", "commit.gpgsign", "false") - source = root / "kernel" / "patches" / "input.patch" - source.parent.mkdir(parents=True) - source.write_text("suite input\n", encoding="utf-8") - git("add", ".") - git("commit", "--quiet", "-m", "baseline") - base = git("rev-parse", "HEAD") - if name == "delete": - source.unlink() - else: - assert destination is not None - target = root / destination - target.parent.mkdir(parents=True, exist_ok=True) - if name == "rename": - source.rename(target) - else: - target.write_text("new input\n", encoding="utf-8") - git("add", "--all") - git("commit", "--quiet", "-m", "changed paths") - for event in ("pull_request", "push"): - with self.subTest(event=event): - output = root / f"output-{event}" - environment = os.environ.copy() - environment.update( - EVENT_NAME=event, - BASE_SHA=base, - HEAD_SHA=git("rev-parse", "HEAD"), - GITHUB_OUTPUT=os.fspath(output), - GITHUB_STEP_SUMMARY=os.fspath(root / "summary"), - RUNNER_TEMP=temporary, - ) - result = subprocess.run( - ["bash", "-c", shim + script], - cwd=root, - env=environment, - capture_output=True, - text=True, - check=False, - ) - self.assertEqual(result.returncode, 0, result.stderr) - actual = dict( - line.split("=", 1) - for line in output.read_text(encoding="utf-8").splitlines() - ) - expected = { - "run-tests": str(test_input).lower(), - "run-workloads": str(test_input).lower(), - "run-openvmm-unit-tests": str( - test_input and event == "push" - ).lower(), - "run-openvmm-vmm-tests": str(test_input).lower(), - } - self.assertEqual(actual, expected) - - @unittest.skipUnless(os.name == "posix", "requires bash") - def test_ci_change_detector_runs_the_full_matrix_unless_classified(self): - workflow = ( - BuildConstants.REPO_ROOT / ".github" / "workflows" / "ci.yml" - ).read_text(encoding="utf-8") - step = _workflow_step(workflow, "openvmm-changes", "Detect relevant changes") - lines = step.splitlines() - start = lines.index(" run: |") + 1 - script = "\n".join(line[10:] for line in lines[start:]) - stub = """ -git() { - case "$1" in - cat-file) return "${GIT_CAT_FILE_STATUS}" ;; - diff) - printf '%b' "${CHANGED_FILES}" - return "${GIT_DIFF_STATUS}" - ;; - esac - echo "unexpected git command: $*" >&2 - return 1 -} -python3() { - if [[ -n "${CLASSIFIER:-}" ]]; then - "${CLASSIFIER}" "$@" - return - fi - printf '%b' "${CLASSIFICATION}" - return "${CLASSIFIER_STATUS}" -} -""" - complete = ( - "run-tests=true\\nrun-workloads=true\\n" - "run-openvmm-unit-tests=false\\nrun-openvmm-vmm-tests=false\\n" - ) - everything = dict.fromkeys(ci.CI_CHANGE_OUTPUTS, "true") - classified = { - "run-tests": "true", - "run-workloads": "true", - "run-openvmm-unit-tests": "false", - "run-openvmm-vmm-tests": "false", - } - cases = ( - ("missing base", {"BASE_SHA": ""}, everything), - ("zero base", {"BASE_SHA": "0" * 40}, everything), - ("missing commit", {"GIT_CAT_FILE_STATUS": "1"}, everything), - ("failed diff", {"GIT_DIFF_STATUS": "128"}, everything), - ("failed classifier", {"CLASSIFIER_STATUS": "1"}, everything), - ( - "missing output", - {"CLASSIFICATION": complete.rsplit("run-openvmm-vmm", 1)[0]}, - everything, - ), - ( - "duplicate output", - {"CLASSIFICATION": complete + "run-tests=false\\n"}, - everything, - ), - ( - "malformed output", - {"CLASSIFICATION": complete.replace("=true", "=yes", 1)}, - everything, - ), - ( - "classified", - {"CLASSIFICATION": "notice: ignored\\n" + complete}, - classified, - ), - ( - "real classifier", - { - "CLASSIFIER": sys.executable, - "CHANGED_FILES": "scripts/nvx_tools/microvm_tests.py\\0", - }, - classified, - ), - ) - for name, overrides, expected in cases: - with self.subTest(name=name), tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - environment = os.environ.copy() - environment.update( - { - "EVENT_NAME": "pull_request", - "BASE_SHA": "1" * 40, - "HEAD_SHA": "2" * 40, - "GITHUB_OUTPUT": os.fspath(root / "output"), - "GITHUB_STEP_SUMMARY": os.fspath(root / "summary"), - "RUNNER_TEMP": temporary, - "GIT_CAT_FILE_STATUS": "0", - "GIT_DIFF_STATUS": "0", - "CHANGED_FILES": "openvmm\\0", - "CLASSIFICATION": complete, - "CLASSIFIER_STATUS": "0", - } - ) - environment.update(overrides) - result = subprocess.run( - ["bash", "-c", stub + script], - cwd=BuildConstants.REPO_ROOT, - env=environment, - capture_output=True, - text=True, - check=False, - ) - self.assertEqual(result.returncode, 0, result.stderr) - output = (root / "output").read_text(encoding="utf-8").splitlines() - self.assertEqual(len(output), len(expected), output) - self.assertEqual(dict(line.split("=", 1) for line in output), expected) - summary = (root / "summary").read_text(encoding="utf-8") - if expected is everything: - self.assertIn("### Full matrix: run\n", summary) - else: - self.assertNotIn("Full matrix", summary) - - def test_flowey_downloads_use_retrying_curl(self): - action = ( - BuildConstants.REPO_ROOT - / ".github" - / "actions" - / "setup-curl" - / "action.yml" - ).read_text(encoding="utf-8") - windows_shim = ( - BuildConstants.REPO_ROOT - / ".github" - / "actions" - / "setup-curl" - / "curl-shim.rs" - ).read_text(encoding="utf-8") - workflow = ( - BuildConstants.REPO_ROOT / ".github" / "workflows" / "ci.yml" - ).read_text(encoding="utf-8") - microvm_workflow = ( - BuildConstants.REPO_ROOT - / ".github" - / "workflows" - / "run-nvx-microvm-tests.yml" - ).read_text(encoding="utf-8") - build_action = ( - BuildConstants.REPO_ROOT - / ".github" - / "actions" - / "build-openvmm" - / "action.yml" - ).read_text(encoding="utf-8") - - for option in ( - "--retry 5", - "--retry-all-errors", - "--retry-delay 2", - "--retry-max-time 90", - ): - self.assertEqual(action.count(option), 1) - for argument in ( - '"--retry"', - '"5"', - '"--retry-all-errors"', - '"--retry-delay"', - '"2"', - '"--retry-max-time"', - '"90"', - ): - self.assertIn(argument, windows_shim) - self.assertIn('system_curl="${NVX_SYSTEM_CURL:-$(command -v curl)}"', action) - self.assertIn("Get-Command curl.exe", action) - self.assertIn('Join-Path $ShimDirectory "curl.exe"', action) - self.assertIn("Get-Command rustc.exe", action) - self.assertNotIn("curl.cmd", action) - for job_name in ( - "openvmm-vmm-tests", - "openvmm-unit-tests", - ): - with self.subTest(job_name=job_name): - self.assertEqual( - _workflow_job(workflow, job_name).count( - "uses: ./.github/actions/setup-curl" - ), - 1, - ) - self.assertEqual( - microvm_workflow.count("uses: ./.github/actions/setup-curl"), - 1, - ) - self.assertIn("uses: ./.github/actions/setup-curl", build_action) - - @unittest.skipUnless(os.name == "nt", "Windows-specific curl resolution") - def test_windows_curl_shim_retries_http_500(self): - rustc = shutil.which("rustc") - system_curl = shutil.which("curl.exe") - if rustc is None or system_curl is None: - self.fail("Windows curl shim test requires rustc and curl.exe") - - class RetryHandler(http.server.BaseHTTPRequestHandler): - request_count = 0 - - def do_GET(self) -> None: - type(self).request_count += 1 - if type(self).request_count == 1: - self.send_error(http.HTTPStatus.INTERNAL_SERVER_ERROR) - return - body = b"ok" - self.send_response(http.HTTPStatus.OK) - self.send_header("Content-Length", str(len(body))) - self.end_headers() - self.wfile.write(body) - - def log_message(self, format: str, *args: object) -> None: - pass - - server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), RetryHandler) - server_thread = threading.Thread(target=server.serve_forever, daemon=True) - server_thread.start() - try: - with tempfile.TemporaryDirectory() as directory: - shim = Path(directory) / "curl.exe" - shim_source = ( - BuildConstants.REPO_ROOT - / ".github" - / "actions" - / "setup-curl" - / "curl-shim.rs" - ) - subprocess.run( - [ - rustc, - "--edition=2021", - "-C", - "opt-level=s", - "-o", - os.fspath(shim), - os.fspath(shim_source), - ], - check=True, - ) - environment = os.environ.copy() - environment["NVX_SYSTEM_CURL"] = system_curl - environment["PATH"] = directory + os.pathsep + environment["PATH"] - port = server.server_address[1] - result = subprocess.run( - [ - os.fspath(shim), - "--fail", - "-L", - f"http://127.0.0.1:{port}/asset", - ], - check=False, - capture_output=True, - env=environment, - ) - finally: - server.shutdown() - server.server_close() - server_thread.join(timeout=5) - - self.assertFalse( - server_thread.is_alive(), - "HTTP test server did not stop within 5 seconds", - ) - self.assertEqual(result.returncode, 0, result.stderr.decode(errors="replace")) - self.assertEqual(result.stdout, b"ok") - self.assertEqual(RetryHandler.request_count, 2) - - def test_ci_shares_openvmm_inputs_and_binary_artifacts(self): - workflow = ( - BuildConstants.REPO_ROOT / ".github" / "workflows" / "ci.yml" - ).read_text(encoding="utf-8") - build_workflow = ( - BuildConstants.REPO_ROOT - / ".github" - / "workflows" - / "build-openvmm-binary.yml" - ).read_text(encoding="utf-8") - microvm_workflow = ( - BuildConstants.REPO_ROOT - / ".github" - / "workflows" - / "run-nvx-microvm-tests.yml" - ).read_text(encoding="utf-8") - platform_workflow = ( - BuildConstants.REPO_ROOT / ".github" / "workflows" / "run-platform.yml" - ).read_text(encoding="utf-8") - build_action = ( - BuildConstants.REPO_ROOT - / ".github" - / "actions" - / "build-openvmm" - / "action.yml" - ).read_text(encoding="utf-8") - - for configuration in (workflow, build_action): - self.assertIn( - "openvmm/flowey-persist/flowey_lib_common__download_gh_release", - configuration, - ) - self.assertIn( - "openvmm/flowey-persist/flowey_lib_common__cache", - configuration, - ) - self.assertIn( - "openvmm-inputs-v1-${{ runner.os }}-${{ runner.arch }}-", - configuration, - ) - producers = { - "build-openvmm-linux-gnu": ( - "kvm", - "openvmm-linux-gnu", - ), - "build-openvmm-linux-musl": ( - "mshv", - "openvmm-linux-musl", - ), - "build-openvmm-windows-msvc": ( - "whp", - "openvmm-windows-msvc", - ), - } - self.assertEqual( - workflow.count("uses: ./.github/workflows/build-openvmm-binary.yml"), - len(producers), - ) - for job_name, (backend, artifact) in producers.items(): - with self.subTest(producer=job_name): - job = _workflow_job(workflow, job_name) - self.assertIn(f"backend: {backend}", job) - self.assertIn(f"artifact: {artifact}", job) - self.assertNotIn("target:", job) - self.assertNotIn("build-mode:", job) - self.assertNotIn("strategy:", job) - - consumers = { - "nvx-microvm-tests-kvm": ( - "build-openvmm-linux-gnu", - "openvmm-linux-gnu", - "run-nvx-microvm-tests.yml", - ), - "nvx-microvm-tests-mshv": ( - "build-openvmm-linux-gnu", - "openvmm-linux-gnu", - "run-nvx-microvm-tests.yml", - ), - "nvx-microvm-tests-whp": ( - "build-openvmm-windows-msvc", - "openvmm-windows-msvc", - "run-nvx-microvm-tests.yml", - ), - "nvx-microvm-debug-kvm": ( - "build-openvmm-linux-gnu", - "openvmm-linux-gnu", - "run-nvx-microvm-tests.yml", - ), - "nvx-microvm-debug-mshv": ( - "build-openvmm-linux-gnu", - "openvmm-linux-gnu", - "run-nvx-microvm-tests.yml", - ), - "nvx-microvm-debug-whp": ( - "build-openvmm-windows-msvc", - "openvmm-windows-msvc", - "run-nvx-microvm-tests.yml", - ), - "platform-kvm": ( - "build-openvmm-linux-gnu", - "openvmm-linux-gnu", - "run-platform.yml", - ), - "platform-mshv": ( - "build-openvmm-linux-musl", - "openvmm-linux-musl", - "run-platform.yml", - ), - "platform-whp": ( - "build-openvmm-windows-msvc", - "openvmm-windows-msvc", - "run-platform.yml", - ), - } - for job_name, (producer, artifact, reusable_workflow) in consumers.items(): - with self.subTest(consumer=job_name): - job = _workflow_job(workflow, job_name) - debug_kernel = ( - "debug-kernel, " - if job_name.startswith("nvx-microvm-debug-") - else "" - ) - cli = ( - "nvx-cli-windows" - if producer == "build-openvmm-windows-msvc" - else "nvx-cli-linux" - ) - self.assertIn( - f"needs: [artifacts, {debug_kernel}{producer}, {cli}, " - "openvmm-changes]", - job, - ) - self.assertIn(f"needs.{producer}.result == 'success'", job) - self.assertIn(f"needs.{cli}.result == 'success'", job) - self.assertIn(f"uses: ./.github/workflows/{reusable_workflow}", job) - self.assertIn(f"openvmm-artifact: {artifact}", job) - for unrelated_producer in producers.keys() - {producer}: - self.assertNotIn(unrelated_producer, job) - - self.assertIn("build/openvmm.provenance.json", build_workflow) - self.assertIn( - "name: ${{ inputs.artifact }}-executable", - build_workflow, - ) - self.assertIn( - "name: ${{ inputs.artifact }}-provenance", - build_workflow, - ) - self.assertEqual( - build_workflow.count("uses: ./.github/actions/build-openvmm"), - 1, - ) - self.assertIn("backend: ${{ inputs.backend }}", build_workflow) - self.assertNotIn("target: ${{ inputs.target }}", build_workflow) - self.assertNotIn("build-mode: ${{ inputs.build-mode }}", build_workflow) - self.assertEqual(build_workflow.count("overwrite: true"), 2) - self.assertEqual(build_workflow.count("retention-days: 1"), 2) - for consumer_workflow, download_count in ( - (microvm_workflow, 4), - (platform_workflow, 3), - ): - self.assertIn("path: openvmm/target/release", consumer_workflow) - self.assertIn("path: build", consumer_workflow) - self.assertEqual( - consumer_workflow.count("uses: actions/download-artifact@v8"), - download_count, - ) - self.assertIn( - "run: chmod +x openvmm/target/release/openvmm", - consumer_workflow, - ) - for configuration in ( - workflow, - build_workflow, - microvm_workflow, - platform_workflow, - ): - self.assertNotIn("path: .", configuration) - self.assertIn("openvmm-binary-v6-${{ inputs.backend }}-", build_action) - self.assertNotIn("openvmm-binary-v5-", build_action) - self.assertNotIn("inputs.target", build_action) - self.assertNotIn("inputs.build-mode", build_action) - self.assertNotIn("rustup target add", build_action) - self.assertNotIn("X86_64_UNKNOWN_LINUX_MUSL", build_action) - self.assertEqual( - build_action.count( - 'python3 scripts/nvx.py build-openvmm --backend "${{ inputs.backend }}"' - ), - 1, - ) - self.assertEqual( - build_action.count( - 'python scripts\\nvx.py build-openvmm --backend "${{ inputs.backend }}"' - ), - 1, - ) - self.assertIn("Verify OpenVMM provenance on Linux", build_action) - self.assertIn("Verify OpenVMM provenance on Windows", build_action) - self.assertNotIn("nvx-microvm-tests-v1", workflow) - self.assertNotIn("cargo-v2-", build_action) - self.assertNotIn("openvmm-tests-v2-", workflow) - self.assertNotIn("Restore OpenVMM test build", workflow) - self.assertNotIn("Save OpenVMM test build", workflow) - self.assertEqual( - build_workflow.count("uses: ./.github/actions/sccache"), - 2, - ) - for job_name, cache_namespace in ( - ("openvmm-vmm-tests", "vmm-tests"), - ("openvmm-unit-tests", "unit-tests"), - ): - with self.subTest(job_name=job_name): - job = _workflow_job(workflow, job_name) - self.assertEqual( - job.count("uses: ./.github/actions/sccache"), - 2, - ) - self.assertIn( - f"key: openvmm-inputs-v1-${{{{ runner.os }}}}-" - f"${{{{ runner.arch }}}}-{cache_namespace}-" - "${{ steps.openvmm.outputs.sha }}", - job, - ) - self.assertIn( - f"key: openvmm-cargo-v1-${{{{ runner.os }}}}-" - f"${{{{ runner.arch }}}}-{cache_namespace}-" - "${{ hashFiles('openvmm/Cargo.lock') }}", - job, - ) - self.assertIn( - "restore-keys: |\n" - " openvmm-inputs-v1-${{ runner.os }}-" - "${{ runner.arch }}-", - job, - ) - self.assertIn( - "restore-keys: |\n" - " openvmm-cargo-v1-${{ runner.os }}-" - "${{ runner.arch }}-", - job, - ) - self.assertNotIn("uses: actions/cache@v5", workflow) - self.assertNotIn("uses: actions/cache@v5", build_workflow) - self.assertNotIn("uses: actions/cache@v5", build_action) - - release_job = _workflow_job(workflow, "release") - performance_gate_job = _workflow_job(workflow, "performance-gate") - performance_persist_job = _workflow_job(workflow, "performance-persist") - for job_name in consumers: - self.assertIn(f" - {job_name}", release_job) - self.assertIn(f"needs.{job_name}.result", release_job) - self.assertIn(f" - {job_name}", performance_persist_job) - self.assertIn(f"needs.{job_name}.result", performance_persist_job) - for job_name in ("platform-kvm", "platform-mshv", "platform-whp"): - self.assertIn(job_name, performance_gate_job) - self.assertIn(f"needs.{job_name}.result", performance_gate_job) - - def test_copilot_setup_restores_ci_cache_entries(self): - github = BuildConstants.REPO_ROOT / ".github" - setup = (github / "workflows" / "copilot-setup-steps.yml").read_text( - encoding="utf-8" - ) - workflow = (github / "workflows" / "ci.yml").read_text(encoding="utf-8") - build_workflow = (github / "workflows" / "build-openvmm-binary.yml").read_text( - encoding="utf-8" - ) - build_action = (github / "actions" / "build-openvmm" / "action.yml").read_text( - encoding="utf-8" - ) - checkout_action = ( - github / "actions" / "checkout-openvmm" / "action.yml" - ).read_text(encoding="utf-8") - - # Guest artifact keys stay in sync because both sides use one action. - guest_artifacts = _workflow_step( - setup, "copilot-setup-steps", "Restore guest artifacts built by CI" - ) - self.assertEqual( - _yaml_field(guest_artifacts, "uses"), - ["./.github/actions/build-guest-artifacts"], - ) - self.assertEqual(_yaml_field(guest_artifacts, "restore-only"), ['"true"']) - - # The OpenVMM binary restore repeats the cache entry that the Linux GNU - # producer saves. A different key, path list, or compression method - # would make every session miss it without failing setup. - producer = _workflow_job(workflow, "build-openvmm-linux-gnu") - [backend] = _yaml_field(producer, "backend") - [binary] = _yaml_field(producer, "binary") - self.assertIn("binary-path: ${{ inputs.binary }}", build_workflow) - saved = _composite_action_step(build_action, "Save OpenVMM binary on Linux") - restored = _workflow_step( - setup, "copilot-setup-steps", "Restore the OpenVMM binary built by CI" - ) - self.assertEqual( - _yaml_field(restored, "key"), - [ - key.replace("${{ inputs.backend }}", backend).replace( - "steps.openvmm.outputs.sha", "steps.openvmm-cache.outputs.sha" - ) - for key in _yaml_field(saved, "key") - ], - ) - self.assertEqual( - _yaml_field(restored, "path"), - [ - path.replace("${{ inputs.binary-path }}", binary) - for path in _yaml_field(saved, "path") - ], - ) - - prepare = _workflow_step( - setup, "copilot-setup-steps", "Prepare OpenVMM cache restore" - ) - for revision_source in (checkout_action, prepare): - self.assertIn("sha=$(git rev-parse :openvmm)", revision_source) - - def tool_links(configuration: str) -> list[str]: - return [ - line.strip() - for line in configuration.splitlines() - if line.strip().startswith("ln -sf ") - ] - - self.assertEqual( - _yaml_field(saved, "PATH"), - ["${{ steps.cache-tools-linux.outputs.path }}"], - ) - self.assertEqual( - _yaml_field(restored, "PATH"), - ["${{ steps.openvmm-cache.outputs.path }}"], - ) - self.assertEqual( - tool_links(prepare), - tool_links( - _composite_action_step(build_action, "Prepare Linux cache tools") - ), - ) - - def test_guest_artifact_restore_only_guards_builds_and_cache_saves(self): - action = ( - BuildConstants.REPO_ROOT - / ".github" - / "actions" - / "build-guest-artifacts" - / "action.yml" - ).read_text(encoding="utf-8") - steps = _composite_action_steps(action) - guarded_steps = [ - step - for step in steps - if "docker build" in step or "uses: actions/cache/save@" in step - ] - - self.assertTrue(guarded_steps) - for step in guarded_steps: - try: - name = _yaml_scalar(step, "name") - except AssertionError: - name = "" - with self.subTest(step=name): - condition = _yaml_scalar(step, "if") - guard = "inputs.restore-only != 'true' && " - self.assertTrue(condition.startswith(guard), condition) - cache_condition = condition.removeprefix(guard) - self.assertIn("cache-hit != 'true'", cache_condition) - if "||" in cache_condition: - self.assertTrue( - _outer_parentheses_enclose(cache_condition), - cache_condition, - ) - - def test_guest_artifact_restore_only_leaves_restore_and_scratch_steps_active(self): - action = ( - BuildConstants.REPO_ROOT - / ".github" - / "actions" - / "build-guest-artifacts" - / "action.yml" - ).read_text(encoding="utf-8") - steps = _composite_action_steps(action) - restore_steps = [ - step for step in steps if "uses: actions/cache/restore@" in step - ] - - self.assertTrue(restore_steps) - scratch = _composite_action_step( - action, "Prepare Ubuntu sandbox scratch template" - ) - for step in (*restore_steps, scratch): - try: - name = _yaml_scalar(step, "name") - except AssertionError: - name = "" - with self.subTest(step=name): - # Other inputs, such as debug-kernel, may select these steps, - # but restore-only must leave them active. - try: - condition = _yaml_scalar(step, "if") - except AssertionError: - continue - self.assertNotIn("restore-only", condition) - - def test_ci_runs_the_debug_kernel_on_kvm_per_pull_request_and_all_on_dev(self): - workflow = ( - BuildConstants.REPO_ROOT / ".github" / "workflows" / "ci.yml" - ).read_text(encoding="utf-8") - microvm_workflow = ( - BuildConstants.REPO_ROOT - / ".github" - / "workflows" - / "run-nvx-microvm-tests.yml" - ).read_text(encoding="utf-8") - - artifacts = _workflow_job(workflow, "artifacts") - self.assertNotIn("debug-kernel", artifacts) - self.assertNotIn("vmlinux-debug", artifacts) - # The debug kernel builds on GitHub-hosted capacity beside the shared - # artifacts, under the same condition, and only the debug jobs wait. - debug_build = _workflow_job(workflow, "debug-kernel") - self.assertIn("runs-on: ubuntu-latest", debug_build) - self.assertIn('debug-kernel: "true"', debug_build) - self.assertIn('guest-images: "false"', debug_build) - self.assertIn("name: guest-debug-kernel", debug_build) - for name in ( - KernelBuildConstants.DEBUG_BINARY_NAME, - KernelBuildConstants.DEBUG_CONFIG_NAME, - ): - self.assertIn(f"build/{name}\n", debug_build) - - def condition(job: str) -> str: - return job[job.index(" if: >-\n") : job.index(" runs-on:")] - - self.assertEqual(condition(debug_build), condition(artifacts)) - for consumer in ("release", "performance-persist"): - with self.subTest(consumer=consumer): - job = _workflow_job(workflow, consumer) - self.assertIn(" - debug-kernel\n", job) - self.assertIn("needs.debug-kernel.result == 'success' &&", job) - kvm = _workflow_job(workflow, "nvx-microvm-debug-kvm") - self.assertIn("github.event.pull_request.head.repo.full_name", kvm) - for job_name in ci.REQUIRED_CI_MICROVM_DEBUG_PUSH_JOBS: - with self.subTest(job=job_name): - job = _workflow_job(workflow, job_name) - self.assertIn("github.event_name == 'push' &&", job) - self.assertNotIn("pull_request", job) - for job_name in ( - *ci.REQUIRED_CI_MICROVM_DEBUG_JOBS, - *ci.REQUIRED_CI_MICROVM_DEBUG_PUSH_JOBS, - ): - with self.subTest(job=job_name): - job = _workflow_job(workflow, job_name) - self.assertIn("debug-kernel: true", job) - self.assertIn("needs.debug-kernel.result == 'success' &&", job) - self.assertIn(job_name, ci.REQUIRED_CI_RESULT_ENVIRONMENTS) - for consumer in ("release", "performance-persist"): - self.assertIn( - f"needs.{job_name}.result == 'success' ||\n" - f" needs.{job_name}.result == 'skipped'", - _workflow_job(workflow, consumer), - ) - - self.assertIn(" debug-kernel:\n", microvm_workflow) - self.assertIn(" type: boolean\n", microvm_workflow) - download = microvm_workflow.index("name: guest-debug-kernel") - self.assertIn( - "if: inputs.debug-kernel", - microvm_workflow[download - 120 : download], - ) - debug_flag = "${{ inputs.debug-kernel && '--debug-kernel' || '' }}" - self.assertEqual(microvm_workflow.count(debug_flag), 2) - # The debug run replaces every production-kernel workload rather than - # adding to it: each step that drives nvx.py without the debug kernel - # flag boots build/vmlinux, so it must skip the debug jobs (#371). - guard = " && !inputs.debug-kernel" - debug_steps: list[str] = [] - production_steps: list[str] = [] - for step in _workflow_steps(microvm_workflow, "test"): - try: - script = _yaml_scalar(step, "run") - except AssertionError: - continue - if "nvx.py" not in script: - continue - name = _yaml_scalar(step, "name") - try: - step_condition = _yaml_scalar(step, "if") - except AssertionError: - step_condition = "" - with self.subTest(step=name): - if debug_flag in script: - debug_steps.append(name) - self.assertNotIn("debug-kernel", step_condition) - continue - production_steps.append(name) - self.assertTrue(step_condition.endswith(guard), step_condition) - selector = step_condition.removesuffix(guard) - if "||" in selector: - self.assertTrue( - _outer_parentheses_enclose(selector), step_condition - ) - self.assertEqual(len(debug_steps), 2) - self.assertTrue(production_steps) - - def test_benchmarks_rely_on_the_microvm_correctness_jobs(self): - # #286: the benchmark action ran a second smp-lapic gate before - # acceptance. The microVM correctness jobs already gate the required - # status, the development release, and performance persistence. - workflow = ( - BuildConstants.REPO_ROOT / ".github" / "workflows" / "ci.yml" - ).read_text(encoding="utf-8") - action = ( - BuildConstants.REPO_ROOT - / ".github" - / "actions" - / "run-benchmark" - / "action.yml" - ).read_text(encoding="utf-8") - - self.assertNotIn("test-microvm", action) - self.assertNotIn("lapic-correctness", action) - for job_name in ci.REQUIRED_CI_MICROVM_TEST_JOBS: - with self.subTest(job=job_name): - self.assertIn(job_name, ci.REQUIRED_CI_RESULT_ENVIRONMENTS) - for consumer in ("release", "performance-persist"): - job = _workflow_job(workflow, consumer) - self.assertIn(f" - {job_name}\n", job) - self.assertIn( - f"needs.{job_name}.result == 'success' ||\n" - f" needs.{job_name}.result == 'skipped'", - job, - ) - - def test_ci_runs_openvmm_vmm_tests_per_backend_and_unit_tests_per_os(self): - workflow = ( - BuildConstants.REPO_ROOT / ".github" / "workflows" / "ci.yml" - ).read_text(encoding="utf-8") - vmm_tests_job = _workflow_job(workflow, "openvmm-vmm-tests") - unit_tests_job = _workflow_job(workflow, "openvmm-unit-tests") - - def legs(job: str) -> list[tuple[str, str, str]]: - return re.findall( - r"^ - name: (.+)\n" - r" backend: (\S+)\n" - r" runner: '(.+)'$", - job, - re.MULTILINE, - ) - - self.assertEqual( - legs(vmm_tests_job), - [ - ( - "Linux / KVM", - "kvm", - '["self-hosted", "linux", "kvm", "virtual-machine"]', - ), - ( - "Linux / MSHV", - "mshv", - '["self-hosted", "linux", "mshv", "virtual-machine"]', - ), - ( - "Windows / WHP", - "whp", - '["self-hosted", "windows", "whp", "virtual-machine"]', - ), - ], - ) - # The unit tests that need /dev/kvm or /dev/mshv are ignored, so one - # Linux runner covers both Linux backends. - self.assertEqual( - legs(unit_tests_job), - [ - ( - "Linux", - "mshv", - '["self-hosted", "linux", "mshv", "virtual-machine"]', - ), - ( - "Windows", - "whp", - '["self-hosted", "windows", "whp", "virtual-machine"]', - ), - ], - ) - self.assertIn( - """ - name: Run OpenVMM unit tests on Linux - if: runner.os != 'Windows' - shell: bash - run: python3 scripts/nvx.py test-openvmm-unit""", - unit_tests_job, - ) - self.assertIn( - """ - name: Run OpenVMM unit tests on Windows - if: runner.os == 'Windows' - shell: powershell - run: python scripts\\nvx.py test-openvmm-unit""", - unit_tests_job, - ) - self.assertEqual( - vmm_tests_job.count( - 'scripts/nvx.py test-openvmm --backend "${{ matrix.backend }}"' - ), - 2, - ) - self.assertEqual( - vmm_tests_job.count( - 'scripts\\nvx.py test-openvmm --backend "${{ matrix.backend }}"' - ), - 1, - ) - self.assertNotIn("test-openvmm-unit", vmm_tests_job) - self.assertEqual(unit_tests_job.count("scripts/nvx.py test-openvmm-unit"), 1) - self.assertEqual( - unit_tests_job.count("scripts\\nvx.py test-openvmm-unit"), - 1, - ) - self.assertNotIn("test-openvmm --backend", unit_tests_job) - - def test_ci_validates_the_cli_once_per_os_after_checking_out_openvmm(self): - # The CLI tests read OpenVMM's pinned CPU profiles from the submodule, - # which a runner's workspace holds at the pinned revision only after - # the job checks it out. The tests do not depend on the backend, so - # one GitHub-hosted job per OS runs them instead of every hypervisor - # job, and the microVM and platform jobs of each OS wait for it. - workflows = BuildConstants.REPO_ROOT / ".github" / "workflows" - workflow = (workflows / "ci.yml").read_text(encoding="utf-8") - self.assertEqual(ci.REQUIRED_CI_CLI_JOBS, ("nvx-cli-linux", "nvx-cli-windows")) - for job_name, runner in ( - ("nvx-cli-linux", "ubuntu-latest"), - ("nvx-cli-windows", "windows-latest"), - ): - with self.subTest(job=job_name): - job = _workflow_job(workflow, job_name) - self.assertIn(f" runs-on: {runner}\n", job) - self.assertNotIn(" needs:", job) - self.assertIn( - "github.event.pull_request.head.repo.full_name " - "== github.repository", - job, - ) - self.assertEqual(job.count("uses: ./.github/actions/validate-nvx"), 1) - for prerequisite in ( - "uses: ./.github/actions/checkout-openvmm", - "uses: actions/setup-python@v6", - ): - self.assertLess( - job.index(prerequisite), - job.index("uses: ./.github/actions/validate-nvx"), - ) - # The release that the self-hosted runners provide. - self.assertIn('python-version: "3.12"', job) - self.assertIn(job_name, ci.REQUIRED_CI_RESULT_ENVIRONMENTS) - windows = _workflow_job(workflow, "nvx-cli-windows") - self.assertIn("rust-toolchain.toml", windows) - self.assertIn("--profile minimal --no-self-update", windows) - self.assertLess( - windows.index("rustup toolchain install"), - windows.index("uses: ./.github/actions/validate-nvx"), - ) - - configurations = { - name: (workflows / name).read_text(encoding="utf-8") - for name in ( - "ci.yml", - "build-openvmm-binary.yml", - "run-nvx-microvm-tests.yml", - "run-platform.yml", - ) - } - self.assertEqual( - sum( - configuration.count("uses: ./.github/actions/validate-nvx") - for configuration in configurations.values() - ), - len(ci.REQUIRED_CI_CLI_JOBS), - ) - for job_name in ( - *ci.REQUIRED_CI_MICROVM_TEST_JOBS, - *ci.REQUIRED_CI_MICROVM_DEBUG_JOBS, - *ci.REQUIRED_CI_MICROVM_DEBUG_PUSH_JOBS, - *ci.REQUIRED_CI_PLATFORM_JOBS, - ): - cli = "nvx-cli-windows" if job_name.endswith("-whp") else "nvx-cli-linux" - with self.subTest(consumer=job_name): - job = _workflow_job(workflow, job_name) - needs = re.search(r"^ needs: \[(.+)\]$", job, re.MULTILINE) - assert needs is not None - self.assertIn(cli, needs.group(1).split(", ")) - self.assertIn(f" needs.{cli}.result == 'success' &&\n", job) - for other in set(ci.REQUIRED_CI_CLI_JOBS) - {cli}: - self.assertNotIn(other, job) - for job_name in ("openvmm-vmm-tests", "openvmm-unit-tests"): - with self.subTest(job=job_name): - self.assertNotIn("nvx-cli", _workflow_job(workflow, job_name)) - for consumer in ("release", "performance-persist", "required-status-check"): - job = _workflow_job(workflow, consumer) - for cli in ci.REQUIRED_CI_CLI_JOBS: - with self.subTest(consumer=consumer, cli=cli): - self.assertIn(f" - {cli}\n", job) - if consumer != "required-status-check": - self.assertIn( - f" needs.{cli}.result == 'success' &&\n", job - ) - - def test_ci_preserves_failed_openvmm_test_diagnostics(self): - workflow = ( - BuildConstants.REPO_ROOT / ".github" / "workflows" / "ci.yml" - ).read_text(encoding="utf-8") - job = _workflow_job(workflow, "openvmm-vmm-tests") - step_name = " - name: Upload OpenVMM test diagnostics" - self.assertIn(step_name, job) - upload = job.split(step_name, 1)[1].split("\n - name:", 1)[0] - - self.assertIn("\n if: failure()", upload) - self.assertIn("uses: actions/upload-artifact@v7", upload) - self.assertIn( - "name: openvmm-vmm-tests-${{ runner.os }}-${{ matrix.backend }}-" - "${{ github.run_id }}-${{ github.run_attempt }}", - upload, - ) - self.assertIn( - "path: ${{ runner.os == 'Windows' && " - "format('{0}/{1}/test_results', runner.temp, matrix.backend) || " - "'openvmm/target/vmm_tests/test_results' }}", - upload, - ) - self.assertIn("if-no-files-found: warn", upload) - self.assertIn("retention-days: 7", upload) - self.assertLess( - job.index(step_name), - job.index(" - name: Report sccache"), - ) - - def test_runner_setup_pins_and_validates_sccache(self): - workflow = ( - BuildConstants.REPO_ROOT / ".github" / "workflows" / "ci.yml" - ).read_text(encoding="utf-8") - action = ( - BuildConstants.REPO_ROOT / ".github" / "actions" / "sccache" / "action.yml" - ).read_text(encoding="utf-8") - validate_runner = ( - BuildConstants.REPO_ROOT - / ".github" - / "actions" - / "validate-runner" - / "action.yml" - ).read_text(encoding="utf-8") - windows_setup = ( - BuildConstants.REPO_ROOT / "scripts" / "setup" / "setup-windows-whp.ps1" - ).read_text(encoding="utf-8") - linux_setup = ( - BuildConstants.REPO_ROOT / "scripts" / "setup" / "setup-linux-runner.sh" - ).read_text(encoding="utf-8") - manifest = ( - BuildConstants.REPO_ROOT / "scripts" / "setup" / "tool-versions.conf" - ).read_text(encoding="utf-8") - - # The manifest pins sccache and its checksums, and both runner setups - # install that release; test_tool_versions.py checks the manifest. - self.assertRegex(manifest, r"(?m)^sccache\.version=[0-9]+\.[0-9]+\.[0-9]+$") - for platform in ("linux-x86_64", "windows-x86_64"): - self.assertRegex( - manifest, - rf"(?m)^sccache\.artifacts\.{platform}\.sha256=[0-9a-f]{{64}}$", - ) - self.assertIn("\n tool_manifest_artifact sccache\n", linux_setup) - self.assertIn( - '\n$sccacheRelease = Get-ToolArtifact $ToolManifest "sccache" $ToolPlatform\n', - windows_setup, - ) - for configuration in (windows_setup, linux_setup): - self.assertIn("RUSTC_WRAPPER", configuration) - self.assertIn("CARGO_INCREMENTAL", configuration) - self.assertIn("SCCACHE_DIR", configuration) - self.assertIn("SCCACHE_CACHE_SIZE", configuration) - self.assertIn("sccache --zero-stats", action) - self.assertIn("sccache --show-stats", action) - self.assertIn("sccache --stop-server", action) - self.assertIn("SCCACHE_IDLE_TIMEOUT", action) - self.assertIn("sccache --version", validate_runner) - self.assertNotRegex( - workflow, - r"(?m)^\s+path: openvmm/target\s*$", - ) - self.assertLess( - linux_setup.index('test -f "${runner_directory}/.runner"'), - linux_setup.index('test -w "$runner_sccache_dir"'), - ) - - def test_linux_unnamed_setup_defers_existing_runner_validation(self): - linux_setup = ( - BuildConstants.REPO_ROOT / "scripts" / "setup" / "setup-linux-runner.sh" - ).read_text(encoding="utf-8") - environment_check = linux_setup.split("check_environment() {\n", 1)[1].split( - "\n}\n", 1 - )[0] - runner_checks = environment_check.split( - " validate_runner_state_paths\n validate_runner_work_paths\n", 1 - )[1] - guard = 'if [ "$configure_runner" = false ] && [ "$check_only" = false ]; then' - self.assertIn(guard, runner_checks) - self.assertLess( - runner_checks.index(guard), - runner_checks.index('test -w "$runner_sccache_dir"'), - ) - if os.name != "posix": - return - - harness = ( - "set -eu\n" - "runner_directory=/missing-runner\n" - "runner_sccache_dir=/missing-runner/_work/_sccache\n" - "validate_runner_state_paths() { echo state-checked; }\n" - "validate_runner_work_paths() { echo work-checked; }\n" - 'run_as_root() { [ "$1" = test ] && [ "$2" = -f ]; }\n' - "check_runner_state() {\n" - " validate_runner_state_paths\n" - " validate_runner_work_paths\n" - f"{runner_checks}\n" - "}\n" - ) - for configure_runner, check_only in ( - ("false", "false"), - ("false", "true"), - ("true", "false"), - ): - with self.subTest(configure_runner=configure_runner, check_only=check_only): - result = subprocess.run( - [ - "sh", - "-c", - harness + f"{configure_runner=}\n{check_only=}\n" - "check_runner_state\n", - ], - capture_output=True, - text=True, - timeout=10, - check=False, - ) - self.assertEqual( - result.returncode == 0, - configure_runner == check_only == "false", - result.stderr, - ) - if result.returncode == 0: - self.assertIn("state-checked", result.stdout) - self.assertIn("work-checked", result.stdout) - - def test_linux_setup_installs_openvmm_perl_modules(self): - setup_directory = BuildConstants.REPO_ROOT / "scripts" / "setup" - configurations = ( - (setup_directory / "setup-linux-runner.sh").read_text(encoding="utf-8"), - (setup_directory / "setup-linux-mshv.sh").read_text(encoding="utf-8"), - ) - - for configuration in configurations: - for package in ( - "perl-FindBin", - "perl-IPC-Cmd", - "perl-Time-Piece", - "perl-lib", - ): - self.assertIn(package, configuration) - - def test_windows_runner_requires_inbox_pcat_firmware(self): - windows_setup = ( - BuildConstants.REPO_ROOT / "scripts" / "setup" / "setup-windows-whp.ps1" - ).read_text(encoding="utf-8") - validate_runner = ( - BuildConstants.REPO_ROOT - / ".github" - / "actions" - / "validate-runner" - / "action.yml" - ).read_text(encoding="utf-8") - - self.assertIn('"Microsoft-Hyper-V"', windows_setup) - for configuration in (windows_setup, validate_runner): - for firmware in ( - "vmfirmwarepcat.dll", - "vmfirmware.dll", - "VmEmulatedDevices.dll", - ): - self.assertIn(firmware, configuration) - - def test_linux_runners_report_the_invariant_tsc_as_evidence(self): - validate_runner = ( - BuildConstants.REPO_ROOT - / ".github" - / "actions" - / "validate-runner" - / "action.yml" - ).read_text(encoding="utf-8") - linux_setup = ( - BuildConstants.REPO_ROOT / "scripts" / "setup" / "setup-linux-runner.sh" - ).read_text(encoding="utf-8") - - # Neither CI nor provisioning rejects a host without an invariant - # TSC: the guest warp probe measures what its guests observe (#265). - step = validate_runner.split(" - name: Report host TSC\n", 1)[1] - step = step.split("\n\n - name: ", 1)[0] - self.assertIn(" if: runner.os != 'Windows'\n", step) - check = "\n".join( - line.removeprefix(" ") - for line in step.split(" run: |\n", 1)[1].splitlines() - ) - function = linux_setup.split("report_invariant_tsc() {\n", 1)[1] - function = "report_invariant_tsc() {\n" + function.split("\n}\n", 1)[0] - function += "\n}\n" - self.assertNotRegex(function, r"\bdie\b") - self.assertLess( - linux_setup.index("report_invariant_tsc /proc/cpuinfo\n"), - linux_setup.index('sudo -n true || die "passwordless sudo is required"'), - ) - if os.name != "posix": - return - - with tempfile.TemporaryDirectory() as temporary: - cpuinfo = Path(temporary) / "cpuinfo" - for flags, invariant in ( - ("fpu tsc constant_tsc nonstop_tsc tsc_known_freq", True), - ("fpu tsc constant_tsc tsc_known_freq", False), - ("fpu tsc constant_tsc nonstop_tsc_x", False), - ): - cpuinfo.write_text( - f"model name\t: Test CPU\nflags\t\t: {flags}\n", - encoding="utf-8", - ) - with self.subTest(flags=flags, check="validate-runner"): - result = subprocess.run( - ["bash", "-c", check.replace("/proc/cpuinfo", str(cpuinfo))], - capture_output=True, - text=True, - timeout=10, - check=False, - ) - self.assertEqual(result.returncode, 0, result.stderr) - self.assertIn("CPU: Test CPU", result.stdout) - self.assertIn( - "TSC flag nonstop_tsc: " - + ("present" if invariant else "absent"), - result.stdout, - ) - self.assertEqual( - "::notice::Runner host does not expose an invariant TSC" - in result.stdout, - not invariant, - ) - self.assertNotIn("::error::", result.stdout + result.stderr) - with self.subTest(flags=flags, check="setup-linux-runner"): - result = subprocess.run( - [ - "sh", - "-c", - f"set -eu\n{function}report_invariant_tsc '{cpuinfo}'\n", - ], - capture_output=True, - text=True, - timeout=10, - check=False, - ) - self.assertEqual(result.returncode, 0, result.stderr) - self.assertEqual( - "warning: host does not expose an invariant TSC" - in result.stderr, - not invariant, - ) - - def test_runners_qualify_their_host_time_before_every_job(self): - validate_runner = ( - BuildConstants.REPO_ROOT - / ".github" - / "actions" - / "validate-runner" - / "action.yml" - ).read_text(encoding="utf-8") - - # The doctor follows the host TSC report on Linux: the backend, the CPU - # fingerprint and generation, and the TSC rate stability. Jobs that - # haven't downloaded OpenVMM skip its CPU profile check (H2) and - # preflight (H3), and only jobs that ask for it run the guest warp - # probe (H6). - self.assertLess( - validate_runner.index(" - name: Report host TSC\n"), - validate_runner.index(" - name: Qualify host time on Linux\n"), - ) - for name, shell, command, summary in ( - ( - "Qualify host time on Linux", - "bash", - "python3 scripts/nvx.py doctor", - '--summary "${GITHUB_STEP_SUMMARY}"', - ), - ( - "Qualify host time on Windows", - "powershell", - "python scripts\\nvx.py doctor", - '--summary "$env:GITHUB_STEP_SUMMARY"', - ), - ): - with self.subTest(step=name): - step = validate_runner.split(f" - name: {name}\n", 1)[1] - step = step.split("\n\n", 1)[0] - self.assertIn(f" shell: {shell}\n", step) - self.assertIn(command, step) - self.assertIn('--backend "${{ inputs.backend }}"', step) - self.assertIn("--checks H1 H2 H4\n", step) - self.assertIn( - "${{ inputs.warp-probe == 'true' && 'H6' || '' }}\n", step - ) - fingerprint = ( - '"${RUNNER_TEMP}/nvx-cpu-fingerprint.json"' - if shell == "bash" - else '"$env:RUNNER_TEMP\\nvx-cpu-fingerprint.json"' - ) - self.assertIn( - "${{ inputs.verify-openvmm == 'true' && " - f"'H3 --cpu-fingerprint {fingerprint}' || '--no-openvmm' }}}}\n", - step, - ) - self.assertIn("--ci-schedule\n", step) - self.assertIn(summary, step) - args = nvx.parse_args( - ["doctor", "--backend", "kvm", "--checks", "H1", "H2", "H4", "H3"] - + ["--cpu-fingerprint", "fingerprint.json", "--ci-schedule"] - ) - self.assertEqual(args.checks, ["H1", "H2", "H4", "H3"]) - self.assertEqual(args.cpu_fingerprint, Path("fingerprint.json")) - args = nvx.parse_args( - ["doctor", "--backend", "kvm", "--checks", "H1", "H2", "H4", "H6", "H3"] - + ["--cpu-fingerprint", "fingerprint.json", "--ci-schedule"] - ) - self.assertEqual(args.checks, ["H1", "H2", "H4", "H6", "H3"]) - args = nvx.parse_args( - ["doctor", "--backend", "kvm", "--checks", "H1", "H2", "H4"] - + ["--no-openvmm", "--ci-schedule"] - ) - self.assertTrue(args.no_openvmm) - self.assertTrue(args.ci_schedule) - self.assertIn(" verify-openvmm:\n", validate_runner) - self.assertIn(' default: "false"\n', validate_runner) - warp_probe = validate_runner.split(" warp-probe:\n", 1)[1] - warp_probe = warp_probe.split("\nruns:\n", 1)[0] - self.assertIn("requires\n verify-openvmm\n", warp_probe) - self.assertIn(' default: "false"\n', warp_probe) - # A failed CPU profile check keeps OpenVMM's fingerprint. - upload = validate_runner.split(" - name: Upload the CPU fingerprint\n")[1] - upload = upload.split("\n\n", 1)[0] - self.assertIn("if: failure() && inputs.verify-openvmm == 'true'\n", upload) - self.assertIn("path: ${{ runner.temp }}/nvx-cpu-fingerprint.json\n", upload) - self.assertIn("if-no-files-found: ignore\n", upload) - # Jobs that run OpenVMM qualify the runner after downloading it. - benchmark = ( - BuildConstants.REPO_ROOT - / ".github" - / "actions" - / "run-benchmark" - / "action.yml" - ).read_text(encoding="utf-8") - self.assertNotIn("download-artifact", benchmark) - for workflow_name in ("run-nvx-microvm-tests.yml", "run-platform.yml"): - with self.subTest(workflow=workflow_name): - workflow = ( - BuildConstants.REPO_ROOT / ".github" / "workflows" / workflow_name - ).read_text(encoding="utf-8") - validate = workflow.index(" - name: Validate runner\n") - self.assertEqual(workflow.count(" - name: Validate runner\n"), 1) - self.assertIn( - 'verify-openvmm: "true"', workflow[validate : validate + 220] - ) - # The microVM scenarios probe after every boot and restore; - # the benchmarks boot guests without such probes. - self.assertEqual( - 'warp-probe: "true"' in workflow[validate : validate + 260], - workflow_name == "run-platform.yml", - ) - for step in ( - " - name: Download guest artifacts\n", - " - name: Download OpenVMM executable\n", - " - name: Make OpenVMM executable\n", - ): - self.assertLess(workflow.index(step), validate) - self.assertLess( - validate_runner.index("Validate Linux toolchain"), - validate_runner.index("Qualify host time on Linux"), - ) - - def test_runner_setups_install_backend_native_openvmm_targets(self): - linux_setup = ( - BuildConstants.REPO_ROOT / "scripts" / "setup" / "setup-linux-runner.sh" - ).read_text(encoding="utf-8") - windows_setup = ( - BuildConstants.REPO_ROOT / "scripts" / "setup" / "setup-windows-whp.ps1" - ).read_text(encoding="utf-8") - - for target in ( - "x86_64-unknown-none", - "x86_64-unknown-uefi", - ): - self.assertIn(target, linux_setup) - self.assertIn(target, windows_setup) - self.assertIn("x86_64-unknown-linux-musl", linux_setup) - for cross_platform_tool in ( - "x86_64-pc-windows-gnu", - "gcc-mingw-w64-x86-64-win32", - "mingw64-gcc", - "x86_64-w64-mingw32-gcc", - "x86_64-w64-mingw32-dlltool", - ): - self.assertNotIn(cross_platform_tool, linux_setup) - - def test_repository_pins_one_exact_rust_toolchain(self): - root = BuildConstants.REPO_ROOT - [toolchain] = re.findall( - r'^channel = "([^"]*)"$', - (root / "rust-toolchain.toml").read_text(encoding="utf-8"), - re.MULTILINE, - ) - # A floating channel such as `stable` lets runners drift apart (#402). - self.assertRegex(toolchain, r"^[0-9]+\.[0-9]+\.[0-9]+$") - self.assertEqual(OpenVMMBuildConstants.RUST_TOOLCHAIN, toolchain) - self.assertIn("rust-toolchain.toml", ReleaseBuildConstants.PROJECT_SOURCE_PATHS) - - setup = root / "scripts" / "setup" - # The setup scripts and the Specula runner setup take the release from - # the setup tool manifest, which rustup cannot read. - self.assertEqual( - re.findall( - r"^rust\.toolchain=(.*)$", - (setup / "tool-versions.conf").read_text(encoding="utf-8"), - re.MULTILINE, - ), - [toolchain], - ) - for path, reader in ( - (setup / "setup-linux-runner.sh", "\n RUST_TOOLCHAIN=$tool_value\n"), - (setup / "setup-linux-mshv.sh", "\n RUST_TOOLCHAIN=$tool_value\n"), - ( - setup / "setup-windows-whp.ps1", - '\n$RustToolchain = Get-ToolManifestValue $ToolManifest "rust.toolchain"\n', - ), - ( - root / ".github" / "specula" / "setup-runner.sh", - '\nRUST_VERSION="$(tool_value rust.toolchain "$VERSION_PATTERN")"\n', - ), - ): - with self.subTest(path=path.name): - self.assertIn(reader, path.read_text(encoding="utf-8")) - - def test_ci_takes_rust_toolchain_from_repository_pin(self): - github = BuildConstants.REPO_ROOT / ".github" - workflows = github / "workflows" - - validate_runner = ( - github / "actions" / "validate-runner" / "action.yml" - ).read_text(encoding="utf-8") - for step_name in ("Validate Linux toolchain", "Validate Windows toolchain"): - with self.subTest(step=step_name): - step = _composite_action_step(validate_runner, step_name) - self.assertIn("rust-toolchain.toml", step) - # Report a missing toolchain without writing the trusted RUSTUP_HOME. - self.assertEqual(_yaml_field(step, "RUSTUP_AUTO_INSTALL"), ['"0"']) - - crate_action = ( - github / "actions" / "check-aci-edge-sandboxes" / "action.yml" - ).read_text(encoding="utf-8") - self.assertEqual(_yaml_field(crate_action, "default"), ['""']) - install = _composite_action_step(crate_action, "Install Rust toolchains") - self.assertIn("../rust-toolchain.toml", install) - self.assertIn("REQUESTED_TOOLCHAIN: ${{ inputs.toolchain }}", install) - self.assertEqual(crate_action.count("${{ inputs.toolchain }}"), 1) - - setup = (workflows / "copilot-setup-steps.yml").read_text(encoding="utf-8") - self.assertIn(" - rust-toolchain.toml\n", setup) - self.assertIn( - "rust-toolchain.toml", - _workflow_step( - setup, "copilot-setup-steps", "Resolve pinned tool versions" - ), - ) - rust = _workflow_step(setup, "copilot-setup-steps", "Install Rust toolchains") - self.assertIn('rustup default "${RUST_TOOLCHAIN}"', rust) - self.assertNotIn("stable", rust) - - # A binary built by another compiler must not satisfy the cache. - build_action = (github / "actions" / "build-openvmm" / "action.yml").read_text( - encoding="utf-8" - ) - self.assertEqual( - build_action.count( - "${{ hashFiles('rust-toolchain.toml') }}-${{ steps.openvmm.outputs.sha }}" - ), - 4, - ) - - # A new release reruns every suite that it builds. - for pattern in ( - ci.CI_TEST_INPUT_PATHS, - ci.OPENVMM_UNIT_TEST_INPUT_PATHS, - ci.OPENVMM_VMM_TEST_INPUT_PATHS, - ): - self.assertIsNotNone(pattern.match("rust-toolchain.toml")) - - def test_release_actions_use_deterministic_immutable_tooling(self): - package_action = ( - BuildConstants.REPO_ROOT - / ".github" - / "actions" - / "package-release" - / "action.yml" - ).read_text(encoding="utf-8") - publish_action = ( - BuildConstants.REPO_ROOT - / ".github" - / "actions" - / "publish-development-release" - / "action.yml" - ).read_text(encoding="utf-8") - - self.assertEqual(package_action.count("archive-release"), 2) - self.assertIn( - "python scripts\\nvx.py materialize-kernel-provenance-inputs", - package_action, - ) - self.assertLess( - package_action.index( - "python scripts\\nvx.py materialize-kernel-provenance-inputs" - ), - package_action.index("- name: Package Windows release"), - ) - self.assertNotIn("git checkout-index", package_action) - self.assertNotIn("tar -czf", package_action) - self.assertNotIn("Compress-Archive", package_action) - self.assertIn( - "python3 -u scripts/publish_development_release.py", - publish_action, - ) - self.assertNotIn("--clobber", publish_action) - - def test_ci_artifact_actions_use_node24(self): - configurations = "\n".join( - path.read_text(encoding="utf-8") - for pattern in ("*.yml", "*.yaml") - for path in (BuildConstants.REPO_ROOT / ".github").rglob(pattern) - ) - - self.assertNotRegex(configurations, r"actions/upload-artifact@v[1-5]\b") - self.assertNotRegex(configurations, r"actions/download-artifact@v[1-6]\b") - - def test_benchmark_diagnostics_preserve_each_workflow_attempt(self): - action = ( - Path(__file__).parents[1] - / ".github" - / "actions" - / "run-benchmark" - / "action.yml" - ).read_text(encoding="utf-8") - diagnostics = _composite_action_step(action, "Upload benchmark diagnostics") - name = ( - "benchmark-diagnostics-${{ inputs.platform }}-${{ github.run_id }}" - "-attempt-${{ github.run_attempt }}" - ) - - self.assertIn(" if: always()", diagnostics) - self.assertIn(" uses: actions/upload-artifact@v7", diagnostics) - self.assertIn(" path: data/runs/${{ inputs.platform }}", diagnostics) - self.assertIn(f" name: {name}", diagnostics) - self.assertIn(" if-no-files-found: error", diagnostics) - self.assertIn(" overwrite: false", diagnostics) - self.assertIn(" retention-days: 1", diagnostics) - self.assertLess( - action.index("- name: Upload benchmark diagnostics"), - action.index("- name: Upload benchmark results"), - ) - - artifacts: dict[str, tuple[str, int]] = {} - for attempt in (1, 2): - for platform in ( - "linux-kvm-virtual-machine", - "linux-mshv-virtual-machine", - "windows-whp-virtual-machine", - ): - artifact = ( - name.replace("${{ inputs.platform }}", platform) - .replace("${{ github.run_id }}", "35771477786") - .replace("${{ github.run_attempt }}", str(attempt)) - ) - self.assertNotIn("${{", artifact) - self.assertNotIn(artifact, artifacts) - artifacts[artifact] = (platform, attempt) - self.assertEqual(len(artifacts), 6) - - def test_benchmark_result_handoff_keeps_run_scoped_names(self): - actions = Path(__file__).parents[1] / ".github" / "actions" - action = (actions / "run-benchmark" / "action.yml").read_text(encoding="utf-8") - prepare = (actions / "prepare-performance-results" / "action.yml").read_text( - encoding="utf-8" - ) - results = _composite_action_step(action, "Upload benchmark results") - name = "benchmark-${{ inputs.platform }}-${{ github.run_id }}" - - self.assertIn(f" name: {name}", results) - self.assertIn(" path: data/runs/${{ inputs.platform }}", results) - self.assertIn(" overwrite: true", results) - self.assertNotIn("github.run_attempt", results) - self.assertNotIn("github.run_attempt", prepare) - for platform in ( - "linux-kvm-virtual-machine", - "linux-mshv-virtual-machine", - "windows-whp-virtual-machine", - ): - with self.subTest(platform=platform): - artifact = name.replace("${{ inputs.platform }}", platform) - self.assertIn(f" name: {artifact}", prepare) - - def test_ci_remeasures_only_unstable_lifecycle_results(self): - action = ( - BuildConstants.REPO_ROOT - / ".github" - / "actions" - / "run-benchmark" - / "action.yml" - ).read_text(encoding="utf-8") - - self.assertEqual(action.count("performance validate-openvmm"), 2) - for step, loop, unstable in ( - ( - "Run Linux acceptance test", - "for attempt in 1 2; do", - '"${validation_status}" -ne 75', - ), - ( - "Run Windows acceptance test", - "foreach ($Attempt in 1, 2)", - "$ValidationStatus -ne 75", - ), - ): - with self.subTest(step=step): - script = _composite_action_script(action, step) - self.assertEqual(script.count("performance validate-openvmm"), 1) - self.assertEqual(script.count(loop), 1) - self.assertEqual(script.count(unstable), 1) - self.assertEqual( - script.count("Lifecycle snapshot generation was unstable"), - 1, - ) - self.assertEqual(script.count("--attempt-log"), 1) - self.assertIn("acceptance-attempts.json", script) - # Publication names the attempt that supplied the lifecycle metrics. - for step, log in ( - ( - "Publish Linux performance results", - '--lifecycle-attempt-log "${run_dir}/acceptance-attempts.json"', - ), - ( - "Publish Windows performance results", - '--lifecycle-attempt-log "$RunDir/acceptance-attempts.json"', - ), - ): - with self.subTest(step=step): - script = _composite_action_script(action, step) - self.assertEqual(script.count(log), 1) - self.assertLess(script.index("--lifecycle-input"), script.index(log)) - - def test_windows_benchmarks_use_provisioned_data_volume_scratch(self): - action = ( - BuildConstants.REPO_ROOT - / ".github" - / "actions" - / "run-benchmark" - / "action.yml" - ).read_text(encoding="utf-8") - setup = ( - BuildConstants.REPO_ROOT / "scripts" / "setup" / "setup-windows-whp.ps1" - ).read_text(encoding="utf-8") - prepare = _composite_action_step(action, "Prepare Windows benchmark scratch") - cleanup = _composite_action_step(action, "Remove Windows benchmark scratch") - - self.assertIn('$BenchmarkScratchVariable = "NVX_BENCHMARK_SCRATCH"', setup) - self.assertIn( - '[Environment]::GetEnvironmentVariable("NVX_BENCHMARK_SCRATCH", "Machine")', - prepare, - ) - self.assertIn('"NVX_BENCHMARK_SCRATCH_DIR=$Scratch"', prepare) - self.assertIn("::warning::NVX_BENCHMARK_SCRATCH is not provisioned", prepare) - self.assertIn(" if: always() && runner.os == 'Windows'", cleanup) - self.assertLess( - action.index("- name: Remove Windows benchmark scratch"), - action.index("- name: Upload benchmark diagnostics"), - ) - for step in ( - "Run Windows acceptance test", - "Run Windows performance suite", - "Run Windows multi-vCPU shell restore", - "Run Windows device operation rates", - ): - with self.subTest(step=step): - script = _composite_action_script(action, step) - self.assertIn( - '$ScratchArgs = @("--scratch-dir", $env:NVX_BENCHMARK_SCRATCH_DIR)', - script, - ) - self.assertIn("@ScratchArgs", script) - self.assertLess( - action.index("- name: Prepare Windows benchmark scratch"), - action.index(f"- name: {step}"), - ) - self.assertIn( - " Configure-SccacheEnvironment\n Configure-BenchmarkScratch\n", - setup, - ) - self.assertIn( - "Assert-ServiceDirectoryAcl -Path $SccacheDirectory -Writable\n" - " Assert-BenchmarkScratch\n", - setup, - ) - - @unittest.skipUnless(os.name == "nt", "requires Windows PowerShell") - def test_windows_acceptance_forwards_benchmark_scratch(self): - action = ( - BuildConstants.REPO_ROOT - / ".github" - / "actions" - / "run-benchmark" - / "action.yml" - ).read_text(encoding="utf-8") - script = _composite_action_script(action, "Run Windows acceptance test") - for name, value in ( - ("backend", "whp"), - ("platform", "windows-whp-virtual-machine"), - ("warmups", "1"), - ("runs", "10"), - ("teardown-mode", "guest-exit"), - ): - script = script.replace("${{ inputs." + name + " }}", value) - stub = """ -function python { - $global:LASTEXITCODE = 0 - if ($args[1] -ne "benchmark") { - return - } - $output = $args[[Array]::IndexOf($args, "--output") + 1] - $index = [Array]::IndexOf($args, "--scratch-dir") - $scratch = "" - if ($index -ge 0) { - $scratch = $args[$index + 1] - } - New-Item -ItemType Directory -Path (Split-Path $output) -Force | Out-Null - Set-Content -LiteralPath $output -Encoding UTF8 -Value ( - ConvertTo-Json -Compress @{ scratch = $scratch } - ) -} -""" - for scratch in ("", r"F:\nvx-benchmark-scratch\job"): - with ( - self.subTest(scratch=scratch), - tempfile.TemporaryDirectory() as temporary, - ): - root = Path(temporary).resolve() - environment = os.environ.copy() - environment.pop("NVX_BENCHMARK_SCRATCH_DIR", None) - if scratch: - environment["NVX_BENCHMARK_SCRATCH_DIR"] = scratch - result = subprocess.run( - [ - "powershell.exe", - "-NoProfile", - "-NonInteractive", - "-Command", - stub + script, - ], - cwd=root, - env=environment, - capture_output=True, - text=True, - timeout=30, - check=False, - ) - - self.assertEqual(result.returncode, 0, result.stderr) - accepted = ( - root - / "data" - / "runs" - / "windows-whp-virtual-machine" - / "microvm-v2" - / "1vcpu" - / "acceptance.json" - ) - self.assertEqual( - json.loads(accepted.read_text(encoding="utf-8-sig")), - {"scratch": scratch}, - ) - - @unittest.skipUnless(os.name == "nt", "requires Windows PowerShell") - def test_windows_benchmark_scratch_steps_manage_per_job_directories(self): - action = ( - BuildConstants.REPO_ROOT - / ".github" - / "actions" - / "run-benchmark" - / "action.yml" - ).read_text(encoding="utf-8") - machine_lookup = ( - '[Environment]::GetEnvironmentVariable("NVX_BENCHMARK_SCRATCH", "Machine")' - ) - prepare = _composite_action_script( - action, "Prepare Windows benchmark scratch" - ).replace("${{ inputs.platform }}", "windows-whp-virtual-machine") - cleanup = _composite_action_script(action, "Remove Windows benchmark scratch") - self.assertIn(machine_lookup, prepare) - - def run_step(script: str, environment: dict[str, str]): - return subprocess.run( - ["powershell.exe", "-NoProfile", "-NonInteractive", "-Command", script], - env=environment, - capture_output=True, - text=True, - timeout=30, - check=False, - ) - - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary).resolve() / "scratch" - root.mkdir() - stale = root / "windows-whp-virtual-machine-1-1" - fresh = root / "windows-whp-virtual-machine-2-1" - for directory in (stale, fresh): - directory.mkdir() - (directory / "memory.bin").write_bytes(b"ram") - old = time.time() - 7 * 60 * 60 - os.utime(stale, (old, old)) - github_env = Path(temporary) / "github-env.txt" - github_env.write_text("", encoding="utf-8") - environment = os.environ.copy() - environment.update( - { - "GITHUB_ENV": str(github_env), - "GITHUB_RUN_ID": "35805840174", - "GITHUB_RUN_ATTEMPT": "2", - } - ) - - result = run_step( - prepare.replace(machine_lookup, f"'{root}'"), - environment, - ) - - self.assertEqual(result.returncode, 0, result.stderr) - scratch = root / "windows-whp-virtual-machine-35805840174-2" - self.assertTrue(scratch.is_dir()) - self.assertFalse(stale.exists()) - self.assertTrue(fresh.is_dir()) - self.assertEqual( - github_env.read_text(encoding="utf-8-sig").splitlines(), - [f"NVX_BENCHMARK_SCRATCH_DIR={scratch}"], - ) - - locked = scratch / "locked.bin" - with locked.open("wb"): - result = run_step( - cleanup, - {**environment, "NVX_BENCHMARK_SCRATCH_DIR": str(scratch)}, - ) - self.assertEqual(result.returncode, 0, result.stderr) - self.assertIn( - "::warning::Could not remove benchmark scratch", result.stdout - ) - - result = run_step( - cleanup, - {**environment, "NVX_BENCHMARK_SCRATCH_DIR": str(scratch)}, - ) - self.assertEqual(result.returncode, 0, result.stderr) - self.assertFalse(scratch.exists()) - - github_env.write_text("", encoding="utf-8") - result = run_step(prepare.replace(machine_lookup, "''"), environment) - self.assertEqual(result.returncode, 0, result.stderr) - self.assertIn( - "::warning::NVX_BENCHMARK_SCRATCH is not provisioned", result.stdout - ) - self.assertEqual(github_env.read_text(encoding="utf-8"), "") - - @unittest.skipUnless(os.name == "nt", "requires Windows PowerShell") - def test_windows_setup_selects_largest_data_volume_for_benchmark_scratch(self): - setup = BuildConstants.REPO_ROOT / "scripts" / "setup" / "setup-windows-whp.ps1" - harness = f""" -Set-StrictMode -Version 2.0 -$ErrorActionPreference = "Stop" -$tokens = $null -$errors = $null -$ast = [System.Management.Automation.Language.Parser]::ParseFile( - '{setup}', - [ref]$tokens, - [ref]$errors -) -if ($errors.Count -ne 0) {{ - throw "setup script has parse errors" -}} -foreach ($name in "Test-SystemVolumePath", "Get-BenchmarkScratchDirectory") {{ - $definition = $ast.Find({{ - param($node) - $node -is [System.Management.Automation.Language.FunctionDefinitionAst] -and - $node.Name -eq $name - }}, $true) - . ([scriptblock]::Create($definition.Extent.Text)) -}} -$BenchmarkScratchVariable = "NVX_TEST_UNSET_{uuid.uuid4().hex}" -$BenchmarkScratchName = "nvx-benchmark-scratch" -$BenchmarkScratchDirectory = $null -$env:SystemDrive = "C:" -$Volumes = @( - [pscustomobject]@{{ DriveLetter = [char]"C"; DriveType = "Fixed"; FileSystem = "NTFS"; Size = 900GB }}, - [pscustomobject]@{{ DriveLetter = [char]"D"; DriveType = "Fixed"; FileSystem = "NTFS"; Size = 64GB }}, - [pscustomobject]@{{ DriveLetter = $null; DriveType = "Fixed"; FileSystem = "NTFS"; Size = 2TB }}, - [pscustomobject]@{{ DriveLetter = [char]"E"; DriveType = "CD-ROM"; FileSystem = ""; Size = 1TB }}, - [pscustomobject]@{{ DriveLetter = [char]"F"; DriveType = "Fixed"; FileSystem = "NTFS"; Size = 512GB }}, - [pscustomobject]@{{ DriveLetter = [char]"G"; DriveType = "Fixed"; FileSystem = "FAT32"; Size = 1TB }} -) -function Get-Volume {{ - $script:Volumes -}} -Write-Output (Get-BenchmarkScratchDirectory) -$Volumes = @($Volumes[0]) -Write-Output ("none=" + ($null -eq (Get-BenchmarkScratchDirectory))) -$BenchmarkScratchDirectory = "H:\\explicit\\scratch" -Write-Output (Get-BenchmarkScratchDirectory) -""" - - result = subprocess.run( - ["powershell.exe", "-NoProfile", "-NonInteractive", "-Command", harness], - capture_output=True, - text=True, - timeout=30, - check=False, - ) - - self.assertEqual(result.returncode, 0, result.stderr) - self.assertEqual( - result.stdout.splitlines(), - [ - r"F:\nvx-benchmark-scratch", - "none=True", - r"H:\explicit\scratch", - ], - ) - - def test_cli_validation_runs_every_script_test_module(self): - # Each scripts/test_*.py module must run in both CLI validation jobs; - # test_control_session.py once ran in neither (#229). - action = ( - BuildConstants.REPO_ROOT - / ".github" - / "actions" - / "validate-nvx" - / "action.yml" - ).read_text(encoding="utf-8") - modules = { - path.name - for path in (BuildConstants.REPO_ROOT / "scripts").glob("test_*.py") - } - self.assertIn("test_control_session.py", modules) - for step in ("Validate NVX CLI on Linux", "Validate NVX CLI on Windows"): - with self.subTest(step=step): - script = _composite_action_script(action, step) - invoked = re.findall(r"(?:^|\s)scripts[/\\](test_\w+\.py)", script) - self.assertEqual(sorted(invoked), sorted(modules)) - - def test_ci_runs_the_public_sandbox_acceptance_on_every_backend(self): - workflows = BuildConstants.REPO_ROOT / ".github" / "workflows" - workflow = (workflows / "ci.yml").read_text(encoding="utf-8") - microvm_workflow = (workflows / "run-nvx-microvm-tests.yml").read_text( - encoding="utf-8" - ) - for backend in ("kvm", "mshv", "whp"): - with self.subTest(backend=backend): - job = _workflow_job(workflow, f"nvx-microvm-tests-{backend}") - self.assertIn( - "uses: ./.github/workflows/run-nvx-microvm-tests.yml", job - ) - self.assertIn(f"backend: {backend}", job) - self.assertNotIn("debug-kernel: true", job) - for system in ("Linux", "Windows"): - with self.subTest(system=system): - acceptance = _workflow_step( - microvm_workflow, - "test", - f"Run public managed sandbox acceptance on {system}", - ) - self.assertIn("!inputs.debug-kernel", acceptance) - self.assertIn("--scenario managed-exec-config", acceptance) - self.assertIn("--scenario sandbox-lifecycle", acceptance) - self.assertIn( - '--output-dir "build/test-results/public-exec-${{ inputs.backend }}"', - acceptance, - ) - smoke = _workflow_step( - microvm_workflow, - "test", - f"Run Ubuntu sandbox layer smoke test on {system}", - ) - for option in ( - "--entrypoint /sbin/nvx-sandbox-smoke", - "--arg limits", - "--arg 33554432", - "--arg 8", - "--memory-max 33554432", - "--pids-max 8", - ): - self.assertIn(option, smoke) - upload = _workflow_step( - microvm_workflow, "test", "Upload NVX microVM failure logs" - ) - self.assertIn("build/test-results/public-exec-${{ inputs.backend }}", upload) - - @unittest.skipUnless(os.name == "nt", "requires Windows PowerShell") - def test_windows_cli_validation_stops_at_each_failed_command(self): - action = ( - BuildConstants.REPO_ROOT - / ".github" - / "actions" - / "validate-nvx" - / "action.yml" - ).read_text(encoding="utf-8") - script = _composite_action_script(action, "Validate NVX CLI on Windows") - command_count = sum(line.startswith("python ") for line in script.splitlines()) - self.assertGreater(command_count, 0) - stub = """ -$CommandCount = 0 -function python { - $script:CommandCount += 1 - Write-Output "python-$script:CommandCount" - $global:LASTEXITCODE = 0 - if ($script:CommandCount -eq $script:FailureAt) { - $global:LASTEXITCODE = 37 - } -} -""" - for failure_at in range(command_count + 1): - with self.subTest(failure_at=failure_at): - result = subprocess.run( - [ - "powershell.exe", - "-NoProfile", - "-NonInteractive", - "-Command", - f"$FailureAt = {failure_at}\n{stub}\n{script}", - ], - capture_output=True, - text=True, - timeout=30, - check=False, - ) - self.assertEqual( - result.returncode, 37 if failure_at else 0, result.stderr - ) - self.assertEqual( - result.stdout.splitlines(), - [ - f"python-{index}" - for index in range(1, (failure_at or command_count) + 1) - ], - ) - - @unittest.skipUnless(os.name == "nt", "requires Windows PowerShell") - def test_windows_acceptance_preserves_attempts_and_publishes_only_valid_data(self): - action = ( - BuildConstants.REPO_ROOT - / ".github" - / "actions" - / "run-benchmark" - / "action.yml" - ).read_text(encoding="utf-8") - script = _composite_action_script(action, "Run Windows acceptance test") - for name, value in ( - ("backend", "whp"), - ("platform", "windows-whp-virtual-machine"), - ("warmups", "1"), - ("runs", "10"), - ("teardown-mode", "guest-exit"), - ): - script = script.replace("${{ inputs." + name + " }}", value) - stub = """ -$BenchmarkCount = 0 -$ValidationCount = 0 -function python { - if ($args[1] -eq "benchmark") { - $script:BenchmarkCount += 1 - $output = $args[[Array]::IndexOf($args, "--output") + 1] - New-Item -ItemType Directory -Path (Split-Path $output) -Force | Out-Null - Set-Content -LiteralPath $output -Encoding UTF8 -Value ( - '{"attempt":' + $script:BenchmarkCount + '}' - ) - $global:LASTEXITCODE = $script:BenchmarkStatus - } - elseif ($args[1] -eq "performance") { - $log = $args[[Array]::IndexOf($args, "--attempt-log") + 1] - Add-Content -LiteralPath "attempt-logs.txt" -Value $log - $global:LASTEXITCODE = $script:ValidationStatuses[$script:ValidationCount] - $script:ValidationCount += 1 - } - else { - throw "unexpected command: $args" - } -} -""" - for name, statuses, benchmark_status in ( - ("first-success", (0,), 0), - ("remeasured-success", (75, 0), 0), - ("both-unstable", (75, 75), 0), - ("invalid-data", (1,), 0), - ("benchmark-failure", (0,), 42), - ): - with self.subTest(name=name), tempfile.TemporaryDirectory() as temporary: - root = Path(temporary).resolve() - output = ( - root - / "data" - / "runs" - / "windows-whp-virtual-machine" - / "microvm-v2" - / "1vcpu" - ) - output.mkdir(parents=True) - accepted = output / "acceptance.json" - accepted.write_text('{"attempt":"stale"}', encoding="utf-8") - # A rerun in the same directory must not keep a previous - # run's attempt log or later attempts. - attempt_log = output / "acceptance-attempts.json" - attempt_log.write_text('{"attempts":"stale"}', encoding="utf-8") - (output / "acceptance-attempt-2.json").write_text( - '{"attempt":"stale"}', encoding="utf-8" - ) - setup = ( - "$ValidationStatuses = @(" - + ", ".join(str(status) for status in statuses) - + f")\n$BenchmarkStatus = {benchmark_status}\n" - ) - result = subprocess.run( - [ - "powershell.exe", - "-NoProfile", - "-NonInteractive", - "-Command", - setup + stub + script, - ], - cwd=root, - capture_output=True, - text=True, - timeout=30, - check=False, - ) - expected_status = benchmark_status or statuses[-1] - self.assertEqual(result.returncode, expected_status, result.stderr) - self.assertFalse(attempt_log.exists()) - validations = root / "attempt-logs.txt" - self.assertEqual( - ( - validations.read_text(encoding="utf-8").splitlines() - if validations.exists() - else [] - ), - [ - "data/runs/windows-whp-virtual-machine/microvm-v2/1vcpu/" - "acceptance-attempts.json" - ] - * (0 if benchmark_status else len(statuses)), - ) - self.assertEqual( - sorted(path.name for path in output.glob("acceptance-attempt-*")), - [ - f"acceptance-attempt-{index}.json" - for index in range(1, len(statuses) + 1) - ], - ) - for index in range(1, len(statuses) + 1): - attempt = output / f"acceptance-attempt-{index}.json" - self.assertEqual( - json.loads(attempt.read_text(encoding="utf-8-sig")), - {"attempt": index}, - ) - self.assertEqual(accepted.exists(), expected_status == 0) - if expected_status == 0: - self.assertEqual( - json.loads(accepted.read_text(encoding="utf-8-sig")), - {"attempt": len(statuses)}, - ) - - @unittest.skipUnless(os.name == "posix", "requires bash") - def test_linux_acceptance_preserves_attempts_and_publishes_only_valid_data(self): - action = ( - BuildConstants.REPO_ROOT - / ".github" - / "actions" - / "run-benchmark" - / "action.yml" - ).read_text(encoding="utf-8") - script = _composite_action_script(action, "Run Linux acceptance test") - platform = "linux-kvm-virtual-machine" - for name, value in ( - ("backend", "kvm"), - ("platform", platform), - ("warmups", "1"), - ("runs", "10"), - ("teardown-mode", "guest-exit"), - ): - script = script.replace("${{ inputs." + name + " }}", value) - stub = """ -benchmark_count=0 -validation_count=0 -python3() { - local argument output="" input="" platform="" attempt_log="" previous="" - for argument in "$@"; do - case "${previous}" in - --output) output="${argument}" ;; - --input) input="${argument}" ;; - --platform) platform="${argument}" ;; - --attempt-log) attempt_log="${argument}" ;; - esac - previous="${argument}" - done - if [[ "${2-}" == benchmark ]]; then - benchmark_count=$((benchmark_count + 1)) - echo "benchmark ${platform} ${output}" - mkdir -p "$(dirname "${output}")" - printf '{"attempt":%d}' "${benchmark_count}" >"${output}" - return "${BENCHMARK_STATUS}" - fi - if [[ "${2-} ${3-}" == "performance validate-openvmm" ]]; then - validation_count=$((validation_count + 1)) - echo "validate ${platform} ${input} ${attempt_log}" - return "${VALIDATION_STATUSES[validation_count - 1]}" - fi - echo "unexpected command: $*" >&2 - return 1 -} -""" - environment = os.environ.copy() - environment.update( - { - "NVX_BENCHMARK_CPUS": "0-7", - "NVX_BENCHMARK_HOST_CPU_RESERVE": "0", - "NVX_BENCHMARK_MARKER_TIMEOUT": "40", - } - ) - run_dir = f"data/runs/{platform}/microvm-v2/1vcpu" - for name, statuses, benchmark_status in ( - ("first-success", (0,), 0), - ("remeasured-success", (75, 0), 0), - ("both-unstable", (75, 75), 0), - ("invalid-data", (2,), 0), - ("unstable-then-invalid", (75, 2), 0), - ("benchmark-failure", (0,), 42), - ): - with self.subTest(name=name), tempfile.TemporaryDirectory() as temporary: - root = Path(temporary).resolve() - output = root / run_dir - output.mkdir(parents=True) - accepted = output / "acceptance.json" - accepted.write_text('{"attempt":"stale"}', encoding="utf-8") - # A rerun in the same directory must not keep a previous - # run's attempt log or later attempts. - attempt_log = output / "acceptance-attempts.json" - attempt_log.write_text('{"attempts":"stale"}', encoding="utf-8") - (output / "acceptance-attempt-2.json").write_text( - '{"attempt":"stale"}', encoding="utf-8" - ) - setup = ( - "VALIDATION_STATUSES=(" - + " ".join(str(status) for status in statuses) - + f")\nBENCHMARK_STATUS={benchmark_status}\n" - ) - result = subprocess.run( - [ - "bash", - "--noprofile", - "--norc", - "-eo", - "pipefail", - "-c", - setup + stub + script, - ], - cwd=root, - env=environment, - capture_output=True, - text=True, - timeout=30, - check=False, - ) - expected_status = benchmark_status or statuses[-1] - self.assertEqual(result.returncode, expected_status, result.stderr) - self.assertFalse(attempt_log.exists()) - expected_stdout: list[str] = [] - for index in range(1, len(statuses) + 1): - attempt = f"{run_dir}/acceptance-attempt-{index}.json" - if index > 1: - expected_stdout.append( - "::warning::Lifecycle snapshot generation was unstable " - f"on attempt {index - 1}; remeasuring once" - ) - expected_stdout.append(f"benchmark {platform} {attempt}") - if not benchmark_status: - expected_stdout.append( - f"validate {platform} {attempt} " - f"{run_dir}/acceptance-attempts.json" - ) - self.assertEqual(result.stdout.splitlines(), expected_stdout) - self.assertEqual( - sorted(path.name for path in output.glob("acceptance-attempt-*")), - [ - f"acceptance-attempt-{index}.json" - for index in range(1, len(statuses) + 1) - ], - ) - for index in range(1, len(statuses) + 1): - attempt = output / f"acceptance-attempt-{index}.json" - self.assertEqual( - json.loads(attempt.read_text(encoding="utf-8")), - {"attempt": index}, - ) - self.assertEqual(accepted.exists(), expected_status == 0) - if expected_status == 0: - self.assertEqual( - json.loads(accepted.read_text(encoding="utf-8")), - {"attempt": len(statuses)}, - ) - - -class BuildConstantsTests(unittest.TestCase): - def test_constants_are_grouped_class_namespaces(self): - for namespace in ( - BuildConstants, - KernelBuildConstants, - OpenVMMBuildConstants, - AlpineBuildConstants, - UbuntuBuildConstants, - InitramfsBuildConstants, - DockerBuildConstants, - ZstdBuildConstants, - ReleaseBuildConstants, - ): - with self.subTest(namespace=namespace.__name__): - self.assertEqual(namespace.__module__, "nvx_tools.build_constants") - self.assertTrue( - all( - name.isupper() - for name in vars(namespace) - if not name.startswith("__") - ) - ) - - def test_build_pins_and_artifact_names_have_one_literal_owner(self): - centralized = { - KernelBuildConstants.VERSION, - KernelBuildConstants.URL, - KernelBuildConstants.SHA256, - KernelBuildConstants.BINARY_NAME, - KernelBuildConstants.CONFIG_NAME, - KernelBuildConstants.PROVENANCE_NAME, - KernelBuildConstants.DEBUG_BINARY_NAME, - KernelBuildConstants.DEBUG_CONFIG_NAME, - KernelBuildConstants.DEBUG_PROVENANCE_NAME, - OpenVMMBuildConstants.GNU_RUST_TARGET, - OpenVMMBuildConstants.MUSL_RUST_TARGET, - OpenVMMBuildConstants.WINDOWS_RUST_TARGET, - OpenVMMBuildConstants.PROVENANCE_NAME, - OpenVMMBuildConstants.CONTROL_CONTRACT_REVISION, - AlpineBuildConstants.VERSION, - AlpineBuildConstants.BRANCH, - AlpineBuildConstants.MINIROOTFS_SHA256, - AlpineBuildConstants.INITRAMFS_NAME, - AlpineBuildConstants.PACKAGE_MANIFEST_NAME, - UbuntuBuildConstants.VERSION, - UbuntuBuildConstants.BASE_URL, - UbuntuBuildConstants.BASE_SHA256, - UbuntuBuildConstants.INITRAMFS_NAME, - UbuntuBuildConstants.PACKAGE_MANIFEST_NAME, - UbuntuBuildConstants.DISTRO_NAME, - UbuntuBuildConstants.DISTRO_MANIFEST_NAME, - InitramfsBuildConstants.PROVENANCE_NAME, - ZstdBuildConstants.VERSION, - ZstdBuildConstants.SHA256, - } - duplicates: list[tuple[str, int, str]] = [] - for path in (BuildConstants.REPO_ROOT / "scripts").rglob("*.py"): - if path.name == "build_constants.py" or path.name.startswith("test_"): - continue - tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) - duplicates.extend( - (path.name, node.lineno, node.value) - for node in ast.walk(tree) - if isinstance(node, ast.Constant) - and isinstance(node.value, str) - and node.value in centralized - ) - self.assertEqual(duplicates, []) - - def test_old_module_level_constants_are_not_reexported(self): - names = ( - "REPO_ROOT", - "BUILD_DIR", - "SOURCE_DIR", - "OPENVMM_DIR", - "DEFAULT_KERNEL_VERSION", - "DEFAULT_KERNEL_URL", - "DEFAULT_KERNEL_SHA256", - "DEFAULT_ALPINE_VERSION", - "DEFAULT_ALPINE_BRANCH", - "DEFAULT_ALPINE_MINIROOTFS_SHA256", - "DEFAULT_UBUNTU_VERSION", - "DEFAULT_UBUNTU_CODENAME", - "DEFAULT_UBUNTU_ARCHITECTURE", - "DEFAULT_UBUNTU_BASE_URL", - "DEFAULT_UBUNTU_BASE_SHA256", - "UBUNTU_PACKAGE_LOCK", - "UBUNTU_EROFS_FORMAT", - "MICROVM_ABI_VERSION", - "CONTROL_SESSION_PROTOCOL_VERSION", - "CONTROL_CONTRACT_REVISION", - "OPENVMM_PROVENANCE_NAME", - "KERNEL_PROVENANCE_NAME", - "INITRAMFS_PROVENANCE_NAME", - "REQUIRED_VIRTIO_CONSOLE_CONFIG", - "REQUIRED_SHARED_STATUS_KERNEL_CONFIG", - "REQUIRED_SANDBOX_KERNEL_CONFIG", - "ZSTD_VERSION", - "ZSTD_ARCHIVE", - "ZSTD_URL", - "ZSTD_SHA256", - "OPENVMM_RUST_TOOLCHAIN", - "OPENVMM_GUEST_RUST_TARGET", - "OPENVMM_UEFI_RUST_TARGET", - "OPENVMM_LINUX_MUSL_RUST_TARGET", - "OPENVMM_RUST_TARGETS", - "APORTS_URL", - "REPOSITORIES", - "UBUNTU_ARCHIVE_KEYRING_URL", - "UBUNTU_ARCHIVE_KEYRING_SHA256", - "UBUNTU_SNAPSHOT_ARCHIVE_URL", - "UBUNTU_SOURCE_INDEXES", - "_UBUNTU_POCKET_SUITES", - "_UBUNTU_COMPONENTS", - "_LAUNCHPAD_ARCHIVE_API", - "_LAUNCHPAD_SERIES_API", - "_GENERATED_OUTPUT_ENTRIES", - "_KNOWN_PACKAGE_CONTROL_ACTIONS", - "PROJECT_SOURCE_PATHS", - "GUEST_RELEASE_NAMES", - "_ZIP_TIMESTAMP", - ) - for module in ( - build, - build_config, - common, - ubuntu, - ci, - collect_alpine_sources, - collect_ubuntu_sources, - release, - benchmark, - archive, - ): - with self.subTest(module=module.__name__): - self.assertFalse(set(names).intersection(vars(module))) - - def test_guest_descriptors_and_source_manifest_share_build_pins(self): - manifest = json.loads( - (BuildConstants.REPO_ROOT / "SOURCE-MANIFEST.json").read_text( - encoding="utf-8" - ) - ) - self.assertEqual(KernelBuildConstants.VERSION, manifest["linux"]["version"]) - self.assertEqual(KernelBuildConstants.URL, manifest["linux"]["upstream_url"]) - self.assertEqual( - KernelBuildConstants.SHA256, - manifest["linux"]["upstream_archive_sha256"], - ) - self.assertEqual( - AlpineBuildConstants.MINIROOTFS_URL, manifest["alpine"]["minirootfs_url"] - ) - self.assertEqual( - AlpineBuildConstants.MINIROOTFS_SHA256, - manifest["alpine"]["minirootfs_sha256"], - ) - for descriptor in (guests.ALPINE_GUEST, guests.UBUNTU_GUEST): - with self.subTest(guest=descriptor.name): - pinned = manifest[descriptor.name] - self.assertEqual(descriptor.release, pinned["version"]) - self.assertEqual(descriptor.architecture, pinned["architecture"]) - self.assertEqual( - f"build/{descriptor.package_manifest_name}", - pinned["package_manifests"][0], - ) - self.assertEqual( - OpenVMMBuildConstants.MICROVM_ABI_VERSION, - manifest["openvmm"]["microvm_abi_version"], - ) - self.assertEqual( - OpenVMMBuildConstants.CONTROL_SESSION_PROTOCOL_VERSION, - manifest["openvmm"]["control_session_protocol_version"], - ) - self.assertEqual( - OpenVMMBuildConstants.CONTROL_CONTRACT_REVISION, - manifest["openvmm"]["control_contract_revision"], - ) - - def test_cache_defaults_are_resolved_for_each_configuration(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - for configured in ("", str(root / "first"), str(root / "second")): - with ( - self.subTest(configured=configured), - patch.dict(os.environ, {"NVX_CACHE_DIR": configured}), - ): - expected = ( - Path(configured) - if configured - else BuildConstants.REPO_ROOT / ".cache" - ).resolve() - self.assertEqual( - build_config.KernelBuildConfig().cache_directory, expected - ) - self.assertEqual(common.cache_root(), expected) - self.assertEqual( - build_config.KernelBuildConfig( - cache_directory=root / "explicit" - ).cache_directory, - root / "explicit", - ) - - def test_openvmm_executable_default_is_resolved_at_configuration_time(self): - for host, binary in (("nt", "openvmm.exe"), ("posix", "openvmm")): - expected = OpenVMMBuildConstants.DIRECTORY / "target" / "release" / binary - with self.subTest(host=host), patch.object(common.os, "name", host): - self.assertEqual(common.openvmm_binary_path(), expected) - self.assertEqual(build_config.OpenVmmBuildConfig().output, expected) - - def test_initramfs_provenance_tracks_constants_source(self): - relative = "scripts/nvx_tools/build_constants.py" - source = BuildConstants.REPO_ROOT / relative - original = build.initramfs_provenance_inputs() - records = cast(list[dict[str, str]], original["source_files"]) - self.assertEqual( - [record for record in records if record["path"] == relative], - [{"path": relative, "sha256": common.sha256_file(source)}], - ) - - def changed_sha256(path: Path) -> str: - return "f" * 64 if path == source else common.sha256_file(path) - - with patch.object(build, "sha256_file", side_effect=changed_sha256): - changed = build.initramfs_provenance_inputs() - self.assertNotEqual(original, changed) - self.assertEqual(original["alpine"], changed["alpine"]) - - def test_constants_are_in_docker_and_release_source_inputs(self): - dockerfile = (BuildConstants.REPO_ROOT / "docker" / "Dockerfile").read_text( - encoding="utf-8" - ) - self.assertIn("COPY scripts/ /repo/scripts/", dockerfile) - self.assertIn("scripts", ReleaseBuildConstants.PROJECT_SOURCE_PATHS) - self.assertTrue( - ( - BuildConstants.REPO_ROOT - / "scripts" - / "nvx_tools" - / "build_constants.py" - ).is_file() - ) - - def test_constants_consumers_import_in_either_order(self): - modules = ( - "nvx_tools.build_constants", - "nvx_tools.guests", - "nvx_tools.common", - "nvx_tools.build_config", - "nvx_tools.build", - "nvx_tools.release", - "nvx", - ) - for ordered in (modules, tuple(reversed(modules))): - with self.subTest(first=ordered[0]): - result = subprocess.run( - [ - sys.executable, - "-c", - "import importlib, sys; " - "[importlib.import_module(name) for name in sys.argv[1:]]", - *ordered, - ], - cwd=BuildConstants.REPO_ROOT / "scripts", - capture_output=True, - text=True, - timeout=30, - check=False, - ) - self.assertEqual(result.returncode, 0, result.stderr) - - -class UbuntuSourceCollectionTests(unittest.TestCase): - def test_malformed_package_manifest_is_reported_as_script_error(self): - with tempfile.TemporaryDirectory() as temporary: - manifest = Path(temporary) / "packages.json" - manifest.write_text("{", encoding="utf-8") - - with self.assertRaises(collect_ubuntu_sources.ScriptError) as context: - collect_ubuntu_sources._source_requirements([manifest]) - - self.assertIn("cannot read Ubuntu package manifest", str(context.exception)) - - -class AlpineSourceCollectionTests(unittest.TestCase): - INSTALL_SCRIPT = b"#!/bin/sh\nexit 0\n" - - @staticmethod - def _aports_commit(cache: Path, files: dict[str, tuple[str, bytes]]) -> str: - environment = { - key: value - for key, value in os.environ.items() - if not key.startswith("GIT_CONFIG_") - } - environment.update( - GIT_AUTHOR_NAME="NVX Tests", - GIT_AUTHOR_EMAIL="nvx-tests@example.com", - GIT_COMMITTER_NAME="NVX Tests", - GIT_COMMITTER_EMAIL="nvx-tests@example.com", - ) - - def git(*args: str, data: bytes | None = None) -> str: - result = subprocess.run( - ["git", "-C", str(cache), *args], - input=data, - env=environment, - capture_output=True, - check=True, - ) - return result.stdout.decode("ascii").strip() - - cache.mkdir(parents=True, exist_ok=True) - git("init", "-q") - git("config", "core.autocrlf", "false") - for path, (mode, contents) in files.items(): - blob = git("hash-object", "-w", "--stdin", data=contents) - git("update-index", "--add", "--cacheinfo", f"{mode},{blob},{path}") - return git("commit-tree", git("write-tree"), "-m", "aports fixture") - - def _symlinked_recipe_commit(self, cache: Path) -> str: - return self._aports_commit( - cache, - { - "main/foo/APKBUILD": ("100644", b"pkgname=foo\n"), - "main/foo/foo.post-install": ("100755", self.INSTALL_SCRIPT), - "main/foo/foo.post-upgrade": ("120000", b"foo.post-install"), - }, - ) - - @staticmethod - def _package_manifest(root: Path, commit: str) -> Path: - manifest = root / "packages.json" - manifest.write_text( - json.dumps( - { - "guest": AlpineBuildConstants.GUEST_NAME, - "alpine_branch": AlpineBuildConstants.BRANCH, - "architecture": AlpineBuildConstants.ARCHITECTURE, - "packages": [ - { - "name": "foo", - "version": "1.0-r0", - "origin": "foo", - "license": "MIT", - "aports_commit": commit, - } - ], - } - ), - encoding="utf-8", - ) - return manifest - - @staticmethod - def _member( - name: str, - kind: bytes = tarfile.REGTYPE, - data: bytes = b"", - *, - linkname: str = "", - mode: int = 0o664, - ) -> tuple[tarfile.TarInfo, bytes]: - member = tarfile.TarInfo(name) - member.type = kind - member.linkname = linkname - member.mode = mode - member.size = len(data) if kind == tarfile.REGTYPE else 0 - return member, data - - @staticmethod - def _archive(*members: tuple[tarfile.TarInfo, bytes]) -> bytes: - buffer = io.BytesIO() - with tarfile.open(fileobj=buffer, mode="w") as archive: - for member, data in members: - archive.addfile(member, io.BytesIO(data) if member.isfile() else None) - return buffer.getvalue() - - def test_malformed_package_manifest_is_reported_as_source_error(self): - with tempfile.TemporaryDirectory() as temporary: - for contents in ("{", "[]"): - manifest = Path(temporary) / "packages.json" - manifest.write_text(contents, encoding="utf-8") - - with ( - self.subTest(contents=contents), - self.assertRaises(collect_alpine_sources.SourceError) as context, - ): - collect_alpine_sources._load_packages([manifest]) - - self.assertIn("package manifest", str(context.exception)) - - def test_recipe_collection_preserves_blob_line_endings(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - cache = root / "aports" - apkbuild = b"pkgname=foo\npkgver=1.0\npkgrel=0\n" - commit = self._aports_commit( - cache, - {"main/foo/APKBUILD": ("100644", apkbuild)}, - ) - subprocess.run( - ["git", "-C", str(cache), "config", "core.autocrlf", "true"], - check=True, - ) - output = root / "alpine" - - with ( - patch.object(collect_alpine_sources, "_prepare_aports"), - contextlib.redirect_stdout(io.StringIO()), - ): - collect_alpine_sources.collect_alpine_sources( - [self._package_manifest(root, commit)], - output, - cache, - skip_upstream=True, - ) - - collected = output / "recipes" / commit / "main/foo/APKBUILD" - self.assertEqual(collected.read_bytes(), apkbuild) - - def test_recipe_symlink_is_collected_as_verified_regular_file(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - cache = root / "aports" - commit = self._symlinked_recipe_commit(cache) - output = root / "alpine" - - with ( - patch.object(collect_alpine_sources, "_prepare_aports"), - contextlib.redirect_stdout(io.StringIO()), - ): - collect_alpine_sources.collect_alpine_sources( - [self._package_manifest(root, commit)], - output, - cache, - skip_upstream=True, - ) - - upgrade = output / "recipes" / commit / "main/foo/foo.post-upgrade" - self.assertFalse(upgrade.is_symlink()) - self.assertEqual(upgrade.read_bytes(), self.INSTALL_SCRIPT) - source_manifest = json.loads( - (output / "manifest.json").read_text(encoding="utf-8") - ) - self.assertEqual( - source_manifest["format"], AlpineBuildConstants.SOURCE_MANIFEST_FORMAT - ) - self.assertEqual( - source_manifest["executables"], - [ - f"recipes/{commit}/main/foo/foo.post-install", - f"recipes/{commit}/main/foo/foo.post-upgrade", - ], - ) - self.assertIn( - f"recipes/{commit}/main/foo/foo.post-upgrade", - dict(common.verify_sha256_sums(output).files), - ) - if os.name == "posix": - self.assertEqual(upgrade.stat().st_mode & 0o777, 0o755) - - def test_collection_replaces_stale_output_and_passes_release_validation(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - cache = root / "aports" - commit = self._symlinked_recipe_commit(cache) - manifest = self._package_manifest(root, commit) - source_dir = root / "sources" - output = source_dir / "alpine" - stale_files = ( - output / "recipes" / ("0" * 40) / "main/stale/APKBUILD", - output / "upstream" / ("0" * 40) / "main/stale/stale.tar.gz", - ) - for stale in stale_files: - stale.parent.mkdir(parents=True) - stale.write_bytes(b"stale") - (output / "manifest.json").write_text("{}", encoding="utf-8") - (output / "SHA256SUMS").write_text("", encoding="ascii") - - def fetch_upstream(bundle: Path, _alpine_version: str) -> None: - upstream = bundle / "upstream" / commit / "main" / "foo" - upstream.mkdir(parents=True) - (upstream / "foo-1.0.tar.gz").write_bytes(b"upstream") - - with ( - patch.object(collect_alpine_sources, "_prepare_aports"), - patch.object( - collect_alpine_sources, - "_fetch_upstream_sources", - side_effect=fetch_upstream, - ), - patch.object(BuildConstants, "SOURCE_DIR", source_dir), - contextlib.redirect_stdout(io.StringIO()), - ): - collect_alpine_sources.collect_alpine_sources([manifest], output, cache) - executables = release._validate_alpine_sources([manifest]) - - self.assertEqual( - executables, - { - f"recipes/{commit}/main/foo/foo.post-install", - f"recipes/{commit}/main/foo/foo.post-upgrade", - }, - ) - for stale in stale_files: - self.assertFalse(stale.parent.parent.exists(), stale) - self.assertEqual( - sorted(path.name for path in output.iterdir()), - ["SHA256SUMS", "manifest.json", "recipes", "upstream"], - ) - - def test_recipe_extraction_materializes_symlinks_inside_recipe(self): - member = self._member - archive = self._archive( - member("main/", tarfile.DIRTYPE, mode=0o775), - member("main/foo/", tarfile.DIRTYPE, mode=0o775), - member("main/foo/patches/", tarfile.DIRTYPE, mode=0o775), - member("main/foo/APKBUILD", data=b"pkgname=foo\n"), - member("main/foo/foo.post-install", data=self.INSTALL_SCRIPT, mode=0o775), - member( - "main/foo/foo.post-upgrade", - tarfile.SYMTYPE, - linkname="foo.post-install", - ), - member( - "main/foo/foo.pre-upgrade", - tarfile.SYMTYPE, - linkname="./foo.post-upgrade", - ), - member("main/foo/patches/fix.patch", data=b"patch\n"), - member( - "main/foo/fix.patch", - tarfile.SYMTYPE, - linkname="patches/../patches/fix.patch", - ), - member( - "main/foo/patches/APKBUILD", tarfile.SYMTYPE, linkname="../APKBUILD" - ), - ) - with tempfile.TemporaryDirectory() as temporary: - destination = Path(temporary) - - executables = collect_alpine_sources._safe_extract( - archive, destination, "main/foo" - ) - - self.assertEqual( - executables, - [ - "main/foo/foo.post-install", - "main/foo/foo.post-upgrade", - "main/foo/foo.pre-upgrade", - ], - ) - recipe = destination / "main" / "foo" - expected = { - "APKBUILD": (b"pkgname=foo\n", 0o644), - "foo.post-install": (self.INSTALL_SCRIPT, 0o755), - "foo.post-upgrade": (self.INSTALL_SCRIPT, 0o755), - "foo.pre-upgrade": (self.INSTALL_SCRIPT, 0o755), - "fix.patch": (b"patch\n", 0o644), - "patches/fix.patch": (b"patch\n", 0o644), - "patches/APKBUILD": (b"pkgname=foo\n", 0o644), - } - for relative, (contents, mode) in expected.items(): - with self.subTest(path=relative): - path = recipe / relative - self.assertFalse(path.is_symlink()) - self.assertEqual(path.read_bytes(), contents) - if os.name == "posix": - self.assertEqual(path.stat().st_mode & 0o777, mode) - - def test_recipe_extraction_accepts_implicit_directories(self): - member = self._member - archive = self._archive( - member("main/foo/APKBUILD", data=b"pkgname=foo\n"), - member("main/foo/scripts/install", data=self.INSTALL_SCRIPT, mode=0o775), - member( - "main/foo/foo.post-install", - tarfile.SYMTYPE, - linkname="scripts/install", - ), - member( - "main/foo/scripts/APKBUILD", tarfile.SYMTYPE, linkname="../APKBUILD" - ), - ) - with tempfile.TemporaryDirectory() as temporary: - destination = Path(temporary) - - executables = collect_alpine_sources._safe_extract( - archive, destination, "main/foo" - ) - - self.assertEqual( - executables, - ["main/foo/foo.post-install", "main/foo/scripts/install"], - ) - recipe = destination / "main" / "foo" - for relative, contents in ( - ("foo.post-install", self.INSTALL_SCRIPT), - ("scripts/APKBUILD", b"pkgname=foo\n"), - ): - with self.subTest(path=relative): - path = recipe / relative - self.assertFalse(path.is_symlink()) - self.assertEqual(path.read_bytes(), contents) - if os.name == "posix": - for directory in (destination / "main", recipe, recipe / "scripts"): - with self.subTest(directory=directory.name): - self.assertEqual(directory.stat().st_mode & 0o777, 0o755) - - def test_recipe_extraction_rejects_unsafe_members(self): - member = self._member - recipe = ( - member("main/", tarfile.DIRTYPE), - member("main/foo/", tarfile.DIRTYPE), - member("main/foo/sub/", tarfile.DIRTYPE), - member("main/foo/APKBUILD", data=b"pkgname=foo\n"), - member("main/foo/sub/file", data=b"file\n"), - ) - cases: dict[str, tuple[tuple[tarfile.TarInfo, bytes], ...]] = { - "absolute path": (member("/main/foo/file"),), - "parent path": (member("main/foo/../../file"),), - "outside recipe": (member("main/bar/file"),), - "duplicate path": (member("main/foo/APKBUILD"),), - "member below a file": (member("main/foo/APKBUILD/file"),), - "member below a symlink": ( - member("main/foo/alias", tarfile.SYMTYPE, linkname="sub"), - member("main/foo/alias/file"), - ), - "hard link": ( - member("main/foo/link", tarfile.LNKTYPE, linkname="main/foo/APKBUILD"), - ), - "fifo": (member("main/foo/pipe", tarfile.FIFOTYPE),), - "character device": (member("main/foo/device", tarfile.CHRTYPE),), - "absolute symlink": ( - member("main/foo/link", tarfile.SYMTYPE, linkname="/etc/passwd"), - ), - "escaping symlink": ( - member("main/foo/link", tarfile.SYMTYPE, linkname="../../../passwd"), - ), - "sibling recipe symlink": ( - member("main/foo/link", tarfile.SYMTYPE, linkname="../bar/APKBUILD"), - ), - "dangling symlink": ( - member("main/foo/link", tarfile.SYMTYPE, linkname="missing"), - ), - "symlink through a missing directory": ( - member( - "main/foo/link", - tarfile.SYMTYPE, - linkname="missing/../APKBUILD", - ), - ), - "directory symlink": ( - member("main/foo/link", tarfile.SYMTYPE, linkname="sub"), - ), - "symlink loop": ( - member("main/foo/first", tarfile.SYMTYPE, linkname="second"), - member("main/foo/second", tarfile.SYMTYPE, linkname="first"), - ), - "symlinked directory traversal": ( - member("main/foo/link", tarfile.SYMTYPE, linkname="alias/file"), - member("main/foo/alias", tarfile.SYMTYPE, linkname="sub"), - ), - } - for name, members in cases.items(): - with self.subTest(case=name), tempfile.TemporaryDirectory() as temporary: - destination = Path(temporary) / "recipes" - - with self.assertRaises(collect_alpine_sources.SourceError): - collect_alpine_sources._safe_extract( - self._archive(*recipe, *members), - destination, - "main/foo", - ) - - self.assertEqual(list(destination.iterdir()), []) - - def test_collection_guards_output_before_replacing_it(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - manifest = self._package_manifest(root, "0" * 40) - output = root / "alpine" - (output / "recipes").mkdir(parents=True) - notes = output / "notes.txt" - notes.write_text("keep", encoding="utf-8") - not_directory = root / "file" - not_directory.write_text("keep", encoding="utf-8") - - for target, message in ( - (output, "unexpected entries: notes.txt"), - (not_directory, "is not a directory"), - ): - with ( - self.subTest(output=target.name), - patch.object(collect_alpine_sources, "_prepare_aports") as prepare, - self.assertRaisesRegex( - collect_alpine_sources.SourceError, - message, - ), - ): - collect_alpine_sources.collect_alpine_sources( - [manifest], - target, - root / "aports", - skip_upstream=True, - ) - prepare.assert_not_called() - self.assertTrue(notes.is_file()) - self.assertTrue((output / "recipes").is_dir()) - - notes.unlink() - with ( - patch.object(collect_alpine_sources, "_prepare_aports"), - patch.object( - collect_alpine_sources.shutil, - "rmtree", - side_effect=PermissionError(13, "Permission denied"), - ), - self.assertRaisesRegex( - collect_alpine_sources.SourceError, - "cannot remove previous Alpine source output.*remove it manually", - ), - ): - collect_alpine_sources.collect_alpine_sources( - [manifest], - output, - root / "aports", - skip_upstream=True, - ) - - def test_collection_refuses_symlinked_output(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - output = root / "alpine" - (output / "recipes").mkdir(parents=True) - link = root / "alpine-link" - try: - link.symlink_to(output, target_is_directory=True) - except OSError as error: - self.skipTest(f"symlinks are unavailable: {error}") - - with ( - patch.object(collect_alpine_sources, "_prepare_aports") as prepare, - self.assertRaisesRegex( - collect_alpine_sources.SourceError, - "Alpine source output is not a directory", - ), - ): - collect_alpine_sources.collect_alpine_sources( - [self._package_manifest(root, "0" * 40)], - link, - root / "aports", - skip_upstream=True, - ) - - prepare.assert_not_called() - self.assertTrue((output / "recipes").is_dir()) - - def test_collection_refuses_output_junctions(self): - if sys.platform != "win32": - self.skipTest("directory junctions exist only on Windows") - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - manifest = self._package_manifest(root, "0" * 40) - elsewhere = root / "elsewhere" - (elsewhere / "recipes").mkdir(parents=True) - kept = elsewhere / "recipes" / "kept" - kept.write_bytes(b"kept") - output = root / "alpine" - output.mkdir() - cases = ( - ( - root / "junction", - elsewhere, - root / "junction", - "Alpine source output is not a directory", - ), - ( - output / "recipes", - elsewhere / "recipes", - output, - "Alpine source generated path is not a directory", - ), - ) - for link, target, collected, message in cases: - subprocess.run( - ["cmd", "/c", "mklink", "/J", str(link), str(target)], - capture_output=True, - check=True, - ) - with ( - self.subTest(link=link.name), - patch.object(collect_alpine_sources, "_prepare_aports"), - self.assertRaisesRegex(collect_alpine_sources.SourceError, message), - ): - collect_alpine_sources.collect_alpine_sources( - [manifest], - collected, - root / "aports", - skip_upstream=True, - ) - self.assertEqual(kept.read_bytes(), b"kept") - - def test_upstream_fetch_restores_bundle_ownership_on_exit(self): - with ( - tempfile.TemporaryDirectory() as temporary, - patch.object(collect_alpine_sources, "_run") as run, - ): - collect_alpine_sources._fetch_upstream_sources(Path(temporary), "3.24") - - command = run.call_args.args[0] - self.assertEqual(command[:4], ["docker", "run", "--rm", "--volume"]) - script = command[-1] - self.assertIn("owner=$(stat -c '%u:%g' /bundle)", script) - self.assertIn('chown -R "$owner" /bundle/upstream', script) - self.assertLess( - script.index("trap restore_owner EXIT"), - script.index("mkdir -p /bundle/upstream"), - ) - - -class BuildTests(unittest.TestCase): - def test_build_config_owns_standard_runtime_paths(self): - config = build_config.BuildConfig() - alpine = config.initramfs_config("alpine") - ubuntu_config = config.initramfs_config("ubuntu") - distro = config.distro_layer_config() - - self.assertIsInstance(config.docker, build_config.DockerBuildConfig) - self.assertIsInstance(config.kernel, build_config.KernelBuildConfig) - self.assertIsInstance(config.openvmm, build_config.OpenVmmBuildConfig) - self.assertIsInstance(alpine, build_config.InitramfsBuildConfig) - self.assertIsInstance(ubuntu_config, build_config.InitramfsBuildConfig) - self.assertIsInstance(distro, build_config.DistroLayerBuildConfig) - self.assertIsNone(config.openvmm.backend) - self.assertEqual(config.selected_guests(), ("alpine",)) - self.assertEqual(config.kernel.work, BuildConstants.BUILD_DIR / "linux") - self.assertEqual(config.kernel.output, BuildConstants.BUILD_DIR / "vmlinux") - self.assertEqual( - alpine.work, - BuildConstants.BUILD_DIR / "initramfs-alpine-work", - ) - self.assertEqual( - alpine.output, - BuildConstants.BUILD_DIR / "initramfs.cpio.gz", - ) - self.assertEqual( - ubuntu_config.output, - BuildConstants.BUILD_DIR / "initramfs-ubuntu.cpio.gz", - ) - self.assertEqual( - distro.output, - BuildConstants.BUILD_DIR / "ubuntu-distro.erofs", - ) - self.assertEqual(config.docker.artifact_destination, BuildConstants.BUILD_DIR) - self.assertEqual( - config.docker.linux_source_destination, - BuildConstants.SOURCE_DIR / "linux", - ) - - def test_detects_openvmm_build_platform_without_runtime_devices(self): - cases: tuple[ - tuple[ - str, - build_config.OpenVmmBackend | None, - build_config.OpenVmmPlatform, - str, - build_config.OpenVmmBuildMode, - ], - ..., - ] = ( - ( - "win32", - None, - "windows-msvc", - "x86_64-pc-windows-msvc", - "native", - ), - ( - "win32", - "whp", - "windows-msvc", - "x86_64-pc-windows-msvc", - "native", - ), - ( - "linux", - None, - "linux-gnu", - "x86_64-unknown-linux-gnu", - "native", - ), - ( - "linux", - "kvm", - "linux-gnu", - "x86_64-unknown-linux-gnu", - "native", - ), - ( - "linux", - "mshv", - "linux-musl", - "x86_64-unknown-linux-musl", - "musl", - ), - ) - for host, backend, expected_platform, expected_target, expected_mode in cases: - with self.subTest(host=host, backend=backend): - with ( - patch.object(build.sys, "platform", host), - patch.object(build.os, "access") as access, - ): - config = build_config.OpenVmmBuildConfig() - platform = build.detect_openvmm_platform(backend) - - self.assertEqual(platform, expected_platform) - self.assertEqual(config.openvmm_target(platform), expected_target) - self.assertEqual(config.openvmm_build_mode(platform), expected_mode) - access.assert_not_called() - - def test_rejects_unsupported_openvmm_build_platforms(self): - cases: tuple[tuple[str, build_config.OpenVmmBackend | None], ...] = ( - ("linux", "whp"), - ("win32", "kvm"), - ("win32", "mshv"), - ("darwin", None), - ("darwin", "kvm"), - ("darwin", "mshv"), - ("darwin", "whp"), - ) - for host, backend in cases: - with self.subTest(host=host, backend=backend): - with ( - patch.object(build.sys, "platform", host), - patch.object(build.os, "access") as access, - self.assertRaisesRegex( - common.ScriptError, - "unsupported", - ), - ): - build.detect_openvmm_platform(backend) - access.assert_not_called() - - def test_linux_openvmm_builds_do_not_require_hypervisor_devices(self): - for devices_usable in (False, True): - for combined in (False, True): - with ( - self.subTest(devices_usable=devices_usable, combined=combined), - tempfile.TemporaryDirectory() as temporary, - ): - openvmm_dir = Path(temporary) / "openvmm" - output = openvmm_dir / "target" / "release" / "openvmm" - output.parent.mkdir(parents=True) - output.write_bytes(b"openvmm") - (openvmm_dir / "Cargo.toml").touch() - config = build_config.OpenVmmBuildConfig( - skip_restore=True, - directory=openvmm_dir, - output=output, - ) - - with ( - patch.object(build.sys, "platform", "linux"), - patch.object( - build.os, "access", return_value=devices_usable - ) as access, - patch.object(build, "run_checked") as run_checked, - patch.object(build, "record_openvmm_provenance"), - patch.object(build, "build_guest") as guest, - ): - if combined: - aggregate = build_config.BuildConfig(openvmm=config) - build.build_all(aggregate) - guest.assert_called_once_with(aggregate) - else: - build.build_openvmm(config) - guest.assert_not_called() - - access.assert_not_called() - run_checked.assert_called_once_with( - [ - "cargo", - "build", - "--release", - "-p", - "openvmm", - "--bin", - "openvmm", - ], - cwd=openvmm_dir, - ) - - def test_openvmm_target_output_matches_build_mode(self): - config = build_config.OpenVmmBuildConfig(directory=Path("checkout")) - cases: tuple[tuple[build_config.OpenVmmPlatform, Path], ...] = ( - ("linux-gnu", Path("target") / "release" / "openvmm"), - ( - "linux-musl", - Path("target") / "x86_64-unknown-linux-musl" / "release" / "openvmm", - ), - ("windows-msvc", Path("target") / "release" / "openvmm.exe"), - ) - for platform, relative_output in cases: - with self.subTest(platform=platform): - self.assertEqual( - config.openvmm_target_output(platform), - config.directory / relative_output, - ) - - def test_native_openvmm_build_normalizes_output_before_recording_provenance(self): - platforms: tuple[tuple[build_config.OpenVmmPlatform, str], ...] = ( - ("linux-gnu", "openvmm"), - ("windows-msvc", "openvmm.exe"), - ) - for platform, executable in platforms: - for existing_output in (False, True): - with ( - self.subTest(platform=platform, existing_output=existing_output), - tempfile.TemporaryDirectory() as temporary, - ): - root = Path(temporary) - openvmm_dir = root / "openvmm" - native_output = openvmm_dir / "target" / "release" / executable - native_output.parent.mkdir(parents=True) - binary = b"newly-built-openvmm" - native_output.write_bytes(binary) - (openvmm_dir / "Cargo.toml").touch() - output = root / "artifacts" / executable - if existing_output: - output.parent.mkdir(parents=True) - output.write_bytes(b"stale-openvmm") - config = build_config.OpenVmmBuildConfig( - skip_restore=True, - build_directory=root / "build", - directory=openvmm_dir, - output=output, - ) - revision = b"0bc357bbcf3a654b63dfb51f1103c5751bf3d31f\n" - results = [ - common.CommandResult(("git",), 0, revision, b""), - common.CommandResult(("git",), 0, revision, b""), - common.CommandResult(("git",), 0, b"", b""), - ] - - with ( - patch.object(build, "run_checked"), - patch.object(common, "run_capture", side_effect=results), - ): - build.build_openvmm(config, platform=platform) - - self.assertEqual(output.read_bytes(), binary) - provenance = json.loads( - ( - config.build_directory - / OpenVMMBuildConstants.PROVENANCE_NAME - ).read_text(encoding="utf-8") - ) - self.assertEqual( - provenance["executable_sha256"], - hashlib.sha256(binary).hexdigest(), - ) - - def test_native_openvmm_build_restores_and_records_provenance(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - openvmm_dir = root / "openvmm" - output = openvmm_dir / "target" / "release" / "openvmm" - output.parent.mkdir(parents=True) - output.write_bytes(b"openvmm") - (openvmm_dir / "Cargo.toml").touch() - config = build_config.OpenVmmBuildConfig( - build_directory=root / "build", - directory=openvmm_dir, - output=output, - ) - - with ( - patch.object(build, "run_checked") as run_checked, - patch.object(build, "record_openvmm_provenance") as provenance, - ): - build.build_openvmm(config, platform="linux-gnu") - - self.assertEqual( - run_checked.call_args_list, - [ - call( - [ - "cargo", - "xflowey", - "restore-packages", - "--no-compat-igvm", - ], - cwd=openvmm_dir, - ), - call( - [ - "cargo", - "build", - "--release", - "-p", - "openvmm", - "--bin", - "openvmm", - ], - cwd=openvmm_dir, - ), - ], - ) - provenance.assert_called_once_with(config) - - def test_openvmm_build_honors_skip_restore(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - openvmm_dir = root / "openvmm" - output = openvmm_dir / "target" / "release" / "openvmm" - output.parent.mkdir(parents=True) - output.write_bytes(b"openvmm") - (openvmm_dir / "Cargo.toml").touch() - config = build_config.OpenVmmBuildConfig( - skip_restore=True, - build_directory=root / "build", - directory=openvmm_dir, - output=output, - ) - - with ( - patch.object(build, "run_checked") as run_checked, - patch.object(build, "record_openvmm_provenance"), - ): - build.build_openvmm(config, platform="linux-gnu") - - self.assertEqual(run_checked.call_count, 1) - self.assertEqual(run_checked.call_args.args[0][:2], ["cargo", "build"]) - - def test_musl_openvmm_build_sets_sysroot_and_normalizes_output(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - openvmm_dir = root / "openvmm" - (openvmm_dir / "Cargo.toml").parent.mkdir(parents=True) - (openvmm_dir / "Cargo.toml").touch() - sysroot = openvmm_dir / ".packages" / "extracted" / "x86_64-sysroot" - (sysroot / "lib").mkdir(parents=True) - (sysroot / "lib" / "libsymcrypt.a").touch() - output = openvmm_dir / "target" / "release" / "openvmm" - config = build_config.OpenVmmBuildConfig( - build_directory=root / "build", - directory=openvmm_dir, - output=output, - backend="mshv", - ) - target_output = config.openvmm_target_output("linux-musl") - target_output.parent.mkdir(parents=True) - target_output.write_bytes(b"musl-openvmm") - - with ( - patch.object(build.sys, "platform", "linux"), - patch.object(build.os, "access", return_value=False) as access, - patch.object(build, "run_checked") as run_checked, - patch.object(build, "record_openvmm_provenance") as provenance, - patch.object(Path, "chmod") as chmod, - ): - build.build_openvmm(config) - - access.assert_not_called() - self.assertEqual(run_checked.call_count, 3) - self.assertEqual( - run_checked.call_args_list[1], - call(["rustup", "target", "add", "x86_64-unknown-linux-musl"]), - ) - cargo_call = run_checked.call_args_list[2] - self.assertIn("x86_64-unknown-linux-musl", cargo_call.args[0]) - environment = cargo_call.kwargs["env"] - resolved_sysroot = ( - openvmm_dir.resolve() / ".packages" / "extracted" / "x86_64-sysroot" - ) - self.assertEqual( - environment["X86_64_UNKNOWN_LINUX_MUSL_OPENSSL_DIR"], - os.fspath(resolved_sysroot), - ) - self.assertEqual( - environment["X86_64_UNKNOWN_LINUX_MUSL_SYMCRYPT_LIB_PATH"], - os.fspath(resolved_sysroot / "lib"), - ) - self.assertEqual(output.read_bytes(), b"musl-openvmm") - chmod.assert_called_once() - provenance.assert_called_once_with(config) - - def test_guest_build_routes_one_config_to_native_or_docker_consumers(self): - with ( - patch.object(build, "build_kernel") as kernel, - patch.object(build, "build_initramfs") as initramfs, - patch.object(build, "build_docker_artifacts") as docker, - ): - native = build_config.BuildConfig(native_guest=True) - build.build_guest(native) - kernel.assert_called_once_with(native.kernel) - initramfs.assert_called_once_with(native.initramfs_config("alpine")) - docker.assert_not_called() - - with ( - patch.object(build, "build_kernel") as kernel, - patch.object(build, "build_initramfs") as initramfs, - patch.object(build, "build_docker_artifacts") as docker, - ): - portable = build_config.BuildConfig() - build.build_guest(portable) - docker.assert_called_once_with(portable.docker, "alpine") - kernel.assert_not_called() - initramfs.assert_not_called() - - with ( - patch.object(build, "build_kernel") as kernel, - patch.object(build, "build_initramfs") as initramfs, - patch.object(build, "build_distro_layer") as distro, - ): - all_guests = build_config.BuildConfig(guest="all", native_guest=True) - with self.assertRaisesRegex( - common.ScriptError, - "Azure Linux initramfs builds require Docker", - ): - build.build_guest(all_guests) - kernel.assert_not_called() - initramfs.assert_not_called() - distro.assert_not_called() - - def test_combined_build_passes_explicit_backend_to_openvmm_build(self): - config = build_config.BuildConfig( - openvmm=build_config.OpenVmmBuildConfig(backend="mshv") - ) - with ( - patch.object(build.sys, "platform", "linux"), - patch.object(build.os, "access", return_value=False) as access, - patch.object(build, "build_guest") as guest, - patch.object(build, "build_openvmm") as openvmm, - ): - build.build_all(config) - - access.assert_not_called() - guest.assert_called_once_with(config) - openvmm.assert_called_once_with(config.openvmm, platform="linux-musl") - - def test_combined_build_validates_platform_before_building_guest(self): - config = build_config.BuildConfig( - openvmm=build_config.OpenVmmBuildConfig(backend="whp") - ) - with ( - patch.object(build.sys, "platform", "linux"), - patch.object(build, "build_guest") as guest, - patch.object(build, "build_openvmm") as openvmm, - self.assertRaisesRegex( - common.ScriptError, "backend 'whp' is unsupported on linux" - ), - ): - build.build_all(config) - - guest.assert_not_called() - openvmm.assert_not_called() - - def test_release_source_collection_passes_docker_build_config(self): - config = build_config.DockerBuildConfig( - linux_source_destination=Path("custom-linux-source") - ) - with ( - patch.object( - release, - "_guest_release_inputs", - return_value=((), (), (), ()), - ) as guest_release_inputs, - patch.object(release, "collect_alpine_sources"), - patch.object(release, "collect_ubuntu_sources"), - patch.object(release, "build_docker_linux_source") as linux_source, - ): - release.collect_release_sources(config) - - guest_release_inputs.assert_called_once_with(include_azurelinux=False) - linux_source.assert_called_once_with(config) - - def test_records_openvmm_revision_cleanliness_and_executable_hash(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - openvmm_dir = root / "openvmm" - executable = openvmm_dir / "target" / "release" / "openvmm" - executable.parent.mkdir(parents=True) - executable.write_bytes(b"openvmm") - revision = b"0bc357bbcf3a654b63dfb51f1103c5751bf3d31f\n" - results = [ - common.CommandResult(("git",), 0, revision, b""), - common.CommandResult(("git",), 0, revision, b""), - common.CommandResult(("git",), 0, b" M src/main.rs\n", b""), - ] - - with ( - patch.object(BuildConstants, "REPO_ROOT", root), - patch.object(common, "run_capture", side_effect=results), - ): - build.record_openvmm_provenance( - build_config.OpenVmmBuildConfig( - build_directory=root / "build", - directory=openvmm_dir, - output=executable, - ) - ) - - provenance = json.loads( - (root / "build" / OpenVMMBuildConstants.PROVENANCE_NAME).read_text( - encoding="utf-8" - ) - ) - self.assertEqual(provenance["source_revision"], revision.decode().strip()) - self.assertFalse(provenance["source_clean"]) - self.assertEqual( - provenance["executable_sha256"], - hashlib.sha256(b"openvmm").hexdigest(), - ) - - def test_kernel_build_rejects_input_config_mutation_during_build(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - input_config = root / "kernel" / "config-microvm" - input_config.parent.mkdir(parents=True) - input_config.write_text( - "\n".join( - ( - *KernelBuildConstants.REQUIRED_DIRECT_BOOT_CONFIG, - *KernelBuildConstants.REQUIRED_VIRTIO_CONSOLE_CONFIG, - *KernelBuildConstants.REQUIRED_SHARED_STATUS_CONFIG, - *KernelBuildConstants.REQUIRED_SANDBOX_CONFIG, - *KernelBuildConstants.REQUIRED_NETWORK_CONFIG, - *KernelBuildConstants.REQUIRED_TIME_ABI_CONFIG, - *KernelBuildConstants.REQUIRED_HARDENING_CONFIG, - ) - ) - + "\n", - encoding="utf-8", - ) - patch_path = root / "kernel" / "patches" / "example.patch" - patch_path.parent.mkdir() - patch_path.write_text("patch", encoding="utf-8") - source = root / "source" - source.mkdir() - work = root / "work" - output = root / "output" / "vmlinux" - prior_provenance = output.with_name(KernelBuildConstants.PROVENANCE_NAME) - prior_provenance.parent.mkdir() - prior_provenance.write_text("stale", encoding="utf-8") - - with patch.object(BuildConstants, "REPO_ROOT", root): - source_fingerprint = build._kernel_source_fingerprint() - - def run_build(command: object, **_kwargs: object) -> None: - if isinstance(command, list) and command[-1] == "vmlinux": - (work / "vmlinux").write_bytes(b"kernel") - input_config.write_text("CONFIG_CHANGED=y\n", encoding="utf-8") - - with ( - patch.object(BuildConstants, "REPO_ROOT", root), - patch.object(build, "_require_linux"), - patch.object(build, "require_tool", return_value="tool"), - patch.object( - build, - "prepare_kernel_source", - return_value=(source, source_fingerprint), - ), - patch.object(build, "run_checked", side_effect=run_build), - self.assertRaisesRegex( - common.ScriptError, - "inputs changed during the build", - ), - ): - build.build_kernel( - build_config.KernelBuildConfig( - work=work, - output=output, - ) - ) - - self.assertFalse(output.exists()) - self.assertFalse(output.with_name("vmlinux.config").exists()) - self.assertFalse(prior_provenance.exists()) - - def test_materialize_kernel_provenance_inputs_bypasses_mutable_index(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - git_environment = { - key: value - for key, value in os.environ.items() - if not key.startswith("GIT_CONFIG_") - } - config = root / "kernel" / "config-microvm" - patch_path = root / "kernel" / "patches" / "example.patch" - stale_patch = root / "kernel" / "patches" / "stale.patch" - config.parent.mkdir(parents=True) - patch_path.parent.mkdir() - config.write_bytes(b"CONFIG_EXAMPLE=y\n") - patch_path.write_bytes(b"patch\n") - (root / ".gitattributes").write_text( - "\n".join( - ( - "kernel/config-microvm text eol=lf", - "kernel/patches/*.patch text eol=lf", - ) - ) - + "\n", - encoding="ascii", - ) - subprocess.run( - ["git", "init", "-q"], - cwd=root, - env=git_environment, - check=True, - ) - subprocess.run( - ["git", "config", "user.name", "NVX Tests"], - cwd=root, - env=git_environment, - check=True, - ) - subprocess.run( - ["git", "config", "user.email", "nvx-tests@example.com"], - cwd=root, - env=git_environment, - check=True, - ) - subprocess.run( - [ - "git", - "add", - ".gitattributes", - "kernel/config-microvm", - "kernel/patches/example.patch", - ], - cwd=root, - env=git_environment, - check=True, - ) - subprocess.run( - ["git", "commit", "-qm", "kernel provenance fixture"], - cwd=root, - env=git_environment, - check=True, - ) - git_dir = Path( - subprocess.run( - ["git", "rev-parse", "--git-dir"], - cwd=root, - env=git_environment, - check=True, - capture_output=True, - text=True, - ).stdout.strip() - ) - (root / git_dir / "info" / "attributes").write_text( - "\n".join( - ( - "kernel/config-microvm -text", - "kernel/patches/*.patch -text", - ) - ) - + "\n", - encoding="ascii", - ) - config.write_bytes(b"CONFIG_EXAMPLE=y\r\n") - patch_path.write_bytes(b"patch\r\n") - stale_patch.write_bytes(b"stale\r\n") - subprocess.run( - [ - "git", - "add", - "kernel/config-microvm", - "kernel/patches/example.patch", - "kernel/patches/stale.patch", - ], - cwd=root, - env=git_environment, - check=True, - ) - subprocess.run( - [ - "git", - "checkout-index", - "--force", - "--", - "kernel/config-microvm", - "kernel/patches/example.patch", - "kernel/patches/stale.patch", - ], - cwd=root, - env=git_environment, - check=True, - ) - self.assertEqual(config.read_bytes(), b"CONFIG_EXAMPLE=y\r\n") - self.assertEqual(patch_path.read_bytes(), b"patch\r\n") - self.assertEqual(stale_patch.read_bytes(), b"stale\r\n") - - with ( - patch.dict(os.environ, git_environment, clear=True), - patch.object(BuildConstants, "REPO_ROOT", root), - ): - build.materialize_kernel_provenance_inputs() - - self.assertEqual(config.read_bytes(), b"CONFIG_EXAMPLE=y\n") - self.assertEqual(patch_path.read_bytes(), b"patch\n") - self.assertFalse(stale_patch.exists()) - - def test_manifest_tracks_every_kernel_patch(self): - manifest = json.loads( - (BuildConstants.REPO_ROOT / "SOURCE-MANIFEST.json").read_text( - encoding="utf-8" - ) - ) - self.assertEqual( - manifest["linux"]["patches"], - [ - path.relative_to(BuildConstants.REPO_ROOT).as_posix() - for path in build._kernel_patch_files() - ], - ) - self.assertNotIn("guest_agent", manifest) - self.assertEqual( - manifest["openvmm"], - { - "microvm_abi_version": 2, - "control_session_protocol_version": 1, - "control_contract_revision": "nvx-microvm-v2-control-v2", - }, - ) - - def test_guest_descriptors_preserve_alpine_and_select_guest_outputs(self): - alpine = guests.guest_descriptor("alpine") - ubuntu_guest = guests.guest_descriptor("ubuntu") - azurelinux_guest = guests.guest_descriptor("azurelinux") - - self.assertEqual(alpine.initramfs_name, "initramfs.cpio.gz") - self.assertEqual(alpine.default_memory_mib, 128) - self.assertTrue(alpine.sandbox_control) - self.assertEqual( - ubuntu_guest.initramfs_name, - "initramfs-ubuntu.cpio.gz", - ) - self.assertEqual(ubuntu_guest.default_memory_mib, 512) - self.assertFalse(ubuntu_guest.sandbox_control) - self.assertEqual( - azurelinux_guest.initramfs_name, - "initramfs-azurelinux.cpio.gz", - ) - self.assertEqual(azurelinux_guest.default_memory_mib, 512) - self.assertFalse(azurelinux_guest.sandbox_control) - - def test_azurelinux_manifest_and_package_lock_match_build_pins(self): - manifest = json.loads( - (BuildConstants.REPO_ROOT / "SOURCE-MANIFEST.json").read_text( - encoding="utf-8" - ) - )["azurelinux"] - self.assertEqual(manifest["image"], AzureLinuxBuildConstants.IMAGE) - self.assertEqual(manifest["package_lock"], "azurelinux/packages.lock.json") - self.assertEqual( - manifest["package_lock_sha256"], - azurelinux.package_lock_sha256(), - ) - packages = azurelinux.load_package_lock() - self.assertEqual( - {"busybox", "util-linux"} - {package["name"] for package in packages}, - set(), - ) - for package in packages: - self.assertTrue( - package["url"].startswith( - f"{AzureLinuxBuildConstants.REPOSITORY_URL}/Packages/" - ) - ) - attributes = (BuildConstants.REPO_ROOT / ".gitattributes").read_text( - encoding="utf-8" - ) - self.assertIn("azurelinux/packages.lock.json text eol=lf", attributes) - - def test_azurelinux_package_lock_rejects_unpinned_records(self): - document = json.loads( - azurelinux.package_lock_path().read_text(encoding="utf-8") - ) - first = document["packages"][0] - cases: tuple[tuple[str, dict[str, object], str], ...] = ( - ("image", {"image": "mcr.microsoft.com/azurelinux/base/core:3.0"}, "image"), - ( - "url", - { - "packages": [ - {**first, "url": first["url"].replace("/prod/", "/preview/")}, - *document["packages"][1:], - ] - }, - "must be fetched from", - ), - ( - "sha256", - { - "packages": [ - {**first, "sha256": "0" * 63}, - *document["packages"][1:], - ] - }, - "invalid SHA-256", - ), - ( - "order", - {"packages": list(reversed(document["packages"]))}, - "sorted by name", - ), - ) - for name, update, message in cases: - with ( - self.subTest(case=name), - tempfile.TemporaryDirectory() as temporary, - ): - path = Path(temporary) / "packages.lock.json" - path.write_text(json.dumps({**document, **update}), encoding="utf-8") - with self.assertRaisesRegex(common.ScriptError, message): - azurelinux.load_package_lock(path) - - def test_azurelinux_download_packages_verifies_each_locked_rpm(self): - destination = Path("rpms") - with patch.object(azurelinux, "download_verified") as download_verified: - downloaded = azurelinux.download_packages(destination) - - packages = azurelinux.load_package_lock() - self.assertEqual( - download_verified.call_args_list, - [ - call( - package["url"], - destination / package["url"].rsplit("/", maxsplit=1)[-1], - package["sha256"], - ) - for package in packages - ], - ) - self.assertEqual(len(downloaded), len(packages)) - - def test_ubuntu_manifest_and_package_lock_match_build_pins(self): - manifest = json.loads( - (BuildConstants.REPO_ROOT / "SOURCE-MANIFEST.json").read_text( - encoding="utf-8" - ) - ) - ubuntu_manifest = manifest["ubuntu"] - self.assertEqual( - ubuntu_manifest["version"], - UbuntuBuildConstants.VERSION, - ) - self.assertEqual( - ubuntu_manifest["base_url"], - UbuntuBuildConstants.BASE_URL, - ) - self.assertEqual( - ubuntu_manifest["base_sha256"], - UbuntuBuildConstants.BASE_SHA256, - ) - self.assertEqual( - ubuntu_manifest["archive_keyring_url"], - UbuntuBuildConstants.ARCHIVE_KEYRING_URL, - ) - self.assertEqual( - ubuntu_manifest["archive_keyring_sha256"], - UbuntuBuildConstants.ARCHIVE_KEYRING_SHA256, - ) - self.assertEqual( - ubuntu_manifest["package_lock_sha256"], - ubuntu.package_lock_sha256(), - ) - self.assertEqual( - ubuntu_manifest["package_lock_sha256"], - "fcdd30223b96fc26e24fcf4b6763a4a2504e9cd25882e68e89fbf6bf45277ef0", - ) - packages = ubuntu.load_package_lock() - self.assertEqual( - [package["name"] for package in packages], - [ - "busybox-static", - "iputils-ping", - "libcap2", - "libidn2-0", - "libunistring5", - "net-tools", - "netcat-openbsd", - ], - ) - - def test_ubuntu_package_lock_read_errors_are_actionable(self): - with tempfile.TemporaryDirectory() as temporary: - path = Path(temporary) / "packages.json" - path.write_text("{", encoding="utf-8") - - with self.assertRaisesRegex( - common.ScriptError, "failed to read Ubuntu package lock" - ): - ubuntu.load_package_lock(path) - - def test_ubuntu_safe_extractor_rejects_archive_symlink_escape(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - archive_path = root / "unsafe.tar.gz" - with tarfile.open(archive_path, "w:gz") as archive_file: - symlink = tarfile.TarInfo("etc") - symlink.type = tarfile.SYMTYPE - symlink.linkname = "../../outside" - archive_file.addfile(symlink) - payload = b"nameserver 192.0.2.1\n" - resolver = tarfile.TarInfo("etc/resolv.conf") - resolver.size = len(payload) - archive_file.addfile(resolver, io.BytesIO(payload)) - - with self.assertRaisesRegex(common.ScriptError, "escapes"): - ubuntu.safe_extract_tar( - archive_path, - root / "extracted", - label="test archive", - ) - self.assertFalse((root / "outside").exists()) - - def test_ubuntu_safe_extractor_roots_absolute_symlinks(self): - self.assertEqual( - ubuntu._virtual_symlink_target( - PurePosixPath("var/run"), - "/run", - "test archive", - ), - "../run", - ) - self.assertEqual( - ubuntu._virtual_symlink_target( - PurePosixPath("etc/alternatives/awk"), - "/usr/bin/mawk", - "test archive", - ), - "../../usr/bin/mawk", - ) - - def test_ubuntu_safe_extractor_rejects_absolute_and_parent_paths(self): - for member_name in ("/absolute", "../parent"): - with self.subTest(member_name=member_name): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - archive_path = root / "unsafe.tar" - with tarfile.open(archive_path, "w") as archive_file: - payload = b"x" - member = tarfile.TarInfo(member_name) - member.size = len(payload) - archive_file.addfile(member, io.BytesIO(payload)) - with self.assertRaisesRegex( - common.ScriptError, - "absolute path|path traversal", - ): - ubuntu.safe_extract_tar( - archive_path, - root / "extracted", - label="test archive", - ) - - def test_ubuntu_rootfs_digest_and_erofs_uuid_are_deterministic(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - first = root / "first" - second = root / "second" - for directory in (first, second): - (directory / "etc").mkdir(parents=True) - (directory / "etc" / "os-release").write_text( - "ID=ubuntu\n", - encoding="ascii", - ) - os.utime(second / "etc" / "os-release", (1234, 1234)) - - first_digest = ubuntu.rootfs_sha256(first) - second_digest = ubuntu.rootfs_sha256(second) - self.assertEqual(first_digest, second_digest) - self.assertEqual( - ubuntu.erofs_uuid(first_digest), - ubuntu.erofs_uuid(second_digest), - ) - - def test_ubuntu_prepare_root_normalizes_root_directory_mode(self): - with tempfile.TemporaryDirectory() as temporary: - temporary_root = Path(temporary) - work = temporary_root / "work" - expected_root = work / "root" - - def extract( - archive: Path, - destination: Path, - *, - label: str, - ) -> None: - self.assertEqual( - archive.name, - f"ubuntu-base-{UbuntuBuildConstants.VERSION}-base-amd64.tar.gz", - ) - self.assertEqual(label, "Ubuntu Base archive") - status = destination / "var" / "lib" / "dpkg" / "status" - status.parent.mkdir(parents=True) - status.write_text("", encoding="utf-8") - - with ( - patch.object( - ubuntu, - "cache_root", - return_value=temporary_root / "cache", - ), - patch.object(ubuntu, "download_verified"), - patch.object(ubuntu, "safe_extract_tar", side_effect=extract), - patch.object(ubuntu, "_validate_ubuntu_identity"), - patch.object(ubuntu, "load_package_lock", return_value=()), - patch.object(ubuntu, "_validate_package_closure"), - patch.object(ubuntu, "_customize_root"), - patch.object(ubuntu.Path, "chmod", autospec=True) as chmod, - ): - self.assertEqual(ubuntu.prepare_root(work), expected_root) - - chmod.assert_called_once_with(expected_root, 0o755) - - def test_ubuntu_customization_installs_busybox_wget(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - for relative in ("etc", "usr/bin", "usr/sbin"): - (root / relative).mkdir(parents=True) - with ( - patch.object(ubuntu, "_clear_directory"), - patch.object(ubuntu, "_validate_accounts"), - patch.object(ubuntu, "_validate_usr_merge"), - patch.object(ubuntu, "_ensure_symlink") as ensure_symlink, - ): - ubuntu._customize_root(root) - - ensure_symlink.assert_any_call(root, "usr/bin/wget", "busybox") - - def test_distro_layer_refuses_existing_output_without_replace(self): - with tempfile.TemporaryDirectory() as temporary: - output = Path(temporary) / "ubuntu.erofs" - output.touch() - with ( - patch.object(build.sys, "platform", "linux"), - self.assertRaisesRegex(common.ScriptError, "refusing to replace"), - ): - build.build_distro_layer( - build_config.DistroLayerBuildConfig(output=output) - ) - - def test_ubuntu_initramfs_manifest_binds_artifact_digest(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - output = root / "initramfs-ubuntu.cpio.gz" - output.write_bytes(b"ubuntu initramfs") - with patch.object( - ubuntu, - "package_manifest", - return_value={"format": 1, "guest": "ubuntu"}, - ): - build._write_ubuntu_manifest(root, output, {}, "a" * 64) - - manifest = json.loads( - output.with_name(f"{output.name}.packages.json").read_text( - encoding="utf-8" - ) - ) - self.assertEqual(manifest["artifact"], output.name) - self.assertEqual( - manifest["artifact_sha256"], - common.sha256_file(output), - ) - self.assertEqual(manifest["input_sha256"], "a" * 64) - - def test_ubuntu_source_requirements_deduplicate_binary_manifests(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - manifests: list[Path] = [] - for name in ("initramfs.json", "layer.json"): - path = root / name - path.write_text( - json.dumps( - { - "guest": "ubuntu", - "release": UbuntuBuildConstants.VERSION, - "architecture": UbuntuBuildConstants.ARCHITECTURE, - "packages": [ - { - "source_name": "glibc", - "source_version": "2.43-2ubuntu2.3", - } - ], - } - ), - encoding="utf-8", - ) - manifests.append(path) - - requirements = collect_ubuntu_sources._source_requirements(manifests) - - self.assertEqual( - requirements, - ( - { - "source_name": "glibc", - "source_version": "2.43-2ubuntu2.3", - }, - ), - ) - - def test_ubuntu_dsc_validation_matches_source_index(self): - payload = b"source" - digest = hashlib.sha256(payload).hexdigest() - with tempfile.TemporaryDirectory() as temporary: - dsc = Path(temporary) / "example_1.0.dsc" - dsc.write_text( - ( - "Format: 3.0 (quilt)\n" - "Source: example\n" - "Version: 1.0\n" - "Checksums-Sha256:\n" - f" {digest} {len(payload)} example_1.0.orig.tar.xz\n" - ), - encoding="utf-8", - ) - record: collect_ubuntu_sources.SourceRecord = { - "source_name": "example", - "source_version": "1.0", - "directory": "pool/main/e/example", - "index_url": "https://archive.invalid/Sources.xz", - "index_sha256": "0" * 64, - "index_release_url": "https://archive.invalid/InRelease", - "index_release_sha256": "1" * 64, - "files": [ - { - "name": "example_1.0.dsc", - "size": dsc.stat().st_size, - "sha256": common.sha256_file(dsc), - "url": "https://archive.invalid/example_1.0.dsc", - }, - { - "name": "example_1.0.orig.tar.xz", - "size": len(payload), - "sha256": digest, - "url": "https://archive.invalid/example_1.0.orig.tar.xz", - }, - ], - } - - collect_ubuntu_sources._validate_dsc(dsc, record) - record["files"][1]["sha256"] = "f" * 64 - with self.assertRaisesRegex( - common.ScriptError, - "do not match", - ): - collect_ubuntu_sources._validate_dsc(dsc, record) - - def test_ubuntu_source_index_is_authenticated_by_signed_release(self): - dsc_payload = b"dsc" - source_payload = b"source" - sources = ( - "Package: example\n" - "Version: 1.0\n" - "Directory: pool/main/e/example\n" - "Checksums-Sha256:\n" - f" {hashlib.sha256(dsc_payload).hexdigest()} " - f"{len(dsc_payload)} example_1.0.dsc\n" - f" {hashlib.sha256(source_payload).hexdigest()} " - f"{len(source_payload)} example_1.0.orig.tar.xz\n" - ).encode() - compressed = lzma.compress(sources) - index_sha256 = hashlib.sha256(compressed).hexdigest() - inrelease = ( - "-----BEGIN PGP SIGNED MESSAGE-----\n" - "Hash: SHA256\n" - "\n" - "Origin: Ubuntu\n" - f"Codename: {UbuntuBuildConstants.CODENAME}\n" - f"Suite: {UbuntuBuildConstants.CODENAME}\n" - "SHA256:\n" - f" {index_sha256} {len(compressed)} main/source/Sources.xz\n" - "-----BEGIN PGP SIGNATURE-----\n" - "test\n" - "-----END PGP SIGNATURE-----\n" - ) - - def download_release(_url: str, destination: Path) -> None: - destination.parent.mkdir(parents=True, exist_ok=True) - destination.write_text(inrelease, encoding="utf-8") - - def download_index( - _url: str, - destination: Path, - expected_sha256: str, - ) -> None: - self.assertEqual(expected_sha256, index_sha256) - destination.write_bytes(compressed) - - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - keyring = root / "ubuntu-archive-keyring.gpg" - keyring.write_bytes(b"keyring") - with ( - patch.object( - collect_ubuntu_sources, - "download", - side_effect=download_release, - ), - patch.object( - collect_ubuntu_sources, - "download_verified", - side_effect=download_index, - ), - patch.object( - collect_ubuntu_sources, - "require_tool", - return_value="gpgv", - ), - patch.object(collect_ubuntu_sources, "run_checked") as run_checked, - ): - records, metadata = ( - collect_ubuntu_sources._load_authenticated_source_index( - root / "cache", - keyring, - "https://archive.invalid/ubuntu", - UbuntuBuildConstants.CODENAME, - "main", - ) - ) - - run_checked.assert_called_once() - self.assertEqual(run_checked.call_args.args[0][0], "gpgv") - self.assertIn("--homedir", run_checked.call_args.args[0]) - record = records[("example", "1.0")] - self.assertEqual( - record["index_release_url"], - "https://archive.invalid/ubuntu/dists/resolute/InRelease", - ) - self.assertEqual( - [item["role"] for item in metadata], - ["ubuntu-inrelease", "ubuntu-source-index"], - ) - self.assertEqual( - metadata[1]["authenticated_by"], - record["index_release_url"], - ) - - def test_launchpad_source_record_retains_verifiable_raw_metadata(self): - source_payload = b"source archive" - source_sha256 = hashlib.sha256(source_payload).hexdigest() - self_link = f"{UbuntuBuildConstants.LAUNCHPAD_ARCHIVE_API}/+sourcepub/123" - query = { - "entries": [ - { - "source_package_name": "example", - "source_package_version": "1.0", - "status": "Superseded", - "date_published": "2026-01-01T00:00:00Z", - "date_superseded": "2026-01-03T00:00:00Z", - "pocket": "Updates", - "component_name": "main", - "self_link": self_link, - } - ] - } - source_urls = [ - "https://launchpad.net/ubuntu/+archive/primary/+sourcefiles/" - "example/1.0/example_1.0.dsc", - "https://launchpad.net/ubuntu/+archive/primary/+sourcefiles/" - "example/1.0/example_1.0.orig.tar.xz", - ] - - def download_json(url: str, destination: Path) -> object: - document: object = source_urls if "sourceFileUrls" in url else query - destination.parent.mkdir(parents=True, exist_ok=True) - destination.write_text( - json.dumps(document, separators=(",", ":")) + "\n", - encoding="utf-8", - ) - return document - - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - snapshot_release = root / "cache" / "snapshot.InRelease" - snapshot_release.parent.mkdir(parents=True) - snapshot_release.write_text("signed release", encoding="utf-8") - snapshot_record: collect_ubuntu_sources.SourceRecord = { - "source_name": "example", - "source_version": "1.0", - "directory": "pool/main/e/example", - "index_url": ( - "https://snapshot.ubuntu.com/ubuntu/20260102T000000Z/" - "dists/resolute-updates/main/source/Sources.xz" - ), - "index_sha256": "2" * 64, - "index_release_url": ( - "https://snapshot.ubuntu.com/ubuntu/20260102T000000Z/" - "dists/resolute-updates/InRelease" - ), - "index_release_sha256": common.sha256_file(snapshot_release), - "files": [ - { - "name": "example_1.0.dsc", - "size": 3, - "sha256": "4" * 64, - "url": ( - "https://snapshot.ubuntu.com/ubuntu/" - "20260102T000000Z/pool/main/e/example/" - "example_1.0.dsc" - ), - }, - { - "name": "example_1.0.orig.tar.xz", - "size": len(source_payload), - "sha256": source_sha256, - "url": ( - "https://snapshot.ubuntu.com/ubuntu/" - "20260102T000000Z/pool/main/e/example/" - "example_1.0.orig.tar.xz" - ), - }, - ], - } - snapshot_metadata: list[collect_ubuntu_sources.SourceMetadata] = [ - { - "role": "ubuntu-inrelease", - "url": snapshot_record["index_release_url"], - "sha256": snapshot_record["index_release_sha256"], - "cache_path": snapshot_release, - "output_name": "snapshot.InRelease", - "authenticated_by": (UbuntuBuildConstants.ARCHIVE_KEYRING_URL), - }, - { - "role": "ubuntu-source-index", - "url": snapshot_record["index_url"], - "sha256": snapshot_record["index_sha256"], - "cache_path": None, - "output_name": None, - "authenticated_by": snapshot_record["index_release_url"], - }, - ] - with ( - patch.object( - collect_ubuntu_sources, - "_download_json", - side_effect=download_json, - ), - patch.object( - collect_ubuntu_sources, - "_load_authenticated_source_index", - return_value=( - {("example", "1.0"): snapshot_record}, - snapshot_metadata, - ), - ), - ): - record, metadata = collect_ubuntu_sources._launchpad_source_record( - root / "cache", - root / "ubuntu-archive-keyring.gpg", - "example", - "1.0", - ) - - self.assertEqual(record, snapshot_record) - self.assertEqual( - [item["role"] for item in metadata], - [ - "ubuntu-inrelease", - "ubuntu-source-index", - "launchpad-publishing-history", - "launchpad-source-file-urls", - ], - ) - for item in metadata: - if item["cache_path"] is None: - continue - self.assertEqual( - item["sha256"], - common.sha256_file(item["cache_path"]), - ) - - output = root / "output" - output.mkdir() - manifest_metadata = collect_ubuntu_sources._materialize_source_metadata( - output, - metadata, - ) - for item, manifest_item in zip(metadata, manifest_metadata, strict=True): - if item["cache_path"] is None: - self.assertNotIn("path", manifest_item) - continue - retained = output / manifest_item["path"] - self.assertEqual( - retained.read_bytes(), - item["cache_path"].read_bytes(), - ) - self.assertEqual( - common.sha256_file(retained), - manifest_item["sha256"], - ) - - def test_ubuntu_source_collection_rejects_unexpected_output_entries(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - output = root / "output" - output.mkdir() - stale = output / "unrelated.txt" - stale.write_text("do not archive", encoding="utf-8") - - with self.assertRaisesRegex( - common.ScriptError, - "unexpected entries: unrelated.txt", - ): - collect_ubuntu_sources.collect_ubuntu_sources( - [], - output, - root / "cache", - ) - - self.assertEqual(stale.read_text(encoding="utf-8"), "do not archive") - - def test_ubuntu_source_collection_validates_output_before_resolving(self): - output = MagicMock(spec=Path) - with patch.object( - collect_ubuntu_sources, - "_validate_source_output", - ) as validate: - - def resolve() -> Path: - validate.assert_called_once_with(output) - raise RuntimeError("stop after ordering assertion") - - output.resolve.side_effect = resolve - with self.assertRaisesRegex(RuntimeError, "ordering assertion"): - collect_ubuntu_sources.collect_ubuntu_sources( - [], - output, - Path("cache"), - ) - - def test_ci_kernel_cache_key_includes_patches(self): - action = ( - BuildConstants.REPO_ROOT - / ".github" - / "actions" - / "build-guest-artifacts" - / "action.yml" - ).read_text(encoding="utf-8") - for cache_input in ( - "SOURCE-MANIFEST.json", - "kernel/config-microvm", - "kernel/patches/**", - "scripts/nvx_tools/build.py", - "scripts/nvx_tools/common.py", - ): - self.assertIn(cache_input, action) - self.assertIn("linux-kernel-v1-", action) - self.assertEqual(action.count("build/vmlinux.provenance.json"), 2) - self.assertEqual(action.count("build/initramfs.provenance.json"), 2) - - def test_ci_guest_cache_keys_include_build_configuration(self): - action = ( - BuildConstants.REPO_ROOT - / ".github" - / "actions" - / "build-guest-artifacts" - / "action.yml" - ).read_text(encoding="utf-8") - cache_names = ( - "KERNEL_INPUT_HASH", - "DEBUG_KERNEL_INPUT_HASH", - "ALPINE_INPUT_HASH", - "UBUNTU_INPUT_HASH", - "AZURELINUX_INPUT_HASH", - ) - # Every artifact, both kernels included, builds inside the image that - # docker/Dockerfile pins, so a toolchain change must miss every cache. - self.assertEqual( - action.count("--file docker/Dockerfile"), - action.count("docker build"), - ) - for cache_name in cache_names: - with self.subTest(cache=cache_name): - cache_input = next( - line - for line in action.splitlines() - if line.strip().startswith(f"{cache_name}:") - ) - self.assertIn("'docker/Dockerfile'", cache_input) - self.assertIn("'scripts/nvx_tools/build_config.py'", cache_input) - self.assertIn("'scripts/nvx_tools/build_constants.py'", cache_input) - - def test_ci_azurelinux_cache_key_covers_build_input_digest(self): - action = ( - BuildConstants.REPO_ROOT - / ".github" - / "actions" - / "build-guest-artifacts" - / "action.yml" - ).read_text(encoding="utf-8") - cache_input = next( - line - for line in action.splitlines() - if line.strip().startswith("AZURELINUX_INPUT_HASH:") - ) - patterns = cache_input.split("'")[1::2] - self.assertIn("scripts/nvx_tools/azurelinux.py", patterns) - for path in azurelinux.input_files(): - relative = path.relative_to(BuildConstants.REPO_ROOT).as_posix() - with self.subTest(path=relative): - self.assertTrue( - any( - relative == pattern - or ( - pattern.endswith("/**") - and relative.startswith(pattern[:-2]) - ) - for pattern in patterns - ) - ) - - def test_kernel_input_config_uses_canonical_lf_line_endings(self): - attributes = (BuildConstants.REPO_ROOT / ".gitattributes").read_text( - encoding="utf-8" - ) - for path in ( - KernelBuildConstants.INPUT_CONFIG, - KernelBuildConstants.DEBUG_CONFIG_FRAGMENT, - ): - with self.subTest(path=path): - self.assertIn( - f"{path.as_posix()} text eol=lf", - attributes.splitlines(), - ) - - def test_ci_debug_kernel_is_opt_in_and_keyed_by_its_fragment(self): - action = ( - BuildConstants.REPO_ROOT - / ".github" - / "actions" - / "build-guest-artifacts" - / "action.yml" - ).read_text(encoding="utf-8") - cache_input = next( - line - for line in action.splitlines() - if line.strip().startswith("DEBUG_KERNEL_INPUT_HASH:") - ) - for path in ( - "kernel/config-microvm", - "kernel/config-microvm-debug", - "kernel/patches/**", - "scripts/nvx_tools/build.py", - "scripts/nvx_tools/build_constants.py", - ): - with self.subTest(path=path): - self.assertIn(f"'{path}'", cache_input) - kernel_input = next( - line - for line in action.splitlines() - if line.strip().startswith("KERNEL_INPUT_HASH:") - ) - self.assertNotIn("config-microvm-debug", kernel_input) - build_step = _composite_action_step(action, "Build Linux debug kernel (Docker)") - self.assertIn("inputs.debug-kernel == 'true'", build_step) - self.assertIn( - f"--target {DockerBuildConstants.DEBUG_KERNEL_TARGET}", build_step - ) - self.assertIn('default: "false"', action) - for name in DockerBuildConstants.DEBUG_KERNEL_ARTIFACT_NAMES: - with self.subTest(artifact=name): - self.assertEqual( - [line.strip() for line in action.splitlines()].count( - f"build/{name}" - ), - 2, - ) - - def test_ci_guest_images_input_leaves_only_the_debug_kernel(self): - action = ( - BuildConstants.REPO_ROOT - / ".github" - / "actions" - / "build-guest-artifacts" - / "action.yml" - ).read_text(encoding="utf-8") - guard = "inputs.guest-images == 'true'" - self.assertIn(" guest-images:\n", action) - self.assertIn('default: "true"', action) - steps = [ - line.strip().removeprefix("- name: ") - for line in action.splitlines() - if line.startswith(" - name: ") - ] - debug_steps = {name for name in steps if "debug kernel" in name} - self.assertEqual(len(debug_steps), 3) - always = {"Resolve guest artifact versions", "Resolve guest cache keys"} - for name in steps: - with self.subTest(step=name): - step = _composite_action_step(action, name) - if name in debug_steps or name in always: - self.assertNotIn("guest-images", step) - else: - self.assertIn(guard, step) - - def test_ci_guest_cache_keys_include_shared_build_modules(self): - action = ( - BuildConstants.REPO_ROOT - / ".github" - / "actions" - / "build-guest-artifacts" - / "action.yml" - ).read_text(encoding="utf-8") - for variable in ("ALPINE_INPUT_HASH", "UBUNTU_INPUT_HASH"): - assignment = next(line for line in action.splitlines() if variable in line) - self.assertIn("'scripts/nvx_tools/common.py'", assignment) - self.assertIn("'scripts/nvx_tools/guests.py'", assignment) - - def test_quality_checks_include_shared_identity_probe(self): - action = ( - BuildConstants.REPO_ROOT - / ".github" - / "actions" - / "check-quality" - / "action.yml" - ).read_text(encoding="utf-8") - - self.assertEqual(action.count("guest/common/nvx-identity-probe"), 2) - - def test_docker_guest_builder_pins_erofs_toolchain(self): - dockerfile = (BuildConstants.REPO_ROOT / "docker" / "Dockerfile").read_text( - encoding="utf-8" - ) - self.assertIn( - "ARG DEBIAN_IMAGE=debian:12-slim@sha256:" - "3783cc01769c7b2b1b83a5c5ad96c815348e28ed7da68e2e3687004faa906251", - dockerfile, - ) - self.assertIn("ARG EROFS_UTILS_VERSION=1.5-1", dockerfile) - self.assertIn("erofs-utils=${EROFS_UTILS_VERSION}", dockerfile) - - def test_azurelinux_initramfs_uses_locked_busybox_applets(self): - dockerfile = (BuildConstants.REPO_ROOT / "docker" / "Dockerfile").read_text( - encoding="utf-8" - ) - base_stage = dockerfile.split("FROM base AS kernel", 1)[0] - self.assertNotIn("busybox", base_stage) - azure_stage = dockerfile.split("FROM base AS azurelinux-initramfs", 1)[1] - self.assertIn("ln -sf ../sbin/busybox /rootfs/usr/bin/busybox", azure_stage) - applets = ( - azure_stage.split("for utility in ", 1)[1] - .split("; do", 1)[0] - .replace("\\\n", " ") - .split() - ) - for applet in ("sh", "arp", "nc", "wget", "mdev", "ifconfig", "route"): - self.assertIn(applet, applets) - # util-linux provides these; the BusyBox applets lack required options. - self.assertNotIn("setpriv", applets) - self.assertNotIn("unshare", applets) - launcher = ( - BuildConstants.REPO_ROOT - / "guest" - / "common" - / "nvx-container-launch-azurelinux" - ).read_text(encoding="utf-8") - self.assertIn("exec unshare --mount --pid --uts --fork --kill-child", launcher) - self.assertIn( - "busybox", [package["name"] for package in azurelinux.load_package_lock()] - ) - - def test_azurelinux_initramfs_gives_nobody_a_home(self): - dockerfile = (BuildConstants.REPO_ROOT / "docker" / "Dockerfile").read_text( - encoding="utf-8" - ) - azure_stage = dockerfile.split("FROM base AS azurelinux-initramfs", 1)[1] - self.assertIn( - "sed -i 's#^\\(nobody:[^:]*:65534:65534:[^:]*:\\)/dev/null:" - "#\\1/nonexistent:#'", - azure_stage, - ) - self.assertIn( - "grep -q '^nobody:[^:]*:65534:65534:[^:]*:/nonexistent:' " - "/rootfs/etc/passwd", - azure_stage, - ) - self.assertIn("install -d -m 0755 /rootfs/nonexistent", azure_stage) - - def test_azurelinux_rootfs_installs_only_locked_rpms(self): - dockerfile = (BuildConstants.REPO_ROOT / "docker" / "Dockerfile").read_text( - encoding="utf-8" - ) - self.assertNotIn("tdnf install", dockerfile) - packages_stage = dockerfile.split("FROM base AS azurelinux-packages", 1)[1] - self.assertIn( - "from nvx_tools.azurelinux import download_packages", - packages_stage.split("\nFROM ", 1)[0], - ) - rootfs_stage = dockerfile.split( - "FROM ${AZURELINUX_IMAGE} AS azurelinux-rootfs", 1 - )[1].split("\nFROM ", 1)[0] - self.assertIn( - "COPY --from=azurelinux-packages /out/rpms/ /tmp/azurelinux-rpms/", - rootfs_stage, - ) - checksig = rootfs_stage.index("rpm --checksig /tmp/azurelinux-rpms/*.rpm") - install = rootfs_stage.index("rpm --upgrade --verbose --hash") - self.assertLess(checksig, install) - - def test_azurelinux_manifest_uses_package_metadata(self): - dockerfile = (BuildConstants.REPO_ROOT / "docker" / "Dockerfile").read_text( - encoding="utf-8" - ) - self.assertIn("rpm -qa --queryformat", dockerfile) - self.assertIn("if name == 'gpg-pubkey':", dockerfile) - self.assertNotIn("'type': 'deb'", dockerfile) - self.assertIn("'packages': json.loads", dockerfile) - - def test_azurelinux_initramfs_packs_normalized_rootfs(self): - dockerfile = (BuildConstants.REPO_ROOT / "docker" / "Dockerfile").read_text( - encoding="utf-8" - ) - azure_stage = dockerfile.split("FROM base AS azurelinux-initramfs", 1)[1] - cleanup = azure_stage.index( - "rm -rf /rootfs/usr/lib/sysimage/tdnf /rootfs/var/cache/tdnf \\\n" - " /rootfs/var/cache/ldconfig /rootfs/var/lib/rpm" - ) - normalize = azure_stage.index( - "find . -exec touch --no-dereference " - f"--date=@{InitramfsBuildConstants.TIMESTAMP} {{}} +" - ) - pack = azure_stage.index("find . -print0 | LC_ALL=C sort -z | cpio") - self.assertLess(cleanup, normalize) - self.assertLess(normalize, pack) - - def test_azurelinux_manifest_records_shared_input_digest(self): - dockerfile = (BuildConstants.REPO_ROOT / "docker" / "Dockerfile").read_text( - encoding="utf-8" - ) - azure_stage = dockerfile.split("FROM base AS azurelinux-initramfs", 1)[1] - self.assertIn("from nvx_tools.azurelinux import input_sha256", azure_stage) - self.assertIn( - "'input_sha256': input_sha256(\n" - " image=os.environ['AZURELINUX_IMAGE'],\n" - " version=os.environ['AZURELINUX_VERSION'],\n" - " ),", - azure_stage, - ) - - def test_azurelinux_input_digest_tracks_guest_sources_and_dockerfile(self): - checkout_digest = azurelinux.input_sha256() - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - for source in azurelinux.input_files(): - destination = root / source.relative_to(BuildConstants.REPO_ROOT) - destination.parent.mkdir(parents=True, exist_ok=True) - shutil.copyfile(source, destination) - with patch.object(BuildConstants, "REPO_ROOT", root): - baseline = azurelinux.input_sha256() - self.assertEqual(baseline, checkout_digest) - self.assertNotEqual( - azurelinux.input_sha256(image="example.invalid/other@sha256:0"), - baseline, - ) - self.assertNotEqual(azurelinux.input_sha256(version="3.1"), baseline) - for relative in ( - "guest/common/init", - "docker/Dockerfile", - "azurelinux/packages.lock.json", - "scripts/nvx_tools/azurelinux.py", - "scripts/nvx_tools/build_constants.py", - "scripts/nvx_tools/common.py", - ): - with self.subTest(path=relative): - path = root / relative - original = path.read_bytes() - path.write_bytes(original + b"\n# changed\n") - self.assertNotEqual(azurelinux.input_sha256(), baseline) - path.write_bytes(original) - self.assertEqual(azurelinux.input_sha256(), baseline) - unrelated = root / "guest" / "ubuntu" / "nvx-bashrc" - unrelated.parent.mkdir(parents=True, exist_ok=True) - unrelated.write_text("changed\n", encoding="utf-8") - self.assertEqual(azurelinux.input_sha256(), baseline) - - def test_guest_init_runs_virtfs_helper_from_installed_path(self): - installed: list[Path] = [] - - def install(_source: Path, destination: Path) -> dict[str, str]: - installed.append(destination) - return {} - - root = Path("rootfs") - with ( - patch.object(build, "_install", side_effect=install), - patch.object(build, "_build_static_helper", return_value={}), - patch.object(build, "_build_device_io_helper", return_value={}), - ): - build._install_guest_files( - build_config.InitramfsBuildConfig(work=Path("work")), - root, - guests.ALPINE_GUEST, - ) - self.assertIn(root / "sbin" / "nvx-hostmount", installed) - - dockerfile = (BuildConstants.REPO_ROOT / "docker" / "Dockerfile").read_text( - encoding="utf-8" - ) - azure_stage = dockerfile.split("FROM base AS azurelinux-initramfs", 1)[1] - azure_sbin_scripts = azure_stage.split( - "install -m 0755 /repo/guest/common/init /rootfs/init\n", 1 - )[1].split("/rootfs/sbin/", 1)[0] - self.assertIn("/repo/guest/common/nvx-hostmount", azure_sbin_scripts) - init_script = ( - BuildConstants.REPO_ROOT / "guest" / "common" / "init" - ).read_text(encoding="utf-8") - self.assertIn( - '/sbin/nvx-hostmount || fatal "virtfs: failed to mount live host directory"', - init_script, - ) - self.assertNotIn("/usr/sbin/nvx-hostmount", init_script) - - def test_openvmm_ci_downloads_guest_artifacts(self): - workflow = ( - BuildConstants.REPO_ROOT / ".github" / "workflows" / "ci.yml" - ).read_text(encoding="utf-8") - openvmm_tests = _workflow_job(workflow, "openvmm-vmm-tests") - self.assertIn("needs: [artifacts, openvmm-changes]", openvmm_tests) - self.assertIn("needs.artifacts.result == 'success'", openvmm_tests) - self.assertIn("- name: Download guest artifacts", openvmm_tests) - self.assertIn("name: guest-artifacts", openvmm_tests) - self.assertIn("path: build", openvmm_tests) - self.assertIn("openvmm-vmm-tests", ci.REQUIRED_CI_OPENVMM_ARTIFACT_TEST_JOBS) - - def test_apk_add_uses_host_ca_bundle_without_overriding_configuration(self): - root = Path("root") - with ( - patch.object(build.ssl, "get_default_verify_paths") as verify_paths, - patch.object(build, "run_checked") as run, - ): - verify_paths.return_value.cafile = "/etc/host-ca.pem" - with patch.dict(os.environ, {}, clear=True): - build._apk_add(root, "example") - - environment = run.call_args.kwargs["env"] - self.assertEqual(environment["SSL_CERT_FILE"], "/etc/host-ca.pem") - self.assertEqual( - environment["LD_LIBRARY_PATH"], - f"{root / 'lib'}:{root / 'usr' / 'lib'}", - ) - - with patch.dict( - os.environ, - {"SSL_CERT_FILE": "/etc/configured-ca.pem"}, - clear=True, - ): - build._apk_add(root, "example") - - environment = run.call_args.kwargs["env"] - self.assertEqual(environment["SSL_CERT_FILE"], "/etc/configured-ca.pem") - - def test_device_io_helper_is_static_with_nonexecutable_stack(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - work = root / "work" - destination = root / "root" / "sbin" / "nvx-device-io" - work.mkdir() - destination.parent.mkdir(parents=True) - - def compile_helper( - command: list[str | os.PathLike[str]], **_kwargs: object - ) -> None: - output = Path(command[command.index("-o") + 1]) - output.write_bytes(b"static-elf") - - with ( - patch.object(build, "require_tool", return_value="cc"), - patch.object(build, "run_checked", side_effect=compile_helper) as run, - ): - provenance = build._build_device_io_helper(work, destination) - - command = run.call_args.args[0] - self.assertIn("-nostdlib", command) - self.assertIn("-static", command) - self.assertIn("-Wl,-z,noexecstack", command) - self.assertEqual(destination.read_bytes(), b"static-elf") - self.assertEqual( - provenance["binary_sha256"], common.sha256_file(destination) - ) - self.assertEqual(len(provenance["source_sha256"]), 64) - - def test_device_io_provenance_uses_shared_helper_source(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - checkout = root / "nvx" - source = checkout / "guest" / "common" / "nvx-device-io.c" - source.parent.mkdir(parents=True) - source.write_bytes(b"shared helper source") - initrd = root / "initramfs.cpio.gz" - manifest = initrd.with_name(f"{initrd.name}.packages.json") - manifest.write_text( - json.dumps( - { - "helpers": { - "nvx-device-io": { - "source_sha256": common.sha256_file(source), - "binary_sha256": "1" * 64, - } - } - } - ), - encoding="utf-8", - ) - - provenance = benchmark._device_io_helper_provenance( - argparse.Namespace(nvx_dir=checkout), - initrd, - ) - - self.assertEqual(provenance["source"], str(source.resolve())) - self.assertEqual( - provenance["source_sha256"], - common.sha256_file(source), - ) - - def test_sandbox_kernel_config_requires_every_feature(self): - with tempfile.TemporaryDirectory() as temporary: - config = Path(temporary) / ".config" - config.write_text( - "\n".join(KernelBuildConstants.REQUIRED_SANDBOX_CONFIG) + "\n", - encoding="utf-8", - ) - build._assert_sandbox_kernel_config(config) - - for missing in KernelBuildConstants.REQUIRED_SANDBOX_CONFIG: - with self.subTest(missing=missing): - config.write_text( - "\n".join( - setting - for setting in KernelBuildConstants.REQUIRED_SANDBOX_CONFIG - if setting != missing - ) - + "\n", - encoding="utf-8", - ) - with self.assertRaisesRegex(common.ScriptError, missing): - build._assert_sandbox_kernel_config(config) - - def test_network_kernel_config_requires_every_feature(self): - with tempfile.TemporaryDirectory() as temporary: - config = Path(temporary) / ".config" - config.write_text( - "\n".join(KernelBuildConstants.REQUIRED_NETWORK_CONFIG) + "\n", - encoding="utf-8", - ) - build._assert_network_kernel_config(config) - - for missing in KernelBuildConstants.REQUIRED_NETWORK_CONFIG: - with self.subTest(missing=missing): - config.write_text( - "\n".join( - setting - for setting in KernelBuildConstants.REQUIRED_NETWORK_CONFIG - if setting != missing - ) - + "\n", - encoding="utf-8", - ) - with self.assertRaisesRegex( - common.ScriptError, "portable network profile" - ) as raised: - build._assert_network_kernel_config(config) - self.assertIn(missing, str(raised.exception)) - - # An enabled SIT device leaves the "is not set" line out. - config.write_text( - "\n".join( - "CONFIG_IPV6_SIT=y" - if setting == "# CONFIG_IPV6_SIT is not set" - else setting - for setting in KernelBuildConstants.REQUIRED_NETWORK_CONFIG - ) - + "\n", - encoding="utf-8", - ) - with self.assertRaisesRegex(common.ScriptError, "CONFIG_IPV6_SIT"): - build._assert_network_kernel_config(config) - - def test_checked_in_config_provides_the_portable_network_profile(self): - config = BuildConstants.REPO_ROOT / KernelBuildConstants.INPUT_CONFIG - build._assert_network_kernel_config(config) - - def test_checked_in_config_preserves_generic_sandbox_capabilities(self): - config = BuildConstants.REPO_ROOT / "kernel" / "config-microvm" - build._assert_sandbox_kernel_config(config) - configured = config.read_text(encoding="utf-8").splitlines() - for setting in ( - "CONFIG_SECCOMP_FILTER=y", - "CONFIG_UNIX=y", - "# CONFIG_OVERLAY_FS_REDIRECT_ALWAYS_FOLLOW is not set", - ): - self.assertIn(setting, configured) - - def test_shared_status_kernel_config_is_required(self): - with tempfile.TemporaryDirectory() as temporary: - config = Path(temporary) / ".config" - config.write_text( - "\n".join(KernelBuildConstants.REQUIRED_SHARED_STATUS_CONFIG) + "\n", - encoding="utf-8", - ) - build._assert_shared_status_kernel_config(config) - - config.write_text("", encoding="utf-8") - with self.assertRaisesRegex( - common.ScriptError, - KernelBuildConstants.REQUIRED_SHARED_STATUS_CONFIG[0], - ): - build._assert_shared_status_kernel_config(config) - - def test_checked_in_config_meets_time_abi_kernel_contract(self): - config = BuildConstants.REPO_ROOT / KernelBuildConstants.INPUT_CONFIG - build._assert_time_abi_kernel_config(config) - build._assert_hardening_kernel_config(config) - build._assert_watchdog_kernel_config(config, debug=False) - configured = config.read_text(encoding="utf-8").splitlines() - for removed in ( - "CONFIG_CPU_FREQ=y", - "CONFIG_X86_INTEL_PSTATE=y", - "CONFIG_SCHED_MC_PRIO=y", - "CONFIG_HYPERV=y", - "CONFIG_KVM_GUEST=y", - "CONFIG_PARAVIRT_CLOCK=y", - "CONFIG_HALTPOLL_CPUIDLE=y", - "CONFIG_ARCH_CPUIDLE_HALTPOLL=y", - ): - with self.subTest(removed=removed): - self.assertNotIn(removed, configured) - self.assertFalse( - any(line.startswith("CONFIG_CPU_FREQ_") for line in configured) - ) - - def test_time_abi_kernel_config_requires_every_setting(self): - with tempfile.TemporaryDirectory() as temporary: - config = Path(temporary) / ".config" - for missing in KernelBuildConstants.REQUIRED_TIME_ABI_CONFIG: - with self.subTest(missing=missing): - config.write_text( - "\n".join( - setting - for setting in KernelBuildConstants.REQUIRED_TIME_ABI_CONFIG - if setting != missing - ) - + "\n", - encoding="utf-8", - ) - with self.assertRaisesRegex(common.ScriptError, missing): - build._assert_time_abi_kernel_config(config) - - def test_hardening_kernel_config_requires_every_setting(self): - with tempfile.TemporaryDirectory() as temporary: - config = Path(temporary) / ".config" - for missing in KernelBuildConstants.REQUIRED_HARDENING_CONFIG: - with self.subTest(missing=missing): - config.write_text( - "\n".join( - setting - for setting in KernelBuildConstants.REQUIRED_HARDENING_CONFIG - if setting != missing - ) - + "\n", - encoding="utf-8", - ) - with self.assertRaisesRegex(common.ScriptError, re.escape(missing)): - build._assert_hardening_kernel_config(config) - - def test_hardening_kernel_config_rejects_loadable_modules(self): - with tempfile.TemporaryDirectory() as temporary: - config = Path(temporary) / ".config" - config.write_text( - "\n".join( - (*KernelBuildConstants.REQUIRED_HARDENING_CONFIG, "CONFIG_FOO=m") - ) - + "\n", - encoding="utf-8", - ) - with self.assertRaisesRegex(common.ScriptError, "CONFIG_FOO=m"): - build._assert_hardening_kernel_config(config) - - def test_watchdog_kernel_config_separates_production_and_debug(self): - with tempfile.TemporaryDirectory() as temporary: - config = Path(temporary) / ".config" - config.write_text( - "\n".join(KernelBuildConstants.REQUIRED_DEBUG_CONFIG) + "\n", - encoding="utf-8", - ) - build._assert_watchdog_kernel_config(config, debug=True) - with self.assertRaisesRegex(common.ScriptError, "debug-only watchdogs"): - build._assert_watchdog_kernel_config(config, debug=False) - - config.write_text("CONFIG_DEBUG_KERNEL=y\n", encoding="utf-8") - build._assert_watchdog_kernel_config(config, debug=False) - with self.assertRaisesRegex( - common.ScriptError, "CONFIG_SOFTLOCKUP_DETECTOR=y" - ): - build._assert_watchdog_kernel_config(config, debug=True) - - def test_kernel_config_fragment_replaces_base_assignments(self): - base = "\n".join( - ( - "#", - "# Kernel hacking", - "#", - "# CONFIG_DEBUG_KERNEL is not set", - "CONFIG_RCU_CPU_STALL_TIMEOUT=21", - "CONFIG_UNRELATED=y", - ) - ) - fragment = "\n".join( - ( - "# Comment lines and blank lines are ignored.", - "", - "CONFIG_DEBUG_KERNEL=y", - "# CONFIG_DEBUG_MISC is not set", - "CONFIG_RCU_CPU_STALL_TIMEOUT=21", - ) - ) - self.assertEqual( - build.merge_kernel_config_fragment(base, fragment).splitlines(), - [ - "#", - "# Kernel hacking", - "#", - "CONFIG_UNRELATED=y", - "CONFIG_DEBUG_KERNEL=y", - "# CONFIG_DEBUG_MISC is not set", - "CONFIG_RCU_CPU_STALL_TIMEOUT=21", - ], - ) - for invalid, message in ( - ("DEBUG_KERNEL=y\n", "line 1 is invalid"), - ("# only a comment\n", "assigns no symbols"), - ( - "CONFIG_DEBUG_KERNEL=y\n# CONFIG_DEBUG_KERNEL is not set\n", - "more than once", - ), - ): - with self.subTest(fragment=invalid): - with self.assertRaisesRegex(common.ScriptError, message): - build.merge_kernel_config_fragment(base, invalid) - - def test_checked_in_debug_fragment_only_adds_watchdogs(self): - base = (BuildConstants.REPO_ROOT / KernelBuildConstants.INPUT_CONFIG).read_text( - encoding="utf-8" - ) - fragment = ( - BuildConstants.REPO_ROOT / KernelBuildConstants.DEBUG_CONFIG_FRAGMENT - ).read_text(encoding="utf-8") - merged = build.merge_kernel_config_fragment(base, fragment).splitlines() - for setting in KernelBuildConstants.REQUIRED_DEBUG_CONFIG: - with self.subTest(setting=setting): - self.assertIn(setting, merged) - for setting in ( - "# CONFIG_DEBUG_MISC is not set", - "# CONFIG_RCU_TRACE is not set", - "# CONFIG_X86_DEBUG_FPU is not set", - "CONFIG_RCU_CPU_STALL_TIMEOUT=21", - ): - with self.subTest(setting=setting): - self.assertIn(setting, merged) - self.assertNotIn("# CONFIG_DEBUG_KERNEL is not set", merged) - for setting in KernelBuildConstants.REQUIRED_TIME_ABI_CONFIG: - with self.subTest(time_abi=setting): - self.assertIn(setting, merged) - - def test_debug_kernel_build_applies_fragment_and_records_provenance(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - input_config = root / KernelBuildConstants.INPUT_CONFIG - input_config.parent.mkdir(parents=True) - input_config.write_text( - "\n".join( - ( - *KernelBuildConstants.REQUIRED_DIRECT_BOOT_CONFIG, - *KernelBuildConstants.REQUIRED_VIRTIO_CONSOLE_CONFIG, - *KernelBuildConstants.REQUIRED_SHARED_STATUS_CONFIG, - *KernelBuildConstants.REQUIRED_SANDBOX_CONFIG, - *KernelBuildConstants.REQUIRED_NETWORK_CONFIG, - *KernelBuildConstants.REQUIRED_TIME_ABI_CONFIG, - *KernelBuildConstants.REQUIRED_HARDENING_CONFIG, - "# CONFIG_DEBUG_KERNEL is not set", - ) - ) - + "\n", - encoding="utf-8", - ) - fragment = root / KernelBuildConstants.DEBUG_CONFIG_FRAGMENT - fragment.write_text( - "# debug fixture\n" - + "\n".join(KernelBuildConstants.REQUIRED_DEBUG_CONFIG) - + "\n", - encoding="utf-8", - ) - patch_path = root / KernelBuildConstants.PATCH_DIRECTORY / "example.patch" - patch_path.parent.mkdir() - patch_path.write_text("patch", encoding="utf-8") - source = root / "source" - source.mkdir() - output_directory = root / "output" - output_directory.mkdir() - production_provenance = ( - output_directory / KernelBuildConstants.PROVENANCE_NAME - ) - production_provenance.write_text("production", encoding="utf-8") - config = build_config.KernelBuildConfig( - work=root / "work-debug", - output=output_directory / KernelBuildConstants.DEBUG_BINARY_NAME, - debug=True, - ) - - with patch.object(BuildConstants, "REPO_ROOT", root): - source_fingerprint = build._kernel_source_fingerprint() - - def run_build(command: object, **_kwargs: object) -> None: - if isinstance(command, list) and command[-1] == "vmlinux": - (config.work / "vmlinux").write_bytes(b"debug kernel") - - with ( - patch.object(BuildConstants, "REPO_ROOT", root), - patch.object(build, "_require_linux"), - patch.object(build, "require_tool", return_value="tool"), - patch.object( - build, - "prepare_kernel_source", - return_value=(source, source_fingerprint), - ), - patch.object(build, "run_checked", side_effect=run_build), - ): - build.build_kernel(config) - - generated = ( - output_directory / KernelBuildConstants.DEBUG_CONFIG_NAME - ).read_text(encoding="utf-8") - self.assertIn("CONFIG_DEBUG_KERNEL=y", generated.splitlines()) - self.assertNotIn("# CONFIG_DEBUG_KERNEL is not set", generated) - provenance = json.loads( - ( - output_directory / KernelBuildConstants.DEBUG_PROVENANCE_NAME - ).read_text(encoding="utf-8") - ) - self.assertEqual( - provenance["debug_config_fragment"], - { - "path": KernelBuildConstants.DEBUG_CONFIG_FRAGMENT.as_posix(), - "sha256": common.sha256_file(fragment), - }, - ) - self.assertEqual( - provenance["input_config"]["sha256"], common.sha256_file(input_config) - ) - self.assertEqual( - provenance["kernel_sha256"], - hashlib.sha256(b"debug kernel").hexdigest(), - ) - self.assertEqual( - production_provenance.read_text(encoding="utf-8"), "production" - ) - self.assertIn( - "debug_config_fragment_sha256", - (config.work / KernelBuildConstants.BUILD_STAMP_NAME).read_text( - encoding="utf-8" - ), - ) - self.assertEqual( - config.provenance_name, KernelBuildConstants.DEBUG_PROVENANCE_NAME - ) - self.assertEqual( - build_config.KernelBuildConfig().provenance_name, - KernelBuildConstants.PROVENANCE_NAME, - ) - self.assertNotIn("debug_config_fragment", build.kernel_provenance_inputs()) - - def test_guest_build_adds_debug_kernel_only_when_requested(self): - with ( - patch.object(build, "build_kernel") as kernel, - patch.object(build, "build_initramfs"), - patch.object(build, "build_docker_debug_kernel") as docker_debug, - ): - native = build_config.BuildConfig(native_guest=True, debug_kernel=True) - build.build_guest(native) - self.assertEqual( - kernel.call_args_list, - [ - call(native.kernel), - call(build_config.KernelBuildConfig.debug_variant()), - ], - ) - docker_debug.assert_not_called() - - with ( - patch.object(build, "build_docker_artifacts") as docker, - patch.object(build, "build_docker_debug_kernel") as docker_debug, - ): - portable = build_config.BuildConfig(debug_kernel=True) - build.build_guest(portable) - docker.assert_called_once_with(portable.docker, "alpine") - docker_debug.assert_called_once_with(portable.docker) - - with ( - patch.object(build, "build_docker_artifacts"), - patch.object(build, "build_docker_debug_kernel") as docker_debug, - ): - build.build_guest(build_config.BuildConfig()) - docker_debug.assert_not_called() - - debug = build_config.KernelBuildConfig.debug_variant() - self.assertTrue(debug.debug) - self.assertEqual(debug.output.name, KernelBuildConstants.DEBUG_BINARY_NAME) - self.assertEqual( - debug.work.name, KernelBuildConstants.DEBUG_WORK_DIRECTORY_NAME - ) - - def test_guest_files_build_time_probe_with_its_own_flags(self): - source = BuildConstants.REPO_ROOT / "guest" / "common" / "nvx-time-probe.c" - header = source.read_text(encoding="utf-8").splitlines()[:2] - self.assertEqual( - header, - [ - "// Copyright (c) Microsoft Corporation.", - "// Licensed under the MIT License.", - ], - ) - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) / "root" - work = Path(temporary) / "work" - for directory in (root / "sbin", root / "etc", work): - directory.mkdir(parents=True) - commands: list[list[str]] = [] - - def compile_helper( - command: list[str | os.PathLike[str]], **_kwargs: object - ) -> None: - commands.append([os.fspath(part) for part in command]) - output = Path(command[command.index("-o") + 1]) - output.write_bytes(f"static-elf {output.name}".encode()) - - def find_tool(name: str) -> str: - return name - - with ( - patch.object(build, "require_tool", side_effect=find_tool), - patch.object(build, "run_checked", side_effect=compile_helper), - ): - helpers = build._install_guest_files( - build_config.InitramfsBuildConfig(work=work), - root, - guests.guest_descriptor("alpine"), - ) - - probe = InitramfsBuildConstants.TIME_PROBE_NAME - flags_by_source = { - Path(command[-1]).stem: command[1 : command.index("-o")] - for command in commands - } - compiler_by_source = { - Path(command[-1]).stem: command[0] for command in commands - } - self.assertEqual( - flags_by_source[probe], list(InitramfsBuildConstants.TIME_PROBE_CFLAGS) - ) - self.assertIn("-pthread", flags_by_source[probe]) - musl = InitramfsBuildConstants.MUSL_COMPILER - self.assertEqual(compiler_by_source[probe], musl) - for name in InitramfsBuildConstants.STATIC_HELPERS: - with self.subTest(helper=name): - self.assertEqual( - flags_by_source[name], - list(InitramfsBuildConstants.STATIC_HELPER_CFLAGS), - ) - self.assertEqual(compiler_by_source[name], "cc") - for name in InitramfsBuildConstants.MUSL_STATIC_HELPERS: - with self.subTest(helper=name): - self.assertEqual( - flags_by_source[name], - list(InitramfsBuildConstants.STATIC_HELPER_CFLAGS), - ) - self.assertEqual(compiler_by_source[name], musl) - dockerfile = (BuildConstants.REPO_ROOT / "docker" / "Dockerfile").read_text( - encoding="utf-8" - ) - self.assertIn(" musl-tools \\\n", dockerfile) - self.assertEqual(helpers[probe]["source_sha256"], common.sha256_file(source)) - self.assertEqual( - helpers[probe]["binary_sha256"], - hashlib.sha256(f"static-elf {probe}".encode()).hexdigest(), - ) - - def test_docker_debug_kernel_target_exports_debug_artifacts(self): - dockerfile = (BuildConstants.REPO_ROOT / "docker" / "Dockerfile").read_text( - encoding="utf-8" - ) - self.assertIn( - "FROM base AS kernel-debug\n" - "RUN python3 scripts/nvx.py build-kernel --debug\n", - dockerfile, - ) - self.assertIn( - f"FROM scratch AS {DockerBuildConstants.DEBUG_KERNEL_TARGET}", dockerfile - ) - for name in DockerBuildConstants.DEBUG_KERNEL_ARTIFACT_NAMES: - with self.subTest(artifact=name): - self.assertIn( - f"COPY --from=kernel-debug /repo/build/{name} /{name}", dockerfile - ) - - with tempfile.TemporaryDirectory() as temporary: - destination = Path(temporary) - - def export(_command: object, **_kwargs: object) -> None: - for name in DockerBuildConstants.DEBUG_KERNEL_ARTIFACT_NAMES: - (destination / name).write_bytes(b"artifact") - - config = build_config.DockerBuildConfig(artifact_destination=destination) - with ( - patch.object(build, "require_tool", return_value="docker"), - patch.object(build, "run_checked", side_effect=export) as run, - ): - build.build_docker_debug_kernel(config) - command = run.call_args.args[0] - self.assertEqual( - command[command.index("--target") + 1], - DockerBuildConstants.DEBUG_KERNEL_TARGET, - ) - - for name in DockerBuildConstants.DEBUG_KERNEL_ARTIFACT_NAMES: - (destination / name).unlink() - with ( - patch.object(build, "require_tool", return_value="docker"), - patch.object(build, "run_checked"), - self.assertRaisesRegex(common.ScriptError, "did not produce"), - ): - build.build_docker_debug_kernel(config) - - -def _posix_shell() -> str | None: - shell = shutil.which("sh") - if shell is None: - git = shutil.which("git") - git_shell = Path(git).parent.parent / "bin" / "sh.exe" if git else None - if git_shell is not None and git_shell.is_file(): - shell = str(git_shell) - return shell - - -def _shell_function(source: str, name: str) -> str: - start = source.index(f"\n{name}() {{\n") + 1 - return source[start : source.index("\n}\n", start) + 3] - - -class SandboxShareAgentTests(unittest.TestCase): - AGENT = Path(__file__).parents[1] / "guest" / "common" / "nvx-init-agent" - - def setUp(self): - shell = _posix_shell() - if shell is None: - self.skipTest("POSIX shell is unavailable") - self.shell = shell - self.source = self.AGENT.read_text(encoding="utf-8") - temporary = tempfile.TemporaryDirectory() - self.addCleanup(temporary.cleanup) - self.root = Path(temporary.name) - self.rootfs = self.root / "rootfs" - self.rootfs.mkdir() - self.runtime = (self.root / "run").as_posix() - - SHARE_FUNCTIONS = ( - "validate_share_target", - "make_share_mount_point", - "record_live_share", - "claim_share_target", - "mount_live_share", - "parse_share_child", - "bind_live_share", - "mount_live_shares", - ) - - def _aggregate(self, *children: str, mode: str = "rw") -> str: - """Returns the bootstrap of an aggregate whose children the - CHILD,TARGET,MODE specifications `children` place.""" - tokens = [ - f"virtfs_dir={self.runtime}/shares", - "virtfs_tag=microvm", - f"virtfs_mode={mode}", - "virtfs_aggregate=1", - *(f"nvx_share={child}" for child in children), - ] - return " ".join(tokens) - - def _run(self, cmdline: str) -> tuple[subprocess.CompletedProcess[str], str]: - mount_log = self.root / "mount.log" - mount_log.unlink(missing_ok=True) - functions = "".join( - _shell_function(self.source, name) for name in self.SHARE_FUNCTIONS - ) - script = ( - "set -eu\n" - "rootfs=$1\ncmdline=$2\nmount_log=$3\nruntime=$4\nshare_mountpoints=\n" - 'fatal() { echo "FATAL: $*" >&2; exit 125; }\n' - 'mount() { printf "%s\\n" "$*" >>"$mount_log"; }\n' - f"{functions}" - "mount_live_shares\n" - "echo \"mountpoints=$(printf '%s' \"$share_mountpoints\" | tr '\\n' ' ')\"\n" - ) - result = subprocess.run( - [ - self.shell, - "-s", - "--", - self.rootfs.as_posix(), - cmdline, - mount_log.as_posix(), - self.runtime, - ], - input=script, - text=True, - capture_output=True, - check=False, - ) - log = mount_log.read_text(encoding="utf-8") if mount_log.exists() else "" - return result, log - - def _run_teardown( - self, script: str, *, shares: tuple[str, ...], failing: str = "" - ) -> tuple[subprocess.CompletedProcess[str], list[str]]: - runtime = self.root / "run" - runtime.mkdir(exist_ok=True) - for name in ("container.pid", "workload-machine-id"): - (runtime / name).write_text("x\n", encoding="utf-8") - log = self.root / "teardown.log" - log.unlink(missing_ok=True) - functions = "".join( - _shell_function(self.source, name) - for name in ("unmount_live_shares", "fatal", "teardown") - ).replace("/sbin/nvx-exit", "nvx_exit") - # The agent records mounted shares one per line, most recent first. - # Windows drops a newline from a process argument, so pass a separator. - mountpoints = "|".join( - f"{self.rootfs.as_posix()}{target}" for target in reversed(shares) - ) - result = subprocess.run( - [ - self.shell, - "-s", - "--", - runtime.as_posix(), - log.as_posix(), - mountpoints, - failing, - ], - input=( - "set -eu\n" - "runtime=$1\nlog=$2\nfailing=$4\n" - "share_mountpoints=$(printf '%s' \"$3\" | tr '|' '\\n')\n" - "rootfs=$runtime/rootfs\nlayers=$runtime/layers\n" - "scratch=$runtime/scratch\n" - 'umount() { printf "umount %s\\n" "$1" >>"$log"; ' - '[ "$1" != "$failing" ]; }\n' - 'mountpoint() { [ "$2" = "$layers/distro" ]; }\n' - 'nvx_exit() { printf "exit %s\\n" "$1" >>"$log"; exit "$1"; }\n' - f"{functions}{script}\n" - ), - text=True, - capture_output=True, - check=False, - ) - lines = log.read_text(encoding="utf-8").splitlines() if log.exists() else [] - return result, lines - - def test_agent_mounts_share_after_lifecycle_checks_for_both_lifecycles(self): - mount_call = self.source.index("\nmount_live_shares\n") - self.assertLess( - self.source.index('fatal "configured workload home is unavailable"'), - mount_call, - ) - self.assertLess( - self.source.index('fatal "unsupported workload lifecycle'), mount_call - ) - machine_id = self.source.index('>"$runtime/workload-machine-id"') - managed_agent = self.source.index("\n /sbin/nvx-managed-agent \\\n") - self.assertLess(mount_call, machine_id) - self.assertLess(machine_id, managed_agent) - self.assertLess(managed_agent, self.source.index("/sbin/nvx-container-launch")) - self.assertNotIn("exec /sbin/nvx-managed-agent", self.source) - self.assertEqual(self.source.count('\nteardown "$status"\n'), 1) - self.assertEqual(self.source.count('\n teardown "$status"\n'), 1) - self.assertLess(managed_agent, self.source.index('\n teardown "$status"\n')) - - def test_agent_teardown_unmounts_share_before_overlay_layers_and_scratch(self): - result, log = self._run_teardown("teardown 7", shares=("/workspace",)) - runtime = (self.root / "run").as_posix() - self.assertEqual(result.returncode, 7, result.stderr) - self.assertEqual( - log, - [ - f"umount {self.rootfs.as_posix()}/workspace", - f"umount {runtime}/rootfs", - f"umount {runtime}/layers/distro", - f"umount {runtime}/scratch", - "exit 7", - ], - ) - self.assertIn("NVX-SANDBOX-EXIT: status=7", result.stdout) - self.assertFalse((self.root / "run" / "workload-machine-id").exists()) - self.assertFalse((self.root / "run" / "container.pid").exists()) - - result, log = self._run_teardown("teardown 0", shares=()) - self.assertEqual(result.returncode, 0, result.stderr) - self.assertEqual(log[0], f"umount {runtime}/rootfs") - - def test_agent_teardown_unmounts_shares_in_reverse_mount_order(self): - rootfs = self.rootfs.as_posix() - runtime = (self.root / "run").as_posix() - result, log = self._run_teardown( - "teardown 0", shares=("/workspace", "/opt/hostedtoolcache") - ) - self.assertEqual(result.returncode, 0, result.stderr) - self.assertEqual( - log[:3], - [ - f"umount {rootfs}/opt/hostedtoolcache", - f"umount {rootfs}/workspace", - f"umount {runtime}/rootfs", - ], - ) - - # A share that fails to unmount does not stop the others. - result, log = self._run_teardown( - "teardown 0", - shares=("/workspace", "/opt/hostedtoolcache"), - failing=f"{rootfs}/opt/hostedtoolcache", - ) - self.assertEqual(result.returncode, 1, result.stdout) - self.assertIn("failed to unmount the live share", result.stderr) - self.assertEqual( - log[:2], - [f"umount {rootfs}/opt/hostedtoolcache", f"umount {rootfs}/workspace"], - ) - self.assertEqual(log[-1], "exit 1") - - def test_agent_teardown_reports_share_unmount_failure(self): - share = f"{self.rootfs.as_posix()}/workspace" - result, log = self._run_teardown( - "teardown 0", shares=("/workspace",), failing=share - ) - self.assertEqual(result.returncode, 1, result.stdout) - self.assertIn("failed to unmount the live share", result.stderr) - self.assertEqual(log[0], f"umount {share}") - self.assertEqual(log[-1], "exit 1") - self.assertIn("NVX-SANDBOX-EXIT: status=1", result.stdout) - - def test_agent_fatal_unmounts_share_before_power_off(self): - result, log = self._run_teardown( - 'fatal "synthetic failure"', shares=("/workspace",) - ) - self.assertEqual(result.returncode, 125, result.stdout) - self.assertIn("NVX-SANDBOX-ERROR: synthetic failure", result.stderr) - self.assertEqual( - log, [f"umount {self.rootfs.as_posix()}/workspace", "exit 125"] - ) - - result, log = self._run_teardown('fatal "early failure"', shares=()) - self.assertEqual(result.returncode, 125, result.stdout) - self.assertEqual(log, ["exit 125"]) - - rootfs = self.rootfs.as_posix() - result, log = self._run_teardown( - 'fatal "late failure"', shares=("/workspace", "/opt/hostedtoolcache") - ) - self.assertEqual(result.returncode, 125, result.stdout) - self.assertEqual( - log, - [ - f"umount {rootfs}/opt/hostedtoolcache", - f"umount {rootfs}/workspace", - "exit 125", - ], - ) - - def test_agent_skips_share_without_bootstrap_tokens(self): - result, log = self._run("console=hvc0 nvx_sandbox=1") - self.assertEqual(result.returncode, 0, result.stderr) - self.assertIn("mountpoints=\n", result.stdout) - self.assertEqual(log, "") - - def test_agent_mounts_share_inside_container_rootfs(self): - (self.rootfs / "opt").mkdir() - for mode in ("rw", "ro"): - with self.subTest(mode=mode): - result, log = self._run( - "nvx_sandbox=1 virtfs_dir=/opt/hostedtoolcache " - f"virtfs_tag=microvm virtfs_mode={mode}" - ) - self.assertEqual(result.returncode, 0, result.stderr) - self.assertTrue((self.rootfs / "opt" / "hostedtoolcache").is_dir()) - self.assertIn( - f"NVX-SANDBOX-SHARE: target=/opt/hostedtoolcache mode={mode}", - result.stdout, - ) - self.assertEqual( - log.split(), - [ - "-t", - "virtiofs", - "-o", - f"{mode},nosuid,nodev", - "microvm", - f"{self.rootfs.as_posix()}/opt/hostedtoolcache", - ], - ) - self.assertIn( - f"mountpoints={self.rootfs.as_posix()}/opt/hostedtoolcache\n", - result.stdout, - ) - - def test_agent_binds_every_child_of_an_aggregate(self): - rootfs = self.rootfs.as_posix() - result, log = self._run( - "nvx_sandbox=1 " - + self._aggregate("0,/workspace,rw", "1,/opt/hostedtoolcache,ro") - ) - self.assertEqual(result.returncode, 0, result.stderr) - # The aggregate stays outside the container rootfs; each child is bound - # inside it with its own mode. - self.assertEqual( - log.splitlines(), - [ - f"-t virtiofs -o rw,nosuid,nodev microvm {self.runtime}/shares", - f"-o bind {self.runtime}/shares/0 {rootfs}/workspace", - f"-o remount,bind,rw,nosuid,nodev {rootfs}/workspace", - f"-o bind {self.runtime}/shares/1 {rootfs}/opt/hostedtoolcache", - f"-o remount,bind,ro,nosuid,nodev {rootfs}/opt/hostedtoolcache", - ], - ) - self.assertIn( - "NVX-SANDBOX-SHARE: target=/workspace mode=rw child=0\n" - "NVX-SANDBOX-SHARE: target=/opt/hostedtoolcache mode=ro child=1\n", - result.stdout, - ) - # Teardown unmounts the most recent bind first, and the aggregate last. - self.assertIn( - f"mountpoints={rootfs}/opt/hostedtoolcache {rootfs}/workspace " - f"{self.runtime}/shares\n", - result.stdout, - ) - - def test_agent_binds_children_by_the_names_that_nvx_gives_them(self): - rootfs = self.rootfs.as_posix() - workspace = sandbox.aggregate_child_name(0, "/workspace") - tools = sandbox.aggregate_child_name(1, "/opt/tools") - result, log = self._run( - self._aggregate( - f"{workspace},/workspace,rw", f"{tools},/opt/tools,ro", "a.b_C-9,/x,ro" - ) - ) - self.assertEqual(result.returncode, 0, result.stderr) - self.assertIn( - f"-o bind {self.runtime}/shares/{workspace} {rootfs}/workspace\n", log - ) - self.assertIn( - f"-o bind {self.runtime}/shares/{tools} {rootfs}/opt/tools\n", log - ) - self.assertIn(f"-o bind {self.runtime}/shares/a.b_C-9 {rootfs}/x\n", log) - self.assertIn( - f"NVX-SANDBOX-SHARE: target=/workspace mode=rw child={workspace}\n", - result.stdout, - ) - # A repeated name is refused, however the child is named. - result, log = self._run( - self._aggregate(f"{workspace},/workspace,rw", f"{workspace},/b,ro") - ) - self.assertEqual(result.returncode, 125, result.stdout) - self.assertIn("child is not unique", result.stderr) - self.assertEqual(log, "") - - def test_hostmount_accepts_only_child_names_inside_the_aggregate(self): - hostmount = self.AGENT.with_name("nvx-hostmount").read_text(encoding="utf-8") - script = ( - 'fail() { echo "FAIL: $*" >&2; exit 1; }\n' - f"{_shell_function(hostmount, 'parse_share')}" - 'parse_share "$1"\n' - 'printf "%s|%s|%s|%s\\n" "$share_child" "$share_target" "$share_mode" ' - '"$share_kind"\n' - ) - - def parse(share: str) -> subprocess.CompletedProcess[str]: - return subprocess.run( - [self.shell, "-s", "--", share], - input=script, - text=True, - capture_output=True, - check=False, - ) - - name = sandbox.aggregate_child_name(0, "/workspace") - for share, expected in ( - (f"{name},/workspace,rw", f"{name}|/workspace|rw|directory\n"), - ("0,/opt/tools,ro", "0|/opt/tools|ro|directory\n"), - ("a.b_C-9,/x,ro", "a.b_C-9|/x|ro|directory\n"), - # A file child is bound at a file. - ("1,/config/settings.json,ro,file", "1|/config/settings.json|ro|file\n"), - ("1,/results/output.txt,rw,file", "1|/results/output.txt|rw|file\n"), - ): - with self.subTest(share=share): - result = parse(share) - self.assertEqual(result.returncode, 0, result.stderr) - self.assertEqual(result.stdout, expected) - for share in ( - ",/workspace,rw", - ".,/workspace,rw", - "..,/workspace,rw", - "a/b,/workspace,rw", - "a:b,/workspace,rw", - "a*,/workspace,rw", - "0,/workspace,file", - "0,/workspace,rw,dir", - ): - with self.subTest(share=share): - result = parse(share) - self.assertEqual(result.returncode, 1, result.stdout) - self.assertIn("invalid live share", result.stderr) - - def test_hostmount_makes_file_mount_points_without_following_links(self): - hostmount = self.AGENT.with_name("nvx-hostmount").read_text(encoding="utf-8") - script = ( - f"{_shell_function(hostmount, 'make_file_mount_point')}" - 'make_file_mount_point "$1"\n' - ) - - def make(target: Path) -> int: - return subprocess.run( - [self.shell, "-s", "--", target.as_posix()], - input=script, - text=True, - capture_output=True, - check=False, - ).returncode - - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - # A missing file is created with its directories, and an existing - # one is kept as it is. - created = root / "config" / "settings.json" - self.assertEqual(make(created), 0) - self.assertEqual(created.read_bytes(), b"") - existing = root / "existing.json" - existing.write_bytes(b"{}") - self.assertEqual(make(existing), 0) - self.assertEqual(existing.read_bytes(), b"{}") - (root / "directory").mkdir() - self.assertEqual(make(root / "directory"), 1) - if os.name == "nt": - return - # A link is refused before anything is created through it, whether - # or not its target exists. - dangling = root / "dangling.json" - dangling.symlink_to(root / "elsewhere.json") - self.assertEqual(make(dangling), 1) - self.assertFalse((root / "elsewhere.json").exists()) - linked = root / "linked.json" - linked.symlink_to(existing) - self.assertEqual(make(linked), 1) - self.assertEqual(existing.read_bytes(), b"{}") - - def test_agent_binds_a_file_child_at_a_file(self): - rootfs = self.rootfs.as_posix() - (self.rootfs / "opt").mkdir() - (self.rootfs / "opt" / "existing.json").write_text("{}", encoding="utf-8") - result, log = self._run( - "nvx_sandbox=1 " - + self._aggregate( - "0,/workspace,rw", - "1,/etc/app/settings.json,ro,file", - "2,/opt/existing.json,rw,file", - ) - ) - self.assertEqual(result.returncode, 0, result.stderr) - # Each file child gets a file mount point, created inside the container - # rootfs or reused there, and is bound with its own mode. - settings = self.rootfs / "etc" / "app" / "settings.json" - self.assertTrue(settings.is_file()) - self.assertEqual(settings.read_text(encoding="utf-8"), "") - self.assertEqual( - (self.rootfs / "opt" / "existing.json").read_text(encoding="utf-8"), "{}" - ) - self.assertTrue((self.rootfs / "workspace").is_dir()) - self.assertEqual( - log.splitlines(), - [ - f"-t virtiofs -o rw,nosuid,nodev microvm {self.runtime}/shares", - f"-o bind {self.runtime}/shares/0 {rootfs}/workspace", - f"-o remount,bind,rw,nosuid,nodev {rootfs}/workspace", - f"-o bind {self.runtime}/shares/1 {rootfs}/etc/app/settings.json", - f"-o remount,bind,ro,nosuid,nodev {rootfs}/etc/app/settings.json", - f"-o bind {self.runtime}/shares/2 {rootfs}/opt/existing.json", - f"-o remount,bind,rw,nosuid,nodev {rootfs}/opt/existing.json", - ], - ) - self.assertIn( - f"mountpoints={rootfs}/opt/existing.json {rootfs}/etc/app/settings.json " - f"{rootfs}/workspace {self.runtime}/shares\n", - result.stdout, - ) - - # A directory is no file mount point, a file no directory one, and - # the runtime binds the machine ID itself. - (self.rootfs / "dir").mkdir() - (self.rootfs / "file").write_text("x", encoding="utf-8") - for child, message in ( - ("0,/dir,ro,file", "not a regular file"), - ("0,/file/settings.json,ro,file", "not a directory"), - ("0,/file,ro", "not a directory"), - ("0,/etc/machine-id,ro,file", "reserved"), - ("0,/etc/machine-id/child,ro", "reserved"), - ): - with self.subTest(child=child): - result, log = self._run(self._aggregate(child)) - self.assertEqual(result.returncode, 125, result.stdout) - self.assertIn(message, result.stderr) - self.assertEqual(log, "") - - def test_agent_rejects_repeated_children_and_overlapping_targets(self): - for children, message in ( - (("0,/workspace,rw", "0,/opt/tools,ro"), "child is not unique"), - (("0,/workspace,rw", "1,/workspace,ro"), "targets overlap"), - (("0,/workspace,rw", "1,/workspace/cache,ro"), "targets overlap"), - (("0,/opt/tools/node,ro", "1,/opt,rw"), "targets overlap"), - ): - with self.subTest(children=children): - result, log = self._run(self._aggregate(*children)) - self.assertEqual(result.returncode, 125, result.stdout) - self.assertIn(message, result.stderr) - # Every child is checked before anything is mounted. - self.assertEqual(log, "") - - # A shared prefix that is not a path component is not an overlap. - result, log = self._run(self._aggregate("0,/work,rw", "1,/workspace,ro")) - self.assertEqual(result.returncode, 0, result.stderr) - self.assertEqual(len(log.splitlines()), 5) - - def test_agent_parses_every_share_before_mounting_any(self): - for cmdline in ( - "virtfs_dir=/workspace virtfs_tag=microvm virtfs_mode=rw " - "virtfs_dir=/opt/tools", - "virtfs_tag=microvm virtfs_mode=ro", - "virtfs_dir=/workspace virtfs_tag=microvm virtfs_mode=rw " - "virtfs_dir= virtfs_tag=microvm virtfs_mode=ro", - "virtfs_aggregate=1", - "nvx_share=0,/workspace,rw", - ): - with self.subTest(cmdline=cmdline): - result, log = self._run(cmdline) - self.assertEqual(result.returncode, 125, result.stdout) - self.assertIn("bootstrap is incomplete", result.stderr) - self.assertEqual(log, "") - - def test_agent_fatal_unmounts_mounted_shares(self): - mount_log = self.root / "mount.log" - functions = "".join( - _shell_function(self.source, name) - for name in ("unmount_live_shares", "fatal", *self.SHARE_FUNCTIONS) - ).replace("/sbin/nvx-exit", "nvx_exit") - result = subprocess.run( - [ - self.shell, - "-s", - "--", - self.rootfs.as_posix(), - self._aggregate("0,/workspace,rw", "1,/opt/tools,ro"), - mount_log.as_posix(), - self.runtime, - ], - input=( - "set -eu\n" - "rootfs=$1\ncmdline=$2\nmount_log=$3\nruntime=$4\n" - "share_mountpoints=\n" - 'mount() { printf "mount %s\\n" "$*" >>"$mount_log"; ' - 'case "$*" in *shares/1*) return 1 ;; esac; }\n' - 'umount() { printf "umount %s\\n" "$1" >>"$mount_log"; }\n' - 'nvx_exit() { printf "exit %s\\n" "$1" >>"$mount_log"; exit "$1"; }\n' - f"{functions}mount_live_shares\n" - ), - text=True, - capture_output=True, - check=False, - ) - self.assertEqual(result.returncode, 125, result.stdout) - self.assertIn("failed to bind live share 1", result.stderr) - rootfs = self.rootfs.as_posix() - self.assertEqual( - mount_log.read_text(encoding="utf-8").splitlines(), - [ - f"mount -t virtiofs -o rw,nosuid,nodev microvm {self.runtime}/shares", - f"mount -o bind {self.runtime}/shares/0 {rootfs}/workspace", - f"mount -o remount,bind,rw,nosuid,nodev {rootfs}/workspace", - f"mount -o bind {self.runtime}/shares/1 {rootfs}/opt/tools", - f"umount {rootfs}/workspace", - f"umount {self.runtime}/shares", - "exit 125", - ], - ) - - def test_agent_fails_closed_for_invalid_share_bootstrap(self): - (self.rootfs / "file").write_text("x", encoding="utf-8") - for cmdline, message in ( - ("virtfs_dir=/workspace", "incomplete"), - ("virtfs_dir=/workspace virtfs_tag=microvm", "incomplete"), - ("virtfs_dir=/workspace virtfs_tag=bad/tag virtfs_mode=rw", "tag"), - ("virtfs_dir=/workspace virtfs_tag=microvm virtfs_mode=rx", "mode"), - ("virtfs_dir=workspace virtfs_tag=microvm virtfs_mode=rw", "absolute"), - ("virtfs_dir=/ virtfs_tag=microvm virtfs_mode=rw", "canonical"), - ("virtfs_dir=/a//b virtfs_tag=microvm virtfs_mode=rw", "canonical"), - ("virtfs_dir=/a/ virtfs_tag=microvm virtfs_mode=rw", "canonical"), - ("virtfs_dir=/a/./b virtfs_tag=microvm virtfs_mode=rw", "canonical"), - ("virtfs_dir=/a/../b virtfs_tag=microvm virtfs_mode=rw", "canonical"), - ("virtfs_dir=/a=b virtfs_tag=microvm virtfs_mode=rw", "reserved char"), - ("virtfs_dir=/proc virtfs_tag=microvm virtfs_mode=rw", "reserved"), - ("virtfs_dir=/sys/fs virtfs_tag=microvm virtfs_mode=rw", "reserved"), - ("virtfs_dir=/dev/shm virtfs_tag=microvm virtfs_mode=rw", "reserved"), - ("virtfs_dir=/.nvx-agent virtfs_tag=microvm virtfs_mode=ro", "reserved"), - ("virtfs_dir=/etc virtfs_tag=microvm virtfs_mode=ro", "reserved"), - ("virtfs_dir=/file virtfs_tag=microvm virtfs_mode=ro", "not a directory"), - ( - "virtfs_dir=/workspace virtfs_tag=microvm virtfs_mode=rw " - "nvx_share=0,/workspace,rw", - "require an aggregate", - ), - ( - "virtfs_dir=/shares virtfs_tag=microvm virtfs_mode=rw " - "virtfs_aggregate=1 nvx_share=0,/workspace,rw", - "must be mounted at", - ), - (self._aggregate(), "has no children"), - (self._aggregate(",/workspace,rw"), "child is invalid"), - (self._aggregate(".,/workspace,rw"), "child is invalid"), - (self._aggregate("..,/workspace,rw"), "child is invalid"), - (self._aggregate("a/b,/workspace,rw"), "child is invalid"), - (self._aggregate("a:b,/workspace,rw"), "child is invalid"), - (self._aggregate("a*,/workspace,rw"), "child is invalid"), - (self._aggregate("0,/workspace"), "child is invalid"), - (self._aggregate("0,/workspace,rx"), "mode"), - (self._aggregate("0,workspace,rw"), "absolute"), - (self._aggregate("0,/a,b,rw"), "reserved char"), - (self._aggregate("0,/workspace,rw", "1,/proc,ro"), "reserved"), - (self._aggregate("0,/file,ro"), "not a directory"), - ): - with self.subTest(cmdline=cmdline): - result, log = self._run(cmdline) - self.assertEqual(result.returncode, 125, result.stdout) - self.assertIn("live share", result.stderr) - self.assertIn(message, result.stderr) - self.assertEqual(log, "") - - def test_agent_rejects_symbolic_link_in_share_target(self): - outside = self.root / "outside" - outside.mkdir() - try: - (self.rootfs / "workspace").symlink_to(outside, target_is_directory=True) - except OSError as error: - self.skipTest(f"symbolic links are unavailable: {error}") - result, log = self._run( - "virtfs_dir=/workspace/project virtfs_tag=microvm virtfs_mode=rw" - ) - self.assertEqual(result.returncode, 125, result.stdout) - self.assertIn("crosses a symbolic link", result.stderr) - self.assertEqual(log, "") - self.assertFalse((outside / "project").exists()) - - -@unittest.skipIf(os.name == "nt", "requires POSIX symbolic links") -class SandboxSmokeShareTests(unittest.TestCase): - SMOKE = Path(__file__).parents[1] / "guest" / "common" / "nvx-sandbox-smoke" - - def setUp(self): - shell = _posix_shell() - if shell is None: - self.skipTest("POSIX shell is unavailable") - self.shell = shell - temporary = tempfile.TemporaryDirectory() - self.addCleanup(temporary.cleanup) - self.root = Path(temporary.name) - self.share = self.root / "share" - self.links = self.share / "nvx-links" - self.links.mkdir(parents=True) - (self.share / "nvx-host-marker").write_text("host-to-guest\n", encoding="utf-8") - # The guest resolves nvx-links/outside to this path through ../.. - self.outside = self.root / "nvx-outside-marker" - source = self.SMOKE.read_text(encoding="utf-8") - self.functions = "".join( - _shell_function(source, name) - for name in ( - "check_share", - "check_share_ownership", - "check_share_denied", - "expect_eperm", - "check_share_policy", - "expect_erofs", - "check_share_symlinks", - "check_shares_isolated", - ) - ).replace("/tmp/nvx-tool", '"$scratch/nvx-tool"') - - def _run(self, call: str) -> subprocess.CompletedProcess[str]: - scratch = self.root / "scratch" - scratch.mkdir(exist_ok=True) - return subprocess.run( - [self.shell, "-s", "--", self.share.as_posix(), scratch.as_posix()], - input=f"set -eu\nshare=$1\nscratch=$2\n{self.functions}{call}\n", - text=True, - capture_output=True, - check=False, - ) - - def test_rw_share_creates_links_with_exact_targets(self): - result = self._run('check_share "$share" rw') - - self.assertEqual(result.returncode, 0, result.stderr) - self.assertIn( - f"NVX-UBUNTU-SANDBOX-SYMLINK-OK target={self.share.as_posix()}", - result.stdout, - ) - self.assertIn("NVX-UBUNTU-SANDBOX-SHARE-OK", result.stdout) - self.assertEqual( - {name: os.readlink(self.links / name) for name in os.listdir(self.links)}, - { - "nvx-tool": "../nvx-tool", - "absolute": "/nvx/absolute/target", - "outside": "../../nvx-outside-marker", - "denied": "../nvx-denied/secret", - "denied-directory": "../nvx-denied", - }, - ) - self.assertEqual( - (self.share / "nvx-guest-marker").read_text(encoding="utf-8"), - "guest-to-host\n", - ) - self.assertTrue((self.share / "nvx-guest-directory").is_dir()) - - def test_links_fail_when_they_expose_host_data(self): - (self.share / "nvx-guest-marker").write_text( - "guest-to-host\n", encoding="utf-8" - ) - self.outside.write_text("host-outside\n", encoding="utf-8") - result = self._run('check_share_symlinks "$share"') - self.assertNotEqual(result.returncode, 0, result.stdout) - self.assertNotIn("NVX-UBUNTU-SANDBOX-SYMLINK-OK", result.stdout) - - for path in self.links.iterdir(): - path.unlink() - self.outside.unlink() - denied = self.share / "nvx-denied" - denied.mkdir() - (denied / "secret").write_text("secret\n", encoding="utf-8") - result = self._run('check_share_symlinks "$share"') - self.assertNotEqual(result.returncode, 0, result.stdout) - self.assertNotIn("NVX-UBUNTU-SANDBOX-SYMLINK-OK", result.stdout) - - def test_ro_share_rejects_links(self): - geteuid = getattr(os, "geteuid", None) - if geteuid is not None and geteuid() == 0: - self.skipTest("root ignores directory write permissions") - self.share.chmod(0o555) - self.addCleanup(self.share.chmod, 0o755) - - result = self._run('check_share "$share" ro') - self.assertEqual(result.returncode, 0, result.stderr) - self.assertIn("NVX-UBUNTU-SANDBOX-SHARE-OK", result.stdout) - self.assertFalse(os.path.lexists(self.share / "nvx-guest-link")) - - self.share.chmod(0o755) - result = self._run('check_share "$share" ro') - self.assertEqual(result.returncode, 1, result.stdout) - - def test_policy_share_requires_openvmm_to_refuse_writes(self): - (self.share / "nvx-writable").mkdir() - allowed = self.share / "nvx-denied" / "nvx-allowed" - allowed.mkdir(parents=True) - (allowed / "payload").write_text("payload\n", encoding="utf-8") - - # Only OpenVMM's access policy can make the share refuse writes - # outside nvx-writable, so a plain directory fails the checks. - result = self._run('check_share "$share" policy') - self.assertEqual(result.returncode, 1, result.stdout) - self.assertNotIn("NVX-UBUNTU-SANDBOX-POLICY-OK", result.stdout) - - # A refusal other than EROFS, such as a permission check, fails too. - geteuid = getattr(os, "geteuid", None) - if geteuid is not None and geteuid() == 0: - return - (self.share / "nvx-guest-marker").unlink(missing_ok=True) - self.share.chmod(0o555) - self.addCleanup(self.share.chmod, 0o755) - result = self._run('check_share "$share" policy') - self.assertEqual(result.returncode, 1, result.stdout) - self.assertIn("unexpected share error", result.stderr) - self.assertIn("Permission denied", result.stderr) - - def test_rw_share_link_cannot_write_into_ro_share(self): - geteuid = getattr(os, "geteuid", None) - if geteuid is not None and geteuid() == 0: - self.skipTest("root ignores file write permissions") - toolcache = self.root / "toolcache" - toolcache.mkdir() - marker = toolcache / "nvx-host-marker" - marker.write_text("host-to-guest\n", encoding="utf-8") - marker.chmod(0o444) - self.addCleanup(marker.chmod, 0o644) - - call = f'check_shares_isolated "$share" "{toolcache.as_posix()}"' - result = self._run(call) - self.assertEqual(result.returncode, 0, result.stderr) - self.assertIn( - f"NVX-UBUNTU-SANDBOX-SHARES-ISOLATED-OK rw={self.share.as_posix()} " - f"ro={toolcache.as_posix()}", - result.stdout, - ) - self.assertFalse(os.path.lexists(self.share / "nvx-cross-share-link")) - self.assertEqual(marker.read_text(encoding="utf-8"), "host-to-guest\n") - - # A write that reaches the read-only share fails the check. - marker.chmod(0o644) - result = self._run(call) - self.assertEqual(result.returncode, 1, result.stdout) - self.assertNotIn("NVX-UBUNTU-SANDBOX-SHARES-ISOLATED-OK", result.stdout) - - def test_caller_share_owns_and_populates_its_own_directories(self): - if sys.platform != "linux": - self.skipTest("ownership checks use GNU stat and Linux identities") - result = self._run('check_share "$share" caller') - - self.assertEqual(result.returncode, 0, result.stderr) - share = self.share.as_posix() - self.assertIn( - f"NVX-UBUNTU-SANDBOX-OWNER-OK target={share} " - f"identity={os.getuid()}:{os.getgid()}", - result.stdout, - ) - self.assertIn(f"NVX-UBUNTU-SANDBOX-SYMLINK-OK target={share}", result.stdout) - self.assertIn( - f"NVX-UBUNTU-SANDBOX-SHARE-OK target={share} mode=caller", result.stdout - ) - links = self.share / "nvx-caller-links" - self.assertEqual( - {name: os.readlink(links / name) for name in os.listdir(links)}, - { - "nvx-tool": "../nvx-tool", - "absolute": "/nvx/absolute/target", - "outside": "../../nvx-outside-marker", - "denied": "../nvx-denied/secret", - "denied-directory": "../nvx-denied", - }, - ) - self.assertEqual(os.listdir(self.links), []) - renamed = self.share / "nvx-guest-directory" / "nested" / "renamed" - self.assertEqual(renamed.read_text(encoding="utf-8"), "nested\n") - - def test_eperm_share_requires_reads_to_fail_with_eperm_and_writes_to_fail(self): - result = self._run('check_share "$share" eperm') - self.assertNotEqual(result.returncode, 0, result.stdout) - self.assertNotIn("NVX-UBUNTU-SANDBOX-SHARE-OK", result.stdout) - - geteuid = getattr(os, "geteuid", None) - if geteuid is not None and geteuid() == 0: - self.skipTest("root ignores directory write permissions") - # Writes need only fail, so a read-only host directory stands in for - # the host's refusal. - self.share.chmod(0o555) - self.addCleanup(self.share.chmod, 0o755) - denied = "".join( - f'{command}() {{ echo "{command}: $1: Operation not permitted" >&2; ' - "return 1; }\n" - for command in ("cat", "ls") - ) - result = self._run(denied + 'check_share "$share" eperm') - self.assertEqual(result.returncode, 0, result.stderr) - self.assertIn( - f"NVX-UBUNTU-SANDBOX-SHARE-OK target={self.share.as_posix()} mode=eperm", - result.stdout, - ) - self.assertFalse((self.share / "nvx-guest-marker").exists()) - - other_error = denied.replace("Operation not permitted", "Permission denied") - result = self._run(other_error + 'check_share "$share" eperm') - self.assertNotEqual(result.returncode, 0, result.stdout) - self.assertIn("unexpected share error", result.stderr) - - self.share.chmod(0o755) - result = self._run(denied + 'check_share "$share" eperm') - self.assertNotEqual(result.returncode, 0, result.stdout) - self.assertNotIn("NVX-UBUNTU-SANDBOX-SHARE-OK", result.stdout) - - -_SANDBOX_MOUNTINFO = ( - "20 1 0:20 / / rw,relatime - overlay overlay rw," - "lowerdir=/run/nvx/layers/distro,upperdir=/run/nvx/scratch/upper," - "workdir=/run/nvx/scratch/work,metacopy=on,xino=on\n" - "21 20 0:21 / /.nvx-agent ro,nosuid,nodev,relatime - tmpfs tmpfs ro,mode=755\n" - "22 20 0:22 / /proc rw,nosuid,nodev,noexec,relatime - proc proc rw\n" - "23 20 0:23 / /sys ro,nosuid,nodev,noexec,relatime - sysfs sysfs ro\n" - "24 20 0:24 / /dev rw,nosuid,relatime - tmpfs tmpfs rw,mode=755\n" - "25 24 0:25 / /dev/pts rw,relatime - devpts devpts rw,mode=620,ptmxmode=666\n" - "26 24 0:26 / /dev/shm rw,nosuid,nodev,relatime - tmpfs tmpfs rw\n" -) -_SANDBOX_STATUS = ( - "Name:\tsh\nUid:\t65534\t65534\t65534\t65534\n" - "Gid:\t65534\t65534\t65534\t65534\nGroups:\t\nNoNewPrivs:\t1\n" - + "".join( - f"{name}:\t0000000000000000\n" - for name in ("CapInh", "CapPrm", "CapEff", "CapBnd", "CapAmb") - ) -) - - -@unittest.skipIf(os.name == "nt", "requires a POSIX shell and process tools") -class SandboxSmokeProfileTests(unittest.TestCase): - SMOKE = Path(__file__).parents[1] / "guest" / "common" / "nvx-sandbox-smoke" - - def setUp(self): - shell = _posix_shell() - if shell is None: - self.skipTest("POSIX shell is unavailable") - self.shell = shell - temporary = tempfile.TemporaryDirectory() - self.addCleanup(temporary.cleanup) - self.root = Path(temporary.name) - self.source = self.SMOKE.read_text(encoding="utf-8") - - def _fixture(self, name: str, content: str) -> str: - path = self.root / name - path.write_text(content, encoding="utf-8") - return path.as_posix() - - def _run( - self, - names: tuple[str, ...], - call: str, - replacements: dict[str, str] | None = None, - ) -> subprocess.CompletedProcess[str]: - # The globals before fail() configure the checks. - script = self.source[: self.source.index("\nfail() {\n") + 1] + "".join( - _shell_function(self.source, name) for name in ("fail", *names) - ) - for old, new in (replacements or {}).items(): - script = script.replace(old, new) - return subprocess.run( - [self.shell, "-s"], - input=f"{script}{call}\n", - text=True, - capture_output=True, - check=False, - ) - - def test_status_fields_join_their_values(self): - status = self._fixture("status", _SANDBOX_STATUS) - result = self._run( - ("status_field",), - f"status_field {status} Uid\nstatus_field {status} Groups\n" - f"status_field {status} Missing", - ) - - self.assertEqual(result.returncode, 1, result.stderr) - self.assertEqual(result.stdout, "65534 65534 65534 65534\n\n") - self.assertIn(f"{status} has no Missing field", result.stderr) - - def test_identity_and_capability_checks_read_the_process_status(self): - identity = ( - 'id() { case "$1" in -u | -g) echo 65534 ;; esac; }\nHOME=/nonexistent\n' - ) - mutations = { - "valid": ("", ""), - "saved-uid": ( - "Uid:\t65534\t65534\t65534\t65534", - "Uid:\t65534\t65534\t0\t65534", - ), - "groups": ("Groups:\t", "Groups:\t27 "), - "bounding": ( - "CapBnd:\t0000000000000000", - "CapBnd:\t000001ffffffffff", - ), - "no-new-privs": ("NoNewPrivs:\t1", "NoNewPrivs:\t0"), - } - for mutation, (old, new) in mutations.items(): - with self.subTest(mutation=mutation): - status = self._fixture("status", _SANDBOX_STATUS.replace(old, new)) - result = self._run( - ("status_field", "check_identity", "check_capabilities"), - f"{identity}check_identity\ncheck_capabilities\necho checked", - {"/proc/self/status": status}, - ) - if mutation == "valid": - self.assertEqual(result.returncode, 0, result.stderr) - self.assertEqual(result.stdout, "checked\n") - else: - self.assertEqual(result.returncode, 1, result.stdout) - self.assertIn("NVX-UBUNTU-SANDBOX-FAIL", result.stderr) - - def test_namespace_check_rejects_the_initial_namespaces(self): - def run( - links: dict[str, str], - *, - init_status: str = _SANDBOX_STATUS, - command_line: str = "console=hvc0 nvx_hostname=nvx-lifecycle", - hostname: str = "nvx-lifecycle", - ) -> subprocess.CompletedProcess[str]: - readlink = ( - 'readlink() { case "$1" in ' - + " ".join( - f"*/{name}) echo '{link}' ;;" for name, link in links.items() - ) - + " esac; }\n" - ) - return self._run( - ("status_field", "check_namespaces"), - f"{readlink}check_namespaces\necho checked", - { - "/proc/1/status": self._fixture("init-status", init_status), - "/proc/cmdline": self._fixture("cmdline", command_line + "\n"), - "/proc/sys/kernel/hostname": self._fixture( - "hostname", hostname + "\n" - ), - }, - ) - - private = { - "mnt": "mnt:[4026532200]", - "pid": "pid:[4026532201]", - "uts": "uts:[4026532202]", - } - result = run(private) - self.assertEqual(result.returncode, 0, result.stderr) - self.assertEqual(result.stdout, "checked\n") - result = run(private, command_line="console=hvc0", hostname="nvx-sandbox") - self.assertEqual(result.returncode, 0, result.stderr) - - for name, initial in ( - ("mnt", "mnt:[4026531832]"), - ("pid", "pid:[4026531836]"), - ("uts", "uts:[4026531838]"), - ): - with self.subTest(namespace=name): - result = run({**private, name: initial}) - self.assertEqual(result.returncode, 1, result.stdout) - self.assertIn(f"shares the initial {name} namespace", result.stderr) - for link in ("pid:[]", "pid:[42a]", "mnt:[42]", "pid:42"): - with self.subTest(link=link): - result = run({**private, "pid": link}) - self.assertEqual(result.returncode, 1, result.stdout) - self.assertIn("unexpected pid namespace link", result.stderr) - result = run( - private, - init_status=_SANDBOX_STATUS.replace( - "Uid:\t65534\t65534\t65534\t65534", "Uid:\t0\t0\t0\t0" - ), - ) - self.assertEqual(result.returncode, 1, result.stdout) - self.assertIn("PID 1 of the workload PID namespace runs as 0", result.stderr) - result = run(private, hostname="nvx-sandbox") - self.assertEqual(result.returncode, 1, result.stdout) - self.assertIn("expected nvx-lifecycle", result.stderr) - - def test_mount_check_requires_private_runtime_mounts_once(self): - functions = ("mount_records", "require_mount", "check_mounts") - mountinfo = self._fixture("mountinfo", _SANDBOX_MOUNTINFO) - result = self._run( - functions, - "check_mounts\necho checked", - {"/proc/self/mountinfo": mountinfo}, - ) - self.assertEqual(result.returncode, 0, result.stderr) - self.assertEqual(result.stdout, "checked\n") - - proc = "22 20 0:22 / /proc rw,nosuid,nodev,noexec,relatime - proc proc rw\n" - mutations = { - "stacked": ( - proc, - proc + proc.replace("22 20 0:22", "27 22 0:27"), - "several mounts are stacked at /proc", - ), - "propagation": ( - proc, - proc.replace("relatime - proc", "relatime shared:5 - proc"), - "the mount at /proc propagates: shared:5", - ), - "writable-sysfs": ( - "/sys ro,nosuid", - "/sys rw,nosuid", - "/sys is mounted rw,nosuid,nodev,noexec,relatime, without ro", - ), - "upper": ( - "upperdir=/run/nvx/scratch/upper", - "upperdir=/run/nvx/upper", - "upper layer is not on scratch", - ), - "lower": ( - "lowerdir=/run/nvx/layers/distro", - "lowerdir=/srv/distro", - "does not use the image layers", - ), - "missing-shm": ( - "26 24 0:26 / /dev/shm rw,nosuid,nodev,relatime - tmpfs tmpfs rw\n", - "", - "nothing is mounted at /dev/shm", - ), - "host-devices": ( - "- tmpfs tmpfs rw,mode=755\n", - "- devtmpfs devtmpfs rw,mode=755\n", - "/dev is devtmpfs, expected tmpfs", - ), - } - for mutation, (old, new, message) in mutations.items(): - with self.subTest(mutation=mutation): - self.assertIn(old, _SANDBOX_MOUNTINFO) - mountinfo = self._fixture( - "mountinfo", _SANDBOX_MOUNTINFO.replace(old, new, 1) - ) - result = self._run( - functions, "check_mounts", {"/proc/self/mountinfo": mountinfo} - ) - self.assertEqual(result.returncode, 1, result.stdout) - self.assertIn(message, result.stderr) - - def test_cgroup_check_requires_the_container_cgroup(self): - for content, returncode in ( - ("0::/container\n", 0), - ("0::/agent\n", 1), - ("12:pids:/container\n", 1), - ): - with self.subTest(content=content): - cgroup = self._fixture("cgroup", content) - result = self._run( - ("check_cgroup",), "check_cgroup", {"/proc/self/cgroup": cgroup} - ) - self.assertEqual(result.returncode, returncode, result.stderr) - - def test_limits_check_validates_its_arguments(self): - for arguments, message in ( - ("abc 8", "limits requires numeric MEMORY_MAX and PIDS_MAX"), - ("33554432 ''", "limits requires numeric MEMORY_MAX and PIDS_MAX"), - ("33554432 2", "requires PIDS_MAX of at least 3"), - ): - with self.subTest(arguments=arguments): - result = self._run( - ("start_sleepers", "check_limits"), f"check_limits {arguments}" - ) - self.assertEqual(result.returncode, 1, result.stdout) - self.assertIn(message, result.stderr) - - @unittest.skipUnless(sys.platform == "linux", "inspects the Linux process table") - def test_sleeper_subshell_records_and_reaps_its_sleepers(self): - record = self.root / "sleepers" - result = self._run( - ("start_sleepers",), - f'(start_sleepers "{record.as_posix()}" 3)\necho done', - # A distinct duration identifies this test's sleepers. - {"sleep 600": "sleep 611.25"}, - ) - - self.assertEqual(result.returncode, 0, result.stderr) - self.assertEqual(record.read_text(encoding="utf-8"), "3\n") - for process in Path("/proc").iterdir(): - if process.name.isdigit(): - try: - command = (process / "cmdline").read_bytes() - except OSError: - continue - self.assertNotEqual(command, b"sleep\x00611.25\x00") - - def test_main_body_runs_the_profile_then_limits_and_shares(self): - body = self.source[self.source.index("\ncheck_identity\n") + 1 :] - stubs = "".join( - f'{name}() {{ echo "{name} $*"; }}\n' - for name in ( - "check_identity", - "check_capabilities", - "check_namespaces", - "check_mounts", - "check_devices", - "check_cgroup", - "check_image", - "check_limits", - "check_share", - "check_shares_isolated", - ) - ) - script = ( - self.source[: self.source.index("\nfail() {\n") + 1] - + _shell_function(self.source, "fail") - + stubs - + body - ) - - def run(*arguments: str) -> subprocess.CompletedProcess[str]: - return subprocess.run( - [self.shell, "-s", "--", *arguments], - input=script, - text=True, - capture_output=True, - check=False, - ) - - result = run("limits", "33554432", "8", "/workspace", "rw", "/tools") - self.assertEqual(result.returncode, 0, result.stderr) - self.assertEqual( - result.stdout.splitlines(), - [ - "check_identity ", - "check_capabilities ", - "check_namespaces ", - "check_mounts ", - "check_devices ", - "check_cgroup ", - "check_image ", - "NVX-UBUNTU-SANDBOX-PROFILE-OK uid=65534 gid=65534", - "check_limits 33554432 8", - "check_share /workspace rw", - "check_share /tools ro", - "check_shares_isolated /workspace /tools", - "NVX-UBUNTU-SANDBOX-OK uid=65534 gid=65534", - ], - ) - result = run("limits", "33554432") - self.assertEqual(result.returncode, 1, result.stdout) - self.assertIn("limits requires MEMORY_MAX and PIDS_MAX", result.stderr) - self.assertNotIn("NVX-UBUNTU-SANDBOX-OK", result.stdout) - - -class ManagedAgentCgroupTests(unittest.TestCase): - SOURCE = Path(__file__).parents[1] / "guest" / "common" / "nvx-managed-agent.c" - - def setUp(self): - if sys.platform != "linux": - self.skipTest("the managed agent requires Linux") - compiler = shutil.which("cc") or shutil.which("gcc") - if compiler is None: - self.skipTest("C compiler is unavailable") - temporary = tempfile.TemporaryDirectory() - self.addCleanup(temporary.cleanup) - root = Path(temporary.name) - self.cgroup = root / "cgroup" - self.cgroup.mkdir() - (self.cgroup / "cgroup.procs").touch() - self.subtree_control = self.cgroup / "cgroup.subtree_control" - self.subtree_control.touch() - self.execution = self.cgroup / "nvx-exec" - self.execution.mkdir() - self.tasks = self.execution / "pids.current" - self.tasks.write_bytes(b"0\n") - (self.execution / "cgroup.kill").touch() - source = root / "cgroup-test.c" - source.write_text( - f"""#define _GNU_SOURCE -#include -#include -static pid_t harness_waitpid(pid_t pid, int *status, int options); -#define waitpid(pid, status, options) harness_waitpid(pid, status, options) -#define CGROUP_ROOT {json.dumps(str(self.cgroup))} -#define main managed_agent_main -#include {json.dumps(str(self.SOURCE))} -#undef main -#undef waitpid - -static pid_t harness_process = -1; - -/* Like the pids controller, stops charging a process once it is reaped. */ -static pid_t harness_waitpid(pid_t pid, int *status, int options) -{{ - pid_t result = waitpid(pid, status, options); - - if (result > 0 && result == harness_process) {{ - int fd = open(EXEC_CGROUP "/pids.current", O_WRONLY | O_TRUNC | O_CLOEXEC); - - if (fd < 0 || write(fd, "0\\n", 2) != 2 || close(fd) != 0) {{ - _exit(3); - }} - harness_process = -1; - }} - return result; -}} - -/* - * Calls `check` while a workload process that the cgroup charges exits, or - * once it has exited if `exited` is set, and requires `check` to reap it. - */ -static int with_exiting_process(int (*check)(void), int exited) -{{ - siginfo_t information = {{0}}; - int result; - int status; - pid_t child = fork(); - - if (child < 0) {{ - _exit(2); - }} - if (child == 0) {{ - const struct timespec delay = {{.tv_nsec = 50L * 1000L * 1000L}}; - - nanosleep(&delay, NULL); - _exit(0); - }} - harness_process = child; - if (exited && waitid(P_PID, (id_t)child, &information, WEXITED | WNOWAIT) != 0) {{ - _exit(2); - }} - result = check(); - status = errno; - if (harness_process >= 0 || waitpid(child, NULL, WNOHANG) >= 0 || errno != ECHILD) {{ - _exit(4); - }} - errno = status; - return result; -}} - -int main(int argc, char **argv) -{{ - int result; - if (argc != 2) {{ - return 2; - }} - errno = 0; - if (strcmp(argv[1], "retry") == 0) {{ - struct control_session session = {{.fd = STDOUT_FILENO}}; - struct agent_config config = {{.direct = 1}}; - struct exec_config exec_config = {{0}}; - char *command[] = {{"/bin/true", NULL}}; - result = run_exec(&session, &config, 42, 0, command, &exec_config); - return result == 0 - ? run_exec(&session, &config, 43, 0, command, &exec_config) - : result; - }} - if (strcmp(argv[1], "tasks") == 0) {{ - result = exec_cgroup_has_tasks(); - }} else if (strcmp(argv[1], "prepare") == 0) {{ - result = prepare_exec_cgroup(); - }} else if (strcmp(argv[1], "settle") == 0) {{ - result = settle_exec_cgroup(); - }} else if (strcmp(argv[1], "settle-reap") == 0) {{ - result = with_exiting_process(settle_exec_cgroup, 0); - }} else if (strcmp(argv[1], "verify-reap") == 0) {{ - result = with_exiting_process(reap_exec_cgroup, 1); - }} else {{ - return 2; - }} - printf("%d %d\\n", result, errno); - return 0; -}} -""", - encoding="utf-8", - ) - self.helper = root / "cgroup-test" - flags = [ - flag - for flag in InitramfsBuildConstants.STATIC_HELPER_CFLAGS - if flag != "-static" - ] - result = subprocess.run( - [compiler, *flags, "-o", str(self.helper), str(source)], - capture_output=True, - text=True, - timeout=30, - check=False, - ) - self.assertEqual(result.returncode, 0, result.stderr) - - def _check(self, mode: str) -> tuple[int, int]: - result = subprocess.run( - [str(self.helper), mode], - capture_output=True, - text=True, - timeout=5, - check=False, - ) - self.assertEqual(result.returncode, 0, result.stderr) - value, status = result.stdout.split() - return int(value), int(status) - - def _assert_further_exec_refused(self): - result = subprocess.run( - [str(self.helper), "retry"], - capture_output=True, - timeout=5, - check=False, - ) - self.assertEqual(result.returncode, 0, result.stderr) - frames = result.stdout - for expected_id in (42, 43): - outer = ManagedAgentStopTests.OUTER - app = ManagedAgentStopTests.APP - *_, length = outer.unpack(frames[: outer.size]) - frame = frames[outer.size : outer.size + length] - _, _, kind, _, request_id, status, payload_len = app.unpack( - frame[: app.size] - ) - payload = frame[app.size :] - self.assertEqual((kind, request_id, status), (0xFF, expected_id, 125)) - self.assertEqual(payload_len, len(payload)) - self.assertEqual(payload, b"containment-failed") - frames = frames[outer.size + length :] - self.assertEqual(frames, b"") - - def test_task_count_requires_a_canonical_decimal(self): - for contents, expected in ( - (b"0\n", 0), - (b"1\n", 1), - (b"10\n", 1), - (b"4194304\n", 1), - ): - with self.subTest(contents=contents): - self.tasks.write_bytes(contents) - self.assertEqual(self._check("tasks")[0], expected) - for contents in ( - b"", - b"\n", - b"0", - b"00\n", - b"01\n", - b"-1\n", - b"+1\n", - b" 1\n", - b"1 \n", - b"0\n0\n", - b"1\x00\n", - b"max\n", - ): - with self.subTest(contents=contents): - self.tasks.write_bytes(contents) - self.assertEqual(self._check("tasks"), (-1, errno.EPROTO)) - self.tasks.write_bytes(b"1" * 40 + b"\n") - self.assertEqual(self._check("tasks"), (-1, errno.EOVERFLOW)) - - def test_missing_and_unreadable_task_counts_are_errors(self): - self.tasks.unlink() - self.assertEqual(self._check("tasks"), (-1, errno.ENOENT)) - self._assert_further_exec_refused() - self.tasks.mkdir() - self.assertEqual(self._check("tasks"), (-1, errno.EISDIR)) - self._assert_further_exec_refused() - - def test_preparation_enables_the_pids_controller_once(self): - self.assertEqual(self._check("prepare")[0], 0) - self.assertEqual(self.subtree_control.read_bytes(), b"") - self.tasks.unlink() - self.assertEqual(self._check("prepare")[0], 0) - self.assertEqual(self.subtree_control.read_bytes(), b"+pids") - self.subtree_control.unlink() - self.assertEqual(self._check("prepare"), (-1, errno.ENOENT)) - - def test_settlement_verifies_emptiness(self): - self.assertEqual(self._check("settle")[0], 0) - self.assertEqual((self.execution / "cgroup.kill").read_bytes(), b"1") - - def test_settlement_and_verification_reap_what_the_workload_left(self): - for mode in ("settle-reap", "verify-reap"): - with self.subTest(mode=mode): - self.tasks.write_bytes(b"1\n") - self.assertEqual(self._check(mode)[0], 0) - self.assertEqual(self.tasks.read_bytes(), b"0\n") - - def test_settlement_does_not_hide_kill_or_verification_failures(self): - self.tasks.write_bytes(b"1\n") - (self.execution / "cgroup.kill").unlink() - self.assertEqual(self._check("settle"), (-1, errno.ENOENT)) - self._assert_further_exec_refused() - (self.execution / "cgroup.kill").touch() - self.tasks.unlink() - self.assertEqual(self._check("settle"), (-1, errno.ENOENT)) - self._assert_further_exec_refused() - - def test_unreaped_tasks_at_deadline_are_not_settled(self): - # cgroup.events no longer counts a killed process that awaits its - # reaper, but the pids controller still charges it. - (self.execution / "cgroup.events").write_bytes(b"populated 0\nfrozen 0\n") - self.tasks.write_bytes(b"1\n") - started = time.monotonic() - self.assertEqual(self._check("settle"), (-1, errno.ETIMEDOUT)) - self.assertGreaterEqual(time.monotonic() - started, 1.9) - self._assert_further_exec_refused() - - def test_malformed_task_count_refuses_further_exec_until_verified_empty(self): - self.tasks.write_bytes(b"max\n") - self.assertEqual(self._check("settle"), (-1, errno.EPROTO)) - self._assert_further_exec_refused() - self.tasks.write_bytes(b"0\n") - self.assertEqual(self._check("tasks")[0], 0) - self.assertEqual(self._check("settle")[0], 0) - - -class GuestExitTests(unittest.TestCase): - """nvx-exit's arguments, and its single one-byte write at port 0x604.""" - - SOURCE = Path(__file__).parents[1] / "guest" / "common" / "nvx-exit.c" - PORT = 0x604 - - def setUp(self): - if sys.platform != "linux": - self.skipTest("nvx-exit builds on Linux") - compiler = shutil.which("cc") or shutil.which("gcc") - if compiler is None: - self.skipTest("C compiler is unavailable") - temporary = tempfile.TemporaryDirectory() - self.addCleanup(temporary.cleanup) - root = Path(temporary.name) - # A regular file stands in for /dev/port, so no test writes a port. - self.device = root / "port" - flags = [ - flag - for flag in InitramfsBuildConstants.STATIC_HELPER_CFLAGS - if flag != "-static" - ] - self.binaries: dict[str, Path] = {} - for name, device in (("ok", self.device), ("missing", root / "no" / "port")): - binary = root / f"nvx-exit-{name}" - result = subprocess.run( - [ - compiler, - *flags, - f'-DNVX_EXIT_DEVICE="{device}"', - "-o", - str(binary), - str(self.SOURCE), - ], - capture_output=True, - text=True, - check=False, - ) - self.assertEqual(result.returncode, 0, result.stderr) - self.binaries[name] = binary - - def _run(self, name: str, *arguments: str) -> subprocess.CompletedProcess[bytes]: - return subprocess.run( - [str(self.binaries[name]), *arguments], - capture_output=True, - check=False, - timeout=10, - ) - - def test_writes_the_exit_code_as_one_byte_at_the_control_port(self): - cases = ( - ((), 1), - (("",), 1), - (("0",), 0), - (("37",), 37), - (("255",), 255), - (("256",), 1), - (("-1",), 1), - (("+5",), 1), - ((" 7",), 1), - (("12a",), 1), - (("99999999999999999999",), 1), - # Out of range after any digit, so the value never wraps. - (("4294967296",), 1), - (("4294967551",), 1), - (("00000000000000000255",), 255), - # Decimal: the old script's printf read a leading zero as octal. - (("010",), 10), - (("7", "9"), 7), - ) - for arguments, expected in cases: - with self.subTest(arguments=arguments): - self.device.write_bytes(b"") - result = self._run("ok", *arguments) - self.assertEqual( - (result.returncode, result.stdout, result.stderr), (0, b"", b"") - ) - self.assertEqual( - self.device.read_bytes(), bytes(self.PORT) + bytes([expected]) - ) - - def test_fails_silently_when_the_port_cannot_be_opened(self): - result = self._run("missing", "0") - self.assertEqual( - (result.returncode, result.stdout, result.stderr), (1, b"", b"") - ) - - -class ManagedAgentStopTests(unittest.TestCase): - SOURCE = Path(__file__).parents[1] / "guest" / "common" / "nvx-managed-agent.c" - OUTER = struct.Struct("<4sHBB16sQQI") - APP = struct.Struct("<4sBBHQiI") - - def setUp(self): - if sys.platform != "linux": - self.skipTest("the managed agent requires a Linux pseudo-terminal") - compiler = shutil.which("cc") or shutil.which("gcc") - if compiler is None: - self.skipTest("C compiler is unavailable") - temporary = tempfile.TemporaryDirectory() - self.addCleanup(temporary.cleanup) - self.agent = Path(temporary.name) / "nvx-managed-agent" - flags = [ - flag - for flag in InitramfsBuildConstants.STATIC_HELPER_CFLAGS - if flag != "-static" - ] - result = subprocess.run( - [compiler, *flags, "-o", str(self.agent), str(self.SOURCE)], - capture_output=True, - text=True, - check=False, - ) - self.assertEqual(result.returncode, 0, result.stderr) - - def _record( - self, record_type: int, instance: bytes, sequence: int, payload: bytes = b"" - ) -> bytes: - header = self.OUTER.pack( - b"NVXS", 1, record_type, 0, instance, 1, sequence, len(payload) - ) - return header + payload - - def _read_exact(self, fd: int, length: int, deadline: float) -> bytes: - data = b"" - while len(data) < length: - remaining = deadline - time.monotonic() - self.assertGreater(remaining, 0, "managed agent did not respond") - readable, _, _ = select.select([fd], [], [], remaining) - if readable: - chunk = os.read(fd, length - len(data)) - self.assertTrue(chunk, "managed agent closed the control tty") - data += chunk - return data - - def _read_record(self, fd: int, deadline: float) -> tuple[int, int, bytes]: - header = self._read_exact(fd, self.OUTER.size, deadline) - magic, version, record_type, flags, _, _, sequence, length = self.OUTER.unpack( - header - ) - self.assertEqual((magic, version, flags), (b"NVXS", 1, 0)) - return record_type, sequence, self._read_exact(fd, length, deadline) - - @staticmethod - def _reap(process: subprocess.Popen[bytes]) -> None: - if process.poll() is None: - process.kill() - process.wait(timeout=5) - if process.stderr is not None: - process.stderr.close() - - def _open_session( - self, rootfs: str - ) -> tuple[subprocess.Popen[bytes], int, bytes, float]: - """Starts an agent and completes the attach and reset handshake.""" - if sys.platform != "linux": - self.skipTest("the managed agent requires a Linux pseudo-terminal") - import pty - - master, slave = pty.openpty() - self.addCleanup(os.close, master) - self.addCleanup(os.close, slave) - process = subprocess.Popen( - [ - str(self.agent), - os.ttyname(slave), - rootfs, - "nvx-sandbox", - "65534", - "65534", - "nobody", - "/nonexistent", - ], - stdin=subprocess.DEVNULL, - stdout=subprocess.DEVNULL, - stderr=subprocess.PIPE, - ) - self.addCleanup(self._reap, process) - deadline = time.monotonic() + 10 - instance = bytes(range(1, 17)) - - self.assertEqual(self._read_record(master, deadline)[0], 1) - os.write(master, self._record(3, instance, 7)) - record_type, sequence, credit = self._read_record(master, deadline) - self.assertEqual((record_type, sequence, len(credit)), (4, 0, 4)) - return process, master, instance, deadline - - def _request( - self, rootfs: str, kind: int, payload: bytes = b"" - ) -> tuple[int, int, int, bytes]: - """Sends one request to a fresh agent; returns the kind, request ID, status, and payload of its answer.""" - _, master, instance, deadline = self._open_session(rootfs) - frame = self.APP.pack(b"NVXC", 1, kind, 0, 42, 0, len(payload)) + payload - os.write(master, self._record(5, instance, 8, frame)) - self.assertEqual(self._read_record(master, deadline)[0], 9) - record_type, _, answer = self._read_record(master, deadline) - self.assertEqual(record_type, 5) - _, _, answered, _, request_id, status, length = self.APP.unpack( - answer[: self.APP.size] - ) - body = answer[self.APP.size :] - self.assertEqual(length, len(body)) - return answered, request_id, status, body - - def test_sandbox_agent_returns_to_init_agent_after_stop(self): - if sys.platform != "linux": - self.skipTest("the managed agent requires a Linux pseudo-terminal") - process, master, instance, deadline = self._open_session("/run/nvx/rootfs") - stop = self.APP.pack(b"NVXC", 1, 3, 0, 42, 0, 0) - os.write(master, self._record(5, instance, 8, stop)) - self.assertEqual(self._read_record(master, deadline)[0], 9) - record_type, _, payload = self._read_record(master, deadline) - self.assertEqual(record_type, 5) - _, _, kind, _, request_id, status, length = self.APP.unpack(payload) - self.assertEqual((kind, request_id, status, length), (0x85, 42, 0, 0)) - - _, stderr = process.communicate(timeout=10) - self.assertEqual(process.returncode, 0, stderr) - - def test_agent_advertises_the_control_features_of_its_mode(self): - cancel, workload_account, exec_cgroup, environment = (1, 4, 8, 16) - exec_cwd, host_mapping_table = (32, 64) - for rootfs, expected in ( - # Only the direct agent maps host paths, gives each workload a cgroup, and - # refuses a working directory that the workload cannot enter. Bit 1, which - # announced nvx_map= kernel tokens, is retired. - ( - "-", - cancel - | workload_account - | exec_cgroup - | environment - | exec_cwd - | host_mapping_table, - ), - ("/run/nvx/rootfs", cancel | workload_account | environment), - ): - with self.subTest(rootfs=rootfs): - kind, request_id, status, body = self._request(rootfs, 5) - self.assertEqual((kind, request_id, status), (0x81, 42, 0)) - self.assertEqual(struct.unpack(" bytes: - """Encodes an extended EXEC payload that carries `cwd` after the arguments.""" - arguments = b"".join(struct.pack(" -int harness_setenv(const char *name, const char *value, int overwrite); -#define setenv(name, value, overwrite) harness_setenv(name, value, overwrite) -#define CGROUP_ROOT {json.dumps(str(cgroup))} -#define setgroups(size, list) harness_setgroups(size, list) -#define setegid(gid) harness_setegid(gid) -#define seteuid(uid) harness_seteuid(uid) -#define main managed_agent_main -#include {json.dumps(str(self.SOURCE))} -#undef main - -int harness_setenv(const char *name, const char *value, int overwrite) -{{ - if (strcmp(name, "PWD") == 0 && getenv("HARNESS_FAIL_PWD") != NULL) {{ - errno = ENOMEM; - return -1; - }} - return (setenv)(name, value, overwrite); -}} - -int harness_setgroups(size_t size, const gid_t *list) -{{ - (void)size; - (void)list; - if (getenv("HARNESS_FAIL_SETGROUPS") != NULL) {{ - errno = EPERM; - return -1; - }} - return 0; -}} - -int harness_setegid(gid_t gid) -{{ - (void)gid; - return 0; -}} - -int harness_seteuid(uid_t uid) -{{ - (void)uid; - return 0; -}} - -int main(int argc, char **argv) -{{ - if (argc >= 2 && strcmp(argv[1], "--exec-config-fd") == 0) {{ - return launch_workload(argc, argv); - }} - - struct control_session session = {{.fd = STDOUT_FILENO}}; - struct agent_config config = {{ - .direct = 1, - .uid = "65534", - .gid = "65534", - .user = "nobody", - .home = "/", - }}; - char *environment[] = {{getenv("HARNESS_ENVIRONMENT"), NULL}}; - struct exec_config exec_config = {{ - .cwd = argc == 3 ? argv[2] : NULL, - .environment = environment[0] == NULL ? NULL : environment, - .environment_count = environment[0] == NULL ? 0 : 1, - .environment_present = environment[0] != NULL, - .inherit_default_env = - environment[0] != NULL && getenv("HARNESS_LAYER_ENVIRONMENT") != NULL, - }}; - char *command[] = {{"/bin/sh", "-c", argv[1], NULL}}; - - if (argc != 2 && argc != 3) {{ - return 2; - }} - return run_exec(&session, &config, 42, 0, command, &exec_config) == 0 ? 0 : 1; -}} -""", - encoding="utf-8", - ) - flags = [ - flag - for flag in InitramfsBuildConstants.STATIC_HELPER_CFLAGS - if flag != "-static" - ] - result = subprocess.run( - [compiler, *flags, "-o", str(self.harness), str(source)], - capture_output=True, - text=True, - timeout=60, - check=False, - ) - self.assertEqual(result.returncode, 0, result.stderr) - self.environment = { - **os.environ, - "PATH": f"{tools}{os.pathsep}{os.environ.get('PATH', '/usr/bin:/bin')}", - } - for name in ( - "HARNESS_BEFORE_LAUNCH", - "HARNESS_ENVIRONMENT", - "HARNESS_FAIL_PWD", - "HARNESS_FAIL_SETGROUPS", - "HARNESS_LAYER_ENVIRONMENT", - ): - self.environment.pop(name, None) - - def _exec( - self, - script: str, - cwd: str | None = None, - environment: dict[str, str] | None = None, - ) -> list[tuple[int, int, int, bytes]]: - """Runs one workload; returns the kind, request ID, status, and payload of each frame.""" - arguments = [str(self.harness), script] - if cwd is not None: - arguments.append(cwd) - result = subprocess.run( - arguments, - capture_output=True, - timeout=30, - env=environment or self.environment, - check=False, - ) - self.assertEqual(result.returncode, 0, result.stderr) - frames: list[tuple[int, int, int, bytes]] = [] - rest = result.stdout - while rest: - _, _, record_type, _, _, _, _, length = self.OUTER.unpack( - rest[: self.OUTER.size] - ) - payload = rest[self.OUTER.size : self.OUTER.size + length] - rest = rest[self.OUTER.size + length :] - self.assertEqual(record_type, 5) - _, _, kind, _, request_id, status, payload_len = self.APP.unpack( - payload[: self.APP.size] - ) - body = payload[self.APP.size :] - self.assertEqual(payload_len, len(body)) - frames.append((kind, request_id, status, body)) - return frames - - def _completed(self, frames: list[tuple[int, int, int, bytes]]) -> bytes: - """Returns the standard output of a workload that exited with status zero.""" - self.assertEqual(frames[-1], (0x84, 42, 0, b"exit"), frames) - self.assertTrue(all(kind == 0x82 for kind, *_ in frames[:-1]), frames) - return b"".join(body for *_, body in frames[:-1]) - - def _refused( - self, frames: list[tuple[int, int, int, bytes]], cwd: str, error: int - ) -> None: - diagnostic = ( - f"nvx-managed-agent: cannot enter working directory {cwd}: " - f"{os.strerror(error)}\n" - ).encode() - self.assertEqual( - frames, - [(0x83, 42, 0, diagnostic), (0xFF, 42, error, b"cwd-failed")], - ) - - def test_workloads_start_in_the_root_directory_by_default(self): - output = self._completed(self._exec('pwd; printf "%s\\n" "$PWD"')) - self.assertEqual(output, b"/\n/\n") - - def test_workloads_start_in_the_requested_directory(self): - for name in ("a", "b"): - directory = self.root / "work" / name - directory.mkdir(parents=True) - with self.subTest(directory=directory): - output = self._completed( - self._exec('pwd; printf "%s\\n" "$PWD"', str(directory)) - ) - self.assertEqual(output, f"{directory}\n{directory}\n".encode()) - - def test_pwd_keeps_the_requested_name_of_a_canonical_path(self): - target = self.root / "target" - target.mkdir() - link = self.root / "link" - link.symlink_to(target) - output = self._completed(self._exec("pwd; /bin/pwd -P", str(link))) - self.assertEqual(output, f"{link}\n{target}\n".encode()) - # Without a canonical name, PWD names the directory itself. - output = self._completed(self._exec('printf "%s\\n" "$PWD"', f"{link}/.")) - self.assertEqual(output, f"{target}\n".encode()) - - def test_unusable_directories_refuse_the_launch(self): - marker = self.root / "ran" - regular = self.root / "file" - regular.touch() - cases = [ - (self.root / "missing", errno.ENOENT), - (regular, errno.ENOTDIR), - ] - # Root may search any directory, so only an unprivileged user observes EACCES. - if sys.platform == "linux" and os.geteuid() != 0: - locked = self.root / "locked" - locked.mkdir(mode=0) - self.addCleanup(locked.chmod, 0o755) - cases.append((locked, errno.EACCES)) - for directory, error in cases: - with self.subTest(directory=directory): - frames = self._exec(f"touch {marker}", str(directory)) - self._refused(frames, str(directory), error) - self.assertFalse(marker.exists()) - - def test_identity_failures_refuse_the_launch(self): - environment = {**self.environment, "HARNESS_FAIL_SETGROUPS": "1"} - frames = self._exec(f"touch {self.root / 'ran'}", None, environment) - self.assertEqual(frames, [(0xFF, 42, 125, b"launch-failed")]) - self.assertFalse((self.root / "ran").exists()) - - def test_workloads_start_in_the_directory_that_was_checked(self): - checked = self.root / "checked" - retarget = self.root / "retarget" - renamed = self.root / "renamed" - checked.mkdir() - retarget.mkdir() - link = self.root / "link" - link.symlink_to(checked) - # Neither a retargeted link nor a renamed directory may fail the - # launch or move the workload to a directory that was not checked. - quote = shlex.quote - for cwd, change, expected in ( - (link, f"ln -sfn {quote(str(retarget))} {quote(str(link))}", checked), - (retarget, f"mv {quote(str(retarget))} {quote(str(renamed))}", renamed), - ): - with self.subTest(cwd=cwd): - environment = {**self.environment, "HARNESS_BEFORE_LAUNCH": change} - frames = self._exec("/bin/pwd -P", str(cwd), environment) - self.assertEqual(self._completed(frames), f"{expected}\n".encode()) - - def test_pwd_failures_refuse_the_launch(self): - marker = self.root / "ran" - environment = {**self.environment, "HARNESS_FAIL_PWD": "1"} - for cwd in (None, str(self.root)): - with self.subTest(cwd=cwd): - frames = self._exec(f"touch {marker}", cwd, environment) - self.assertEqual(frames, [(0xFF, 42, 125, b"launch-failed")]) - self.assertFalse(marker.exists()) - - def test_exact_environments_control_pwd(self): - # The agent leaves PWD to a request that replaces the environment, so - # it does not try to set PWD, and cannot fail to. - environment = { - **self.environment, - "HARNESS_ENVIRONMENT": "MODE=exact", - "HARNESS_FAIL_PWD": "1", - } - output = self._completed( - self._exec( - 'printf "%s\\n" "$MODE"; /bin/pwd -P', str(self.root), environment - ) - ) - self.assertEqual(output, f"exact\n{self.root}\n".encode()) - - def test_layered_environments_keep_the_agents_pwd(self): - # Entries layered over the default environment keep the PWD that the - # agent points at the directory, which, unlike a PWD that the shell - # computes, names a link as the request did. - target = self.root / "target" - target.mkdir() - link = self.root / "link" - link.symlink_to(target) - layered = { - **self.environment, - "HARNESS_ENVIRONMENT": "MODE=layered", - "HARNESS_LAYER_ENVIRONMENT": "1", - } - output = self._completed( - self._exec('printf "%s %s\\n" "$MODE" "$PWD"', str(link), layered) - ) - self.assertEqual(output, f"layered {link}\n".encode()) - frames = self._exec("true", str(link), {**layered, "HARNESS_FAIL_PWD": "1"}) - self.assertEqual(frames, [(0xFF, 42, 125, b"launch-failed")]) - - -class AciSandboxRunnerTests(unittest.TestCase): - def _artifacts(self, root: Path) -> dict[str, Path]: - names = { - KernelBuildConstants.BINARY_NAME: b"kernel", - AlpineBuildConstants.INITRAMFS_NAME: b"initrd", - "openvmm": b"vmm", - } - paths: dict[str, Path] = {} - for name, contents in names.items(): - paths[name] = root / name - paths[name].write_bytes(contents) - return paths - - def _environment(self, root: Path, paths: dict[str, Path]) -> dict[str, str]: - def artifact(name: str) -> Path: - return root / name - - def openvmm() -> Path: - return paths["openvmm"] - - with ( - patch.object(aci_edge_sandboxes_tests, "artifact_path", artifact), - patch.object(aci_edge_sandboxes_tests, "openvmm_binary_path", openvmm), - ): - return aci_edge_sandboxes_tests.e2e_environment( - "kvm", root / "state", root / "out" - ) - - def test_cli_registers_the_lifecycle_test(self): - args = nvx.parse_args(["test-aci-edge-sandboxes", "--backend", "mshv"]) - - self.assertEqual(args.backend, "mshv") - self.assertEqual(args.cargo, "cargo") - self.assertFalse(hasattr(args, "scratch_template")) - self.assertIs( - args.handler, aci_edge_sandboxes_tests.command_test_aci_edge_sandboxes - ) - - def test_environment_resolves_repository_artifacts(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - paths = self._artifacts(root) - environment = self._environment(root, paths) - - self.assertEqual(environment["ACI_EDGE_SANDBOXES_E2E_HYPERVISOR"], "kvm") - self.assertEqual( - Path(environment["ACI_EDGE_SANDBOXES_E2E_OPENVMM"]), - paths["openvmm"].resolve(), - ) - self.assertEqual( - Path(environment["ACI_EDGE_SANDBOXES_E2E_INITRD"]), - paths[AlpineBuildConstants.INITRAMFS_NAME].resolve(), - ) - self.assertEqual( - Path(environment["ACI_EDGE_SANDBOXES_E2E_STATE_ROOT"]), - (root / "state").resolve(), - ) - self.assertFalse( - any( - name.startswith("ACI_EDGE_SANDBOXES_E2E_DISTRO") - for name in environment - ) - ) - self.assertNotIn("ACI_EDGE_SANDBOXES_E2E_SCRATCH", environment) - - def test_environment_requires_the_alpine_initramfs(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - paths = self._artifacts(root) - paths[AlpineBuildConstants.INITRAMFS_NAME].unlink() - with self.assertRaisesRegex(common.ScriptError, "Alpine initramfs"): - self._environment(root, paths) - - def test_command_runs_only_the_ignored_lifecycle_test(self): - command = aci_edge_sandboxes_tests.e2e_command("cargo") - - self.assertEqual(command[:2], ["cargo", "test"]) - self.assertEqual( - Path(command[command.index("--manifest-path") + 1]), - BuildConstants.REPO_ROOT / "aci_edge_sandboxes" / "Cargo.toml", - ) - self.assertIn("--locked", command) - self.assertEqual(command[command.index("--test") + 1], "openvmm_e2e") - self.assertEqual(command[-2:], ["--ignored", "--nocapture"]) - - def test_handler_returns_the_test_status(self): - observed: list[str] = [] - - def fake_run( - command: list[str], *, env: dict[str, str], check: bool - ) -> subprocess.CompletedProcess[bytes]: - self.assertFalse(check) - self.assertEqual(command, aci_edge_sandboxes_tests.e2e_command("cargo")) - observed.append(env["ACI_EDGE_SANDBOXES_E2E_STATE_ROOT"]) - return subprocess.CompletedProcess(command, 3) - - def fake_environment( - backend: str, state_root: Path, output_dir: Path - ) -> dict[str, str]: - self.assertEqual(backend, "whp") - return {"ACI_EDGE_SANDBOXES_E2E_STATE_ROOT": os.fspath(state_root)} - - with tempfile.TemporaryDirectory() as temporary: - args = argparse.Namespace( - backend="whp", - output_dir=Path(temporary) / "results", - cargo="cargo", - ) - with ( - patch.object(aci_edge_sandboxes_tests, "validate_openvmm_test_backend"), - patch.object( - aci_edge_sandboxes_tests, "e2e_environment", fake_environment - ), - patch.object(aci_edge_sandboxes_tests.subprocess, "run", fake_run), - ): - self.assertEqual( - aci_edge_sandboxes_tests.command_test_aci_edge_sandboxes(args), 3 - ) - self.assertTrue((Path(temporary) / "results").is_dir()) - - self.assertEqual(len(observed), 1) - self.assertEqual(Path(observed[0]).name, "state") - - def _run_leaving( - self, records: dict[str, object] | None, *, interrupt: bool = False - ) -> tuple[Path, str]: - """Runs the handler with a test command that leaves one sandbox provisioned. - - The sandbox gets `records` as extra state files; `None` leaves only the lock - directories that deprovisioning keeps. Returns the state root that the command - used and the handler's standard error. - """ - state_roots: list[Path] = [] - - def fake_environment( - backend: str, state_root: Path, output_dir: Path - ) -> dict[str, str]: - state_roots.append(state_root) - return {} - - def fake_run( - command: list[str], *, env: dict[str, str], check: bool - ) -> subprocess.CompletedProcess[bytes]: - (state_roots[0] / "filesystem" / ".locks").mkdir(parents=True) - if records is not None: - sandbox = state_roots[0] / "network" / ("a" * 32) - sandbox.mkdir(parents=True) - (sandbox / "sandbox.json").write_text("{}", encoding="utf-8") - for name, record in records.items(): - (sandbox / name).write_text(json.dumps(record), encoding="utf-8") - if interrupt: - raise KeyboardInterrupt - return subprocess.CompletedProcess(command, 0) - - with tempfile.TemporaryDirectory() as temporary: - args = argparse.Namespace( - backend="kvm", output_dir=Path(temporary) / "results", cargo="cargo" - ) - with ( - patch.object(aci_edge_sandboxes_tests, "validate_openvmm_test_backend"), - patch.object( - aci_edge_sandboxes_tests, "e2e_environment", fake_environment - ), - patch.object(aci_edge_sandboxes_tests.subprocess, "run", fake_run), - patch("sys.stderr", io.StringIO()) as stderr, - ): - if interrupt: - with self.assertRaises(KeyboardInterrupt): - aci_edge_sandboxes_tests.command_test_aci_edge_sandboxes(args) - else: - self.assertEqual( - aci_edge_sandboxes_tests.command_test_aci_edge_sandboxes(args), - 0, - ) - state_root = state_roots[0] - if state_root.parent.exists(): - self.addCleanup(shutil.rmtree, state_root.parent, True) - return state_root, stderr.getvalue() - - def test_handler_deletes_state_once_every_sandbox_is_deprovisioned(self): - state_root, stderr = self._run_leaving(None) - - self.assertFalse(state_root.parent.exists()) - self.assertEqual(stderr, "") - - def test_handler_keeps_state_of_sandboxes_left_provisioned(self): - state_root, stderr = self._run_leaving({"runtime.json": {"pid": 4242}}) - - sandbox = state_root / "network" / ("a" * 32) - self.assertTrue((sandbox / "runtime.json").is_file()) - self.assertIn(f"kept {state_root}", stderr) - self.assertIn(f"{sandbox} (OpenVMM pid 4242)", stderr) - - def test_handler_keeps_state_when_the_test_is_interrupted(self): - state_root, stderr = self._run_leaving( - {"launch.json": {"process": {"pid": 77}}}, interrupt=True - ) - - sandbox = state_root / "network" / ("a" * 32) - self.assertTrue((sandbox / "launch.json").is_file()) - self.assertIn(f"{sandbox} (OpenVMM pid 77)", stderr) - - -def _write_managed_runtime(state: Path, pid: int, start_time: int | None) -> None: - runtime: dict[str, object] = { - "format": sandbox_lifecycle.STATE_FORMAT, - "pid": pid, - "control_endpoint": os.fspath(state / sandbox_lifecycle.CONTROL_SOCKET_NAME), - } - if start_time is not None: - runtime["start_time"] = start_time - sandbox_lifecycle._write_json(state / sandbox_lifecycle.RUNTIME_NAME, runtime) - (state / sandbox_lifecycle.CAPABILITY_NAME).write_bytes(b"x" * 32) - - -def _provision_managed_sandbox(root: Path) -> Path: - layer_path = root / "distro.erofs" - scratch_path = root / "scratch.ext4" - layer_path.write_bytes(b"layer") - scratch_path.write_bytes(b"scratch") - state = root / "state" - sandbox_lifecycle.provision( - state, - sandbox.SandboxLaunch( - layers=( - sandbox.SandboxLayer( - role="distro", - path=layer_path, - uuid="11111111-1111-1111-1111-111111111111", - ), - ), - scratch=scratch_path, - ), - hypervisor="whp", - memory_mib=256, - net=None, - network_profile=None, - network_egress=None, - network_ingress=None, - network_egress_allow=(), - network_egress_deny=(), - host_loopback=None, - network_proxy=None, - host_loopback_forward=(), - cmdline="quiet", - timeout=10, - ) - return state - - -def _stdin_bound_child() -> subprocess.Popen[bytes]: - return subprocess.Popen( - [sys.executable, "-c", "import sys; sys.stdin.read()"], - stdin=subprocess.PIPE, - ) - - -def _finish_child(child: subprocess.Popen[bytes]) -> None: - if child.stdin is not None: - child.stdin.close() - child.wait(timeout=30) - - -def _exit_unreaped(child: subprocess.Popen[bytes]) -> None: - if sys.platform != "linux": - raise unittest.SkipTest("zombie processes require a Linux host") - assert child.stdin is not None - child.stdin.close() - # WNOWAIT leaves the exited child unreaped, so it stays a zombie. - os.waitid(os.P_PID, child.pid, os.WEXITED | os.WNOWAIT) - - -def _end_child(child: subprocess.Popen[bytes]) -> None: - if child.poll() is None: - child.kill() - child.wait(timeout=30) - - -def _openvmm_mock(status: int | None) -> MagicMock: - """Mocks an OpenVMM process that exits on its own with `status`. - - With None, it runs until it is terminated and then exits with status 1. - """ - process = MagicMock() - process.pid = 123 - process.stdin = io.BytesIO() - process.poll.return_value = status - - def wait(timeout: float) -> int: - if process.terminate.called: - return 1 - if status is None: - raise subprocess.TimeoutExpired("openvmm", timeout) - return status - - process.wait.side_effect = wait - return process - - -# Stands in for OpenVMM when the managed guest exits during startup. Like OpenVMM, -# it closes the control endpoint before it publishes its outcome report through a -# staging file, and a slow flush keeps that file in place for a second. It takes -# the size of the attach request and then OpenVMM's arguments. -_EXITING_OPENVMM = r""" -import json -import os -import socket -import sys -import tempfile -import time - -attach_size = int(sys.argv[1]) -arguments = sys.argv[2:] -endpoint = arguments[arguments.index("--microvm-control-console") + 1] -endpoint = endpoint.removeprefix("listen=") -report = arguments[arguments.index("--microvm-report") + 1] -sys.stdin.buffer.read() -received = 0 -if os.name == "nt": - import _winapi - - pipe = _winapi.CreateNamedPipe( - endpoint.replace("/", "\\"), - _winapi.PIPE_ACCESS_DUPLEX, - _winapi.PIPE_WAIT, - 1, - 65536, - 65536, - 0, - _winapi.NULL, - ) - try: - _winapi.ConnectNamedPipe(pipe, False) - except OSError as error: - if error.winerror != _winapi.ERROR_PIPE_CONNECTED: - raise - while received < attach_size: - data, _ = _winapi.ReadFile(pipe, attach_size - received) - received += len(data) - _winapi.CloseHandle(pipe) -else: - listener = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - listener.bind(endpoint) - listener.listen(1) - connection, _ = listener.accept() - while received < attach_size: - chunk = connection.recv(attach_size - received) - if not chunk: - sys.exit("the control client closed before it attached") - received += len(chunk) - connection.close() - listener.close() -report_document = json.dumps( - { - "schema_version": 1, - "outcome": { - "operation": "managed", - "category": "guest-exit", - "status_code": 125, - }, - "network_policy": {"status": "not-requested"}, - "teardown": {"vm_stopped": True}, - } -).encode() -staging = tempfile.NamedTemporaryFile( - dir=os.path.dirname(report), prefix=".tmp", delete=False -) -staging.write(report_document[: len(report_document) // 2]) -staging.flush() -time.sleep(1) -staging.write(report_document[len(report_document) // 2 :]) -staging.close() -os.rename(staging.name, report) -sys.exit(125) -""" - -# Stands in for the OpenVMM of a managed sandbox that runs until it is stopped. -# Like OpenVMM, it reads its capability from standard input, serves the control -# endpoint one client at a time, rejects an attach with another capability, -# acknowledges a ping, and on a stop publishes its outcome report and exits. -_SERVING_OPENVMM = r""" -import json -import os -import secrets -import socket -import struct -import sys - -OUTER = struct.Struct("<4sHBB16sQQI") -APP = struct.Struct("<4sBBHQiI") -arguments = sys.argv[1:] -endpoint = arguments[arguments.index("--microvm-control-console") + 1] -endpoint = endpoint.removeprefix("listen=") -report = arguments[arguments.index("--microvm-report") + 1] -capability = sys.stdin.buffer.read() -instance = secrets.token_bytes(16) -if os.name == "nt": - import _winapi - - def accept(): - pipe = _winapi.CreateNamedPipe( - endpoint.replace("/", "\\"), - _winapi.PIPE_ACCESS_DUPLEX, - _winapi.PIPE_WAIT, - 1, - 65536, - 65536, - 0, - _winapi.NULL, - ) - try: - _winapi.ConnectNamedPipe(pipe, False) - except OSError as error: - if error.winerror != _winapi.ERROR_PIPE_CONNECTED: - raise - return pipe - - def receive(pipe, length): - return _winapi.ReadFile(pipe, length)[0] - - def send(pipe, data): - _winapi.WriteFile(pipe, data) - - def close(pipe): - _winapi.CloseHandle(pipe) - -else: - listener = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - listener.bind(endpoint) - listener.listen(1) - - def accept(): - return listener.accept()[0] - - def receive(connection, length): - return connection.recv(length) - - def send(connection, data): - connection.sendall(data) - - def close(connection): - connection.close() - - -def read_exact(connection, length): - data = b"" - while len(data) < length: - chunk = receive(connection, length - len(data)) - if not chunk: - raise EOFError - data += chunk - return data - - -def read_record(connection): - header = OUTER.unpack(read_exact(connection, OUTER.size)) - return header[2], read_exact(connection, header[7]) if header[7] else b"" - - -def write_record(connection, kind, sequence, payload=b""): - header = OUTER.pack(b"NVXS", 1, kind, 0, instance, 1, sequence, len(payload)) - send(connection, header + payload) - - -while True: - connection = accept() - try: - kind, payload = read_record(connection) - if kind != 2 or payload != capability: - write_record(connection, 8, 0) - continue - write_record(connection, 7, 0) - sequence = 1 - while True: - request = APP.unpack(read_record(connection)[1][: APP.size]) - stopping = request[2] == 3 - if stopping: - with open(report, "w", encoding="utf-8") as output: - json.dump( - { - "schema_version": 1, - "outcome": { - "operation": "managed", - "category": "success", - "status_code": 0, - }, - "network_policy": {"status": "not-requested"}, - "teardown": {"vm_stopped": True}, - }, - output, - ) - reply = APP.pack(b"NVXC", 1, 0x85 if stopping else 0x81, 0, request[4], 0, 0) - write_record(connection, 5, sequence, reply) - sequence += 1 - if stopping: - sys.exit(0) - except (EOFError, OSError): - pass - finally: - close(connection) -""" - - -class SandboxTests(unittest.TestCase): - def test_launch_contract_orders_roles_and_builds_agent_command_line(self): - custom = sandbox.SandboxLayer.parse( - "custom,custom.erofs,22222222-2222-2222-2222-222222222222" - ) - distro = sandbox.SandboxLayer.parse( - "distro,distro.erofs,11111111-1111-1111-1111-111111111111" - ) - launch = sandbox.SandboxLaunch( - layers=(custom, distro), - scratch=Path("scratch.ext4"), - entrypoint="/bin/workload", - args=("--serve",), - hostname="example", - workload_identity=(1000, 1001), - memory_max=268435456, - pids_max=64, - ) - - self.assertEqual( - [layer.role for layer in launch.ordered_layers()], - ["distro", "custom"], - ) - self.assertEqual( - launch.kernel_command_line("quiet"), - ( - "quiet nvx_sandbox=1 " - "nvx_layer=distro,0xd0003000,11111111-1111-1111-1111-111111111111 " - "nvx_layer=custom,0xd0005000,22222222-2222-2222-2222-222222222222 " - "nvx_scratch=0xd0006000,ext4 " - "nvx_entrypoint=/bin/workload nvx_hostname=example " - "nvx_arg=--serve nvx_memory_max=268435456 nvx_pids_max=65" - ), - ) - self.assertEqual( - launch.openvmm_arguments(), - [ - "--machine", - "microvm", - "--microvm-sandbox-block", - "distro:file:distro.erofs,ro", - "--microvm-sandbox-block", - "custom:file:custom.erofs,ro", - "--microvm-sandbox-block", - "scratch:file:scratch.ext4", - "--microvm-workload-identity", - "1000:1001", - ], - ) - - def test_launch_contract_rejects_duplicates_and_reserved_tokens(self): - distro = sandbox.SandboxLayer.parse( - "distro,distro.erofs,11111111-1111-1111-1111-111111111111" - ) - with self.assertRaisesRegex(common.ScriptError, "duplicate.*distro"): - sandbox.SandboxLaunch( - layers=(distro, distro), - scratch=Path("scratch.ext4"), - ) - - launch = sandbox.SandboxLaunch( - layers=(distro,), - scratch=Path("scratch.ext4"), - ) - with self.assertRaisesRegex(common.ScriptError, "owned"): - launch.kernel_command_line("nvx_sandbox=0") - with self.assertRaisesRegex(common.ScriptError, "owned"): - launch.kernel_command_line(r"foo=bar\ nvx_memory_max=max") - with self.assertRaisesRegex(common.ScriptError, "1024-byte"): - launch.kernel_command_line("x" * sandbox.SANDBOX_COMMAND_LINE_MAX_SIZE) - with self.assertRaisesRegex(common.ScriptError, "between 1"): - sandbox.parse_workload_identity("0:0") - - def test_layer_parser_rejects_invalid_role_and_uuid(self): - with self.assertRaisesRegex(common.ScriptError, "unsupported layer role"): - sandbox.SandboxLayer.parse( - "unknown,layer.erofs,11111111-1111-1111-1111-111111111111" - ) - with self.assertRaisesRegex(common.ScriptError, "UUID is invalid"): - sandbox.SandboxLayer.parse("distro,layer.erofs,not-a-uuid") - - def test_mount_parser_defaults_to_read_only_and_accepts_rw(self): - default = sandbox.SandboxMount.parse("/workspace,host-dir") - self.assertEqual(default.guest_target, "/workspace") - self.assertEqual(default.host_path, Path("host-dir")) - self.assertEqual(default.access, "ro") - self.assertEqual(default.denied_paths, ()) - - writable = sandbox.SandboxMount.parse( - "/opt/hostedtoolcache,host-dir,rw", ("logs", "secrets") - ) - self.assertEqual(writable.access, "rw") - self.assertEqual(writable.denied_paths, ("logs", "secrets")) - self.assertEqual( - sandbox.mounts_openvmm_arguments((writable,)), - [ - "--mount", - f"/opt/hostedtoolcache,{os.fspath(Path('host-dir'))},rw", - "--mount-deny", - "logs", - "--mount-deny", - "secrets", - ], - ) - self.assertEqual( - writable.command_line_fragment(), - " virtfs_dir=/opt/hostedtoolcache virtfs_tag=microvm virtfs_mode=rw", - ) - - def test_mount_parser_rejects_malformed_specifications(self): - for value, message in ( - ("/workspace", "GUEST_TARGET,HOST_PATH"), - ("/workspace,host,rw,extra", "GUEST_TARGET,HOST_PATH"), - ("/workspace,", "host path is empty"), - ("/workspace,host,rx", "unsupported sandbox mount mode"), - ): - with self.subTest(value=value): - with self.assertRaisesRegex(common.ScriptError, message): - sandbox.SandboxMount.parse(value) - with self.assertRaisesRegex(common.ScriptError, "commas"): - sandbox.SandboxMount(guest_target="/workspace", host_path=Path("a,b")) - with self.assertRaisesRegex(common.ScriptError, "parent component"): - sandbox.SandboxMount.parse("/workspace,link/../share") - - def test_mount_owner_defaults_to_vmm_and_forwards_caller(self): - default = sandbox.SandboxMount.parse("/workspace,host-dir,rw") - self.assertEqual(default.owner, "vmm") - self.assertNotIn("--mount-owner", sandbox.mounts_openvmm_arguments((default,))) - - caller = sandbox.SandboxMount.parse( - "/workspace,host-dir,rw", ("secrets",), "caller" - ) - self.assertEqual( - sandbox.mounts_openvmm_arguments((caller,)), - [ - "--mount", - f"/workspace,{os.fspath(Path('host-dir'))},rw", - "--mount-deny", - "secrets", - "--mount-owner", - "caller", - ], - ) - self.assertEqual(caller.absolute().owner, "caller") - self.assertEqual( - caller.command_line_fragment(), default.command_line_fragment() - ) - with self.assertRaisesRegex( - common.ScriptError, "unsupported sandbox mount owner" - ): - sandbox.SandboxMount.parse("/workspace,host-dir", (), "root") - - def test_mount_access_policy_is_validated_and_forwarded(self): - mount = sandbox.SandboxMount.parse( - "/tmp/mcp-runtime,host-dir,rw", - ("mcp-logs",), - allowed_paths=("mcp-logs/payloads",), - writable_paths=("agent", "cache"), - ) - self.assertEqual( - sandbox.mounts_openvmm_arguments((mount,)), - [ - "--mount", - f"/tmp/mcp-runtime,{os.fspath(Path('host-dir'))},rw", - "--mount-deny", - "mcp-logs", - "--mount-allow", - "mcp-logs/payloads", - "--mount-write", - "agent", - "--mount-write", - "cache", - ], - ) - absolute = mount.absolute() - self.assertEqual(absolute.allowed_paths, ("mcp-logs/payloads",)) - self.assertEqual(absolute.writable_paths, ("agent", "cache")) - # OpenVMM enforces the policy on the host, so the guest's bootstrap - # tokens are those of the plain share. - self.assertEqual( - mount.command_line_fragment(), - " virtfs_dir=/tmp/mcp-runtime virtfs_tag=microvm virtfs_mode=rw", - ) - for create_mount, message in ( - ( - lambda: sandbox.SandboxMount.parse( - "/w,host,ro", writable_paths=("out",) - ), - "/w is read-only", - ), - ( - lambda: sandbox.SandboxMount.parse( - "/w,host,rw", allowed_paths=("logs/payloads",) - ), - "which has none", - ), - ( - lambda: sandbox.SandboxMount.parse( - "/w,host,rw", ("logs",), allowed_paths=("logs/a", "logs/a") - ), - "allowed paths must be unique", - ), - ( - lambda: sandbox.SandboxMount.parse("/w,host,rw", writable_paths=("",)), - "writable paths must be nonempty", - ), - ( - lambda: sandbox.SandboxMount.parse( - "/w,host,rw", - writable_paths=tuple(f"path-{index}" for index in range(129)), - ), - "at most 128 writable paths", - ), - ): - with self.subTest(message=message): - with self.assertRaisesRegex(common.ScriptError, message): - create_mount() - - def test_caller_owner_requires_linux_and_a_directory_that_root_does_not_own(self): - with patch.object(sandbox.os, "name", "nt"): - with self.assertRaisesRegex(common.ScriptError, "requires a Linux host"): - sandbox.require_mount_owner_supported("caller") - sandbox.require_mount_owner_supported("vmm") - if sys.platform == "win32": - with tempfile.TemporaryDirectory() as temporary: - mount = sandbox.SandboxMount( - guest_target="/workspace", host_path=Path(temporary), owner="caller" - ) - with self.assertRaisesRegex( - common.ScriptError, "requires a Linux host" - ): - mount.validated() - return - - with tempfile.TemporaryDirectory() as temporary: - mount = sandbox.SandboxMount( - guest_target="/workspace", host_path=Path(temporary), owner="caller" - ) - owner = Path(temporary).stat() - if owner.st_uid != 0 and owner.st_gid != 0: - self.assertIs(mount.validated(), mount) - filesystem_root = Path("/") - if filesystem_root.stat().st_uid == 0: - root_owned = sandbox.SandboxMount( - guest_target="/workspace", host_path=filesystem_root, owner="caller" - ) - with self.assertRaisesRegex(common.ScriptError, "UID 0 or GID 0"): - root_owned.validated() - sandbox.SandboxMount( - guest_target="/workspace", host_path=filesystem_root - ).validated() - - def test_mount_absolute_path_keeps_symbolic_link_components(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - (root / "link-target" / "share").mkdir(parents=True) - try: - (root / "link").symlink_to( - root / "link-target", target_is_directory=True - ) - except OSError as error: - self.skipTest(f"symbolic links are unavailable: {error}") - previous = Path.cwd() - os.chdir(root) - try: - mount = sandbox.SandboxMount.parse("/workspace,link/share,rw") - absolute = mount.absolute() - expected = Path.cwd() / "link" / "share" - finally: - os.chdir(previous) - - self.assertEqual(absolute.host_path, expected) - self.assertNotIn("link-target", absolute.host_path.parts) - self.assertEqual(absolute.absolute(), absolute) - with self.assertRaisesRegex(common.ScriptError, "unique"): - sandbox.SandboxMount.parse("/workspace,host", ("logs", "logs")) - with self.assertRaisesRegex(common.ScriptError, "nonempty"): - sandbox.SandboxMount.parse("/workspace,host", ("",)) - with self.assertRaisesRegex(common.ScriptError, "at most 128"): - sandbox.SandboxMount.parse( - "/workspace,host", tuple(f"path-{index}" for index in range(129)) - ) - - def test_mount_target_validation_rejects_unsafe_and_reserved_targets(self): - for target in ( - "", - "/", - "workspace", - "/workspace/", - "/work//space", - "/work/./space", - "/work/../space", - "/work space", - "/work=space", - "/work\\space", - "/" + "x" * 4096, - ): - with self.subTest(target=target): - with self.assertRaisesRegex(common.ScriptError, "invalid sandbox"): - sandbox.validate_mount_target(target) - for target in ( - "/proc", - "/proc/self", - "/sys", - "/sys/fs", - "/dev", - "/dev/shm", - "/.nvx-agent", - "/.nvx-agent/tools", - "/etc", - ): - with self.subTest(target=target): - with self.assertRaisesRegex(common.ScriptError, "reserved"): - sandbox.validate_mount_target(target) - for target in ("/workspace", "/procfs", "/etc/app", "/opt/hostedtoolcache"): - with self.subTest(target=target): - self.assertEqual(sandbox.validate_mount_target(target), target) - - def test_launch_contract_attaches_mount_and_reserves_bootstrap_tokens(self): - distro = sandbox.SandboxLayer.parse( - "distro,distro.erofs,11111111-1111-1111-1111-111111111111" - ) - launch = sandbox.SandboxLaunch( - layers=(distro,), - scratch=Path("scratch.ext4"), - mounts=(sandbox.SandboxMount.parse("/workspace,share,rw", ("secrets",)),), - ) - - arguments = launch.openvmm_arguments() - self.assertEqual( - arguments[arguments.index("--mount") :], - [ - "--mount", - f"/workspace,{os.fspath(Path('share'))},rw", - "--mount-deny", - "secrets", - ], - ) - self.assertNotIn("virtfs_", launch.kernel_command_line("quiet")) - for token in ("virtfs_dir=/other", "virtfs_tag=other", "virtfs_mode=rw"): - with self.subTest(token=token): - with self.assertRaisesRegex(common.ScriptError, "--mount option"): - launch.kernel_command_line(token) - - unmounted = sandbox.SandboxLaunch(layers=(distro,), scratch=Path("s.ext4")) - base = unmounted.kernel_command_line() - fragment = sandbox.mounts_command_line_fragment(launch.mounts) - self.assertEqual( - fragment, " virtfs_dir=/workspace virtfs_tag=microvm virtfs_mode=rw" - ) - fitting = "x" * ( - sandbox.SANDBOX_COMMAND_LINE_MAX_SIZE - len(base) - len(fragment) - 2 - ) - launch.kernel_command_line(fitting) - with self.assertRaisesRegex(common.ScriptError, "1024-byte"): - launch.kernel_command_line(fitting + "x") - - def test_launch_contract_attaches_several_shares_as_an_aggregate(self): - distro = sandbox.SandboxLayer.parse( - "distro,distro.erofs,11111111-1111-1111-1111-111111111111" - ) - launch = sandbox.SandboxLaunch( - layers=(distro,), - scratch=Path("scratch.ext4"), - mounts=( - sandbox.SandboxMount.parse("/workspace,work,rw", ("secrets",)), - sandbox.SandboxMount.parse( - "/opt/hostedtoolcache,tools,ro", ("credentials",) - ), - sandbox.SandboxMount.parse("/srv/cache,cache,ro"), - ), - ) - - arguments = launch.openvmm_arguments() - workspace, toolcache, cache = ( - sandbox.aggregate_child_name(index, mount.guest_target) - for index, mount in enumerate(launch.mounts) - ) - # Several shares are the children of one aggregate, and OpenVMM - # attributes each denied path to the child that contains it, so - # relative paths are joined to their share. - self.assertNotIn("--mount", arguments) - self.assertEqual( - arguments[arguments.index("--mount-aggregate") :], - [ - "--mount-aggregate", - "/run/nvx/shares", - "--mount-child", - f"{workspace},{os.fspath(Path('work'))},rw", - "--mount-child", - f"{toolcache},{os.fspath(Path('tools'))},ro", - "--mount-child", - f"{cache},{os.fspath(Path('cache'))},ro", - "--mount-deny", - os.fspath(Path.cwd() / "work" / "secrets"), - "--mount-deny", - os.fspath(Path.cwd() / "tools" / "credentials"), - ], - ) - fragment = sandbox.mounts_command_line_fragment(launch.mounts) - self.assertEqual( - fragment, - " virtfs_dir=/run/nvx/shares virtfs_tag=microvm virtfs_mode=rw" - " virtfs_aggregate=1", - ) - # The guest binds each child at its target, in order. - command_line = launch.kernel_command_line() - self.assertNotIn("virtfs_", command_line) - self.assertTrue( - command_line.endswith( - f" nvx_share={workspace},/workspace,rw" - f" nvx_share={toolcache},/opt/hostedtoolcache,ro" - f" nvx_share={cache},/srv/cache,ro" - ), - command_line, - ) - self.assertEqual( - sandbox.mounts_command_line_fragment(launch.mounts[1:]), - " virtfs_dir=/run/nvx/shares virtfs_tag=microvm virtfs_mode=ro" - " virtfs_aggregate=1", - ) - with self.assertRaisesRegex(common.ScriptError, "owned by the sandbox"): - launch.kernel_command_line("nvx_share=0,/etc,rw") - - # Every share's tokens count toward the x86 budget. - base = sandbox.SandboxLaunch( - layers=(distro,), scratch=Path("s.ext4") - ).kernel_command_line() - shares = len(command_line) - len(base) - fitting = "x" * ( - sandbox.SANDBOX_COMMAND_LINE_MAX_SIZE - - len(base) - - shares - - len(fragment) - - 2 - ) - launch.kernel_command_line(fitting) - with self.assertRaisesRegex(common.ScriptError, "1024-byte"): - launch.kernel_command_line(fitting + "x") - - caller = tuple( - sandbox.SandboxMount( - guest_target=target, host_path=Path(path), owner="caller" - ) - for target, path in (("/workspace", "work"), ("/tools", "tools")) - ) - arguments = sandbox.mounts_openvmm_arguments(caller) - self.assertEqual(arguments.count("--mount-owner"), 1) - self.assertEqual(arguments[-2:], ["--mount-owner", "caller"]) - - def test_aggregate_child_names_pin_their_targets(self): - name = sandbox.aggregate_child_name(0, "/workspace") - self.assertEqual(name, f"0-{hashlib.sha256(b'/workspace').hexdigest()[:32]}") - # Equal targets give equal names, so a restore names its children as - # the boot did, and another target or position gives another name. - self.assertEqual(sandbox.aggregate_child_name(0, "/workspace"), name) - self.assertNotEqual(sandbox.aggregate_child_name(0, "/workspace2"), name) - self.assertNotEqual(sandbox.aggregate_child_name(1, "/workspace"), name) - # Every name follows OpenVMM's rule for the name of a child. - for index, target in ((0, "/w"), (999, "/" + "x" * 1000)): - self.assertRegex( - sandbox.aggregate_child_name(index, target), - r"\A[0-9]+-[0-9a-f]{32}\Z", - ) - self.assertLessEqual(len(sandbox.aggregate_child_name(999, "/w")), 64) - - def test_launch_bounds_shares_by_the_kernel_command_line(self): - distro = sandbox.SandboxLayer.parse( - "distro,distro.erofs,11111111-1111-1111-1111-111111111111" - ) - - def launch(count: int) -> sandbox.SandboxLaunch: - return sandbox.SandboxLaunch( - layers=(distro,), - scratch=Path("s.ext4"), - mounts=tuple( - sandbox.SandboxMount.parse(f"/share/{index:02},host{index}") - for index in range(count) - ), - ) - - # No fixed number of shares is the limit, only the command line. - launch(10).kernel_command_line() - with self.assertRaisesRegex(common.ScriptError, "fewer --mount shares"): - launch(20).kernel_command_line() - - def test_launch_rejects_invalid_share_sets(self): - distro = sandbox.SandboxLayer.parse( - "distro,distro.erofs,11111111-1111-1111-1111-111111111111" - ) - - def mount(value: str, owner: str = "vmm") -> sandbox.SandboxMount: - return sandbox.SandboxMount.parse(value, (), owner) - - for mounts, message in ( - ((mount("/workspace,a,rw"), mount("/workspace,b")), "overlap"), - ((mount("/workspace,a,rw"), mount("/workspace/cache,b")), "overlap"), - ((mount("/opt/tools/node,a"), mount("/opt,b,rw")), "overlap"), - ( - (mount("/workspace,a,rw"), mount("/tools,b", "caller")), - "same --mount-owner", - ), - ): - with self.subTest(mounts=[item.guest_target for item in mounts]): - with self.assertRaisesRegex(common.ScriptError, message): - sandbox.SandboxLaunch( - layers=(distro,), scratch=Path("s.ext4"), mounts=mounts - ) - # A shared prefix that is not a path component is not an overlap. - sandbox.SandboxLaunch( - layers=(distro,), - scratch=Path("s.ext4"), - mounts=(mount("/work,a,rw"), mount("/workspace,b")), - ) - - def test_launch_validation_rejects_overlapping_or_foreign_host_paths(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - layer = root / "distro.erofs" - scratch = root / "scratch.ext4" - work = root / "work" - tools = root / "tools" - layer.write_bytes(b"layer") - scratch.write_bytes(b"scratch") - (work / "nested").mkdir(parents=True) - (tools / "credentials").mkdir(parents=True) - - def launch(*mounts: sandbox.SandboxMount) -> sandbox.SandboxLaunch: - return sandbox.SandboxLaunch( - layers=( - sandbox.SandboxLayer( - role="distro", - path=layer, - uuid="11111111-1111-1111-1111-111111111111", - ), - ), - scratch=scratch, - mounts=mounts, - ) - - workspace = sandbox.SandboxMount.parse(f"/workspace,{work},rw") - for host_path in (work / "nested", work, root): - with self.subTest(host_path=host_path): - with self.assertRaisesRegex(common.ScriptError, "overlap"): - launch( - workspace, - sandbox.SandboxMount.parse( - f"/opt/hostedtoolcache,{host_path}" - ), - ).validated() - - # Each denied path must be inside its own share. - foreign = sandbox.SandboxMount.parse( - f"/workspace,{work},rw", (os.fspath(tools / "credentials"),) - ) - toolcache = sandbox.SandboxMount.parse(f"/opt/hostedtoolcache,{tools}") - with self.assertRaisesRegex(common.ScriptError, "not inside"): - launch(foreign, toolcache).validated() - # A denied path may name a share's root, which OpenVMM then hides - # behind its allowed paths, but no allowed or writable path may. - launch( - sandbox.SandboxMount.parse( - f"/workspace,{work},rw", - (".",), - allowed_paths=(os.fspath(work / "nested"),), - ), - toolcache, - ).validated() - for allowed_paths, writable_paths in (((".",), ()), ((), (".",))): - with self.subTest(allowed=allowed_paths, writable=writable_paths): - with self.assertRaisesRegex(common.ScriptError, "not inside"): - launch( - sandbox.SandboxMount.parse( - f"/workspace,{work},rw", - ("nested",), - allowed_paths=allowed_paths, - writable_paths=writable_paths, - ), - toolcache, - ).validated() - - validated = launch( - sandbox.SandboxMount.parse(f"/workspace,{work},rw", ("nested",)), - sandbox.SandboxMount.parse( - f"/opt/hostedtoolcache,{tools}", - (os.fspath(tools / "credentials"),), - ), - ).validated() - self.assertEqual(len(validated.mounts), 2) - - # Resolving a path doesn't see through a bind mount, so one directory - # reached through two paths, or a share inside a bind mount of the - # other share's directory, is rejected by its file identities. - sandbox.validate_mount_host_paths((workspace, toolcache)) - with patch.object(sandbox, "_path_identity", return_value=(1, 1)): - with self.assertRaisesRegex(common.ScriptError, "same directory"): - sandbox.validate_mount_host_paths((workspace, toolcache)) - alias = root / "alias" - (alias / "cache").mkdir(parents=True) - - def identity(path: Path) -> tuple[int, int]: - # `alias` is a bind mount of the workspace's directory. - if path in (work.resolve(), alias.resolve()): - return 1, 1 - return 0, hash(path) - - cache = sandbox.SandboxMount.parse( - f"/opt/hostedtoolcache,{alias / 'cache'}" - ) - with patch.object(sandbox, "_path_identity", side_effect=identity): - sandbox.validate_mount_host_paths((workspace, toolcache)) - for mounts in ((workspace, cache), (cache, workspace)): - with self.assertRaisesRegex(common.ScriptError, "inside the other"): - sandbox.validate_mount_host_paths(mounts) - - def test_launch_validation_requires_plain_mount_directory(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - layer = root / "distro.erofs" - scratch = root / "scratch.ext4" - share = root / "share" - layer.write_bytes(b"layer") - scratch.write_bytes(b"scratch") - - def launch(host_path: Path) -> sandbox.SandboxLaunch: - return sandbox.SandboxLaunch( - layers=( - sandbox.SandboxLayer( - role="distro", - path=layer, - uuid="11111111-1111-1111-1111-111111111111", - ), - ), - scratch=scratch, - mounts=( - sandbox.SandboxMount( - guest_target="/workspace", host_path=host_path - ), - ), - ) - - with self.assertRaisesRegex(common.ScriptError, "plain directory"): - launch(share).validated() - share.write_bytes(b"file") - with self.assertRaisesRegex(common.ScriptError, "plain directory"): - launch(share).validated() - share.unlink() - share.mkdir() - validated = launch(share).validated() - self.assertEqual(validated.mounts[0].host_path, share) - - def test_sandbox_command_forwards_mount_to_openvmm(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - layer = root / "distro.erofs" - scratch = root / "scratch.ext4" - share = root / "share" - layer.write_bytes(b"distro") - scratch.write_bytes(b"scratch") - share.mkdir() - args = nvx.parse_args( - [ - "sandbox", - "--layer", - f"distro,{layer},11111111-1111-1111-1111-111111111111", - "--scratch", - str(scratch), - "--mount", - f"/workspace,{share},rw", - "--mount-deny", - "secrets", - "--dry-run", - ] - ) - - def require(path: Path, _description: str) -> Path: - return path - - with ( - patch.object(nvx, "require_file", side_effect=require), - patch.object( - nvx, "_format_command", return_value="formatted" - ) as format_command, - ): - nvx.command_sandbox(args) - - command = format_command.call_args.args[0] - self.assertEqual( - command[command.index("--mount") + 1], f"/workspace,{share},rw" - ) - self.assertEqual(command[command.index("--mount-deny") + 1], "secrets") - self.assertNotIn("--mount-owner", command) - - def test_sandbox_command_shares_a_file(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - layer = root / "distro.erofs" - scratch = root / "scratch.ext4" - settings = root / "settings.json" - layer.write_bytes(b"distro") - scratch.write_bytes(b"scratch") - settings.write_bytes(b"{}") - - def sandbox_args(*mount: str) -> argparse.Namespace: - return nvx.parse_args( - [ - "sandbox", - "--layer", - f"distro,{layer},11111111-1111-1111-1111-111111111111", - "--scratch", - str(scratch), - *mount, - "--dry-run", - ] - ) - - def require(path: Path, _description: str) -> Path: - return path - - with ( - patch.object(nvx, "require_file", side_effect=require), - patch.object( - nvx, "_format_command", return_value="formatted" - ) as format_command, - ): - nvx.command_sandbox( - sandbox_args("--mount", f"/etc/app/settings.json,{settings},ro") - ) - command = format_command.call_args.args[0] - name = sandbox.aggregate_child_name(0, "/etc/app/settings.json") - # The regular file is a child of its own, without its directory, - # which OpenVMM must find to be a regular file. - self.assertNotIn("--mount", command) - self.assertEqual( - command[command.index("--mount-child") + 1], - f"{name},{settings},ro,file", - ) - self.assertIn( - f"nvx_share={name},/etc/app/settings.json,ro,file", - command[command.index("--cmdline") + 1], - ) - with self.assertRaisesRegex(common.ScriptError, "shares a file"): - nvx.command_sandbox( - sandbox_args( - "--mount", - f"/etc/app/settings.json,{settings},rw", - "--mount-write", - str(settings), - ) - ) - - def test_run_attaches_a_file_as_an_aggregate_child(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - settings = root / "settings.json" - settings.write_bytes(b"{}") - work = root / "work" - work.mkdir() - with ( - patch.object(nvx, "require_file", return_value=Path("artifact")), - patch.object( - nvx, "_format_command", return_value="formatted" - ) as format_command, - ): - nvx.command_run( - nvx.parse_args( - [ - "run", - "--mount", - f"/config/settings.json,{settings},ro", - "--dry-run", - ] - ) - ) - command = format_command.call_args.args[0] - name = sandbox.aggregate_child_name(0, "/config/settings.json") - # Even alone, a file is the child of an aggregate. - self.assertNotIn("--mount", command) - self.assertEqual( - command[command.index("--mount-aggregate") + 1], "/run/nvx/shares" - ) - self.assertEqual( - command[command.index("--mount-child") + 1], - f"{name},{settings},ro,file", - ) - self.assertEqual( - command[command.index("--cmdline") + 1], - f"nvx_share={name},/config/settings.json,ro,file", - ) - - # A directory and a file share the aggregate, and only the - # file's token is marked. - nvx.command_run( - nvx.parse_args( - [ - "run", - "--mount", - f"/work,{work},rw", - "--mount", - f"/config/settings.json,{settings},ro", - "--dry-run", - ] - ) - ) - command = format_command.call_args.args[0] - work_name, settings_name = ( - sandbox.aggregate_child_name(index, target) - for index, target in enumerate(("/work", "/config/settings.json")) - ) - self.assertEqual( - command[command.index("--cmdline") + 1], - f"nvx_share={work_name},/work,rw " - f"nvx_share={settings_name},/config/settings.json,ro,file", - ) - - for arguments, message in ( - (["--mount-deny", "secrets"], "absolute host path"), - (["--cmdline", "x" * 1024], "1024-byte"), - ): - with self.subTest(arguments=arguments): - invalid = nvx.parse_args( - [ - "run", - "--mount", - f"/config/settings.json,{settings},ro", - *arguments, - "--dry-run", - ] - ) - with self.assertRaisesRegex(common.ScriptError, message): - nvx.command_run(invalid) - invalid = nvx.parse_args( - ["run", f"--mount=/a b,{settings}", "--dry-run"] - ) - with self.assertRaisesRegex( - common.ScriptError, "invalid --mount target" - ): - nvx.command_run(invalid) - - def test_sandbox_command_forwards_mount_owner(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - layer = root / "distro.erofs" - scratch = root / "scratch.ext4" - share = root / "share" - layer.write_bytes(b"distro") - scratch.write_bytes(b"scratch") - share.mkdir() - args = nvx.parse_args( - [ - "sandbox", - "--layer", - f"distro,{layer},11111111-1111-1111-1111-111111111111", - "--scratch", - str(scratch), - "--mount", - f"/workspace,{share},rw", - "--mount-owner", - "caller", - "--dry-run", - ] - ) - - def require(path: Path, _description: str) -> Path: - return path - - with ( - patch.object(nvx, "require_file", side_effect=require), - patch.object( - nvx, "_format_command", return_value="formatted" - ) as format_command, - ): - if sys.platform == "win32": - with self.assertRaisesRegex(common.ScriptError, "Linux host"): - nvx.command_sandbox(args) - format_command.assert_not_called() - return - if share.stat().st_uid == 0: - self.skipTest("caller ownership refuses a root-owned share") - nvx.command_sandbox(args) - - command = format_command.call_args.args[0] - self.assertEqual(command[command.index("--mount-owner") + 1], "caller") - self.assertLess(command.index("--mount"), command.index("--mount-owner")) - - def test_sandbox_command_rejects_misplaced_mount_options(self): - deny_only = nvx.parse_args( - [ - "sandbox", - "--layer", - "distro,distro.erofs,11111111-1111-1111-1111-111111111111", - "--scratch", - "scratch.ext4", - "--mount-deny", - "secrets", - ] - ) - with self.assertRaisesRegex(common.ScriptError, "requires --mount"): - nvx.command_sandbox(deny_only) - owner_only = nvx.parse_args( - [ - "sandbox", - "--layer", - "distro,distro.erofs,11111111-1111-1111-1111-111111111111", - "--scratch", - "scratch.ext4", - "--mount-owner", - "caller", - ] - ) - with self.assertRaisesRegex(common.ScriptError, "--mount-owner requires"): - nvx.command_sandbox(owner_only) - for operation in ("start", "exec", "stop", "deprovision"): - with self.subTest(operation=operation): - args = nvx.parse_args( - [ - "sandbox", - operation, - "--state-dir", - "state", - "--mount", - "/workspace,share,rw", - ] - ) - with self.assertRaisesRegex(common.ScriptError, "only valid"): - nvx.command_sandbox(args) - - def test_sandbox_command_attributes_policy_paths_to_the_preceding_mount(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - layer = root / "distro.erofs" - scratch = root / "scratch.ext4" - work = root / "work" - tools = root / "tools" - layer.write_bytes(b"distro") - scratch.write_bytes(b"scratch") - work.mkdir() - tools.mkdir() - common_args = [ - "sandbox", - "--layer", - f"distro,{layer},11111111-1111-1111-1111-111111111111", - "--scratch", - str(scratch), - ] - args = nvx.parse_args( - [ - *common_args, - "--mount", - f"/workspace,{work},rw", - "--mount-deny", - "logs", - "--mount-allow", - "logs/payloads", - "--mount-write", - "out", - "--mount", - f"/opt/hostedtoolcache,{tools},ro", - "--mount-deny", - "credentials", - "--dry-run", - ] - ) - workspace, toolcache = nvx._sandbox_mounts(args) - self.assertEqual( - ( - workspace.denied_paths, - workspace.allowed_paths, - workspace.writable_paths, - ), - (("logs",), ("logs/payloads",), ("out",)), - ) - self.assertEqual( - ( - toolcache.denied_paths, - toolcache.allowed_paths, - toolcache.writable_paths, - ), - (("credentials",), (), ()), - ) - - def require(path: Path, _description: str) -> Path: - return path - - with ( - patch.object(nvx, "require_file", side_effect=require), - patch.object( - nvx, "_format_command", return_value="formatted" - ) as format_command, - ): - nvx.command_sandbox(args) - - command = format_command.call_args.args[0] - - def values(option: str) -> list[str]: - return [ - command[index + 1] - for index, value in enumerate(command) - if value == option - ] - - # With several shares, OpenVMM attributes absolute paths. - self.assertEqual( - values("--mount-allow"), [os.fspath(work / "logs" / "payloads")] - ) - self.assertEqual(values("--mount-write"), [os.fspath(work / "out")]) - - for arguments, message in ( - (["--mount-write", "out"], "--mount-write requires --mount"), - # A --mount-write after the read-only share would narrow it. - ( - [ - "--mount", - f"/workspace,{work},rw", - "--mount", - f"/opt/hostedtoolcache,{tools},ro", - "--mount-write", - "cache", - ], - "is read-only", - ), - ( - [ - "--mount-allow", - "logs", - "--mount", - f"/workspace,{work},rw", - "--mount", - f"/opt/hostedtoolcache,{tools}", - ], - "each --mount-allow must follow the --mount", - ), - # A path in one share cannot be attributed to the other. - ( - [ - "--mount", - f"/workspace,{work},rw", - "--mount-write", - os.fspath(tools / "cache"), - "--mount", - f"/opt/hostedtoolcache,{tools}", - ], - "--mount-write .* is not inside the host directory", - ), - # Only a denied path may name a share's root. - ( - [ - "--mount", - f"/workspace,{work},rw", - "--mount-deny", - "logs", - "--mount-allow", - os.fspath(work), - "--mount", - f"/opt/hostedtoolcache,{tools}", - ], - "--mount-allow .* is not inside the host directory", - ), - ( - [ - "--mount", - f"/workspace,{work},rw", - "--mount-write", - os.fspath(work), - "--mount", - f"/opt/hostedtoolcache,{tools}", - ], - "--mount-write .* is not inside the host directory", - ), - ): - with self.subTest(message=message): - with self.assertRaisesRegex(common.ScriptError, message): - nvx.command_sandbox(nvx.parse_args([*common_args, *arguments])) - - def test_sandbox_command_hides_a_share_root_behind_its_allowed_paths(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - layer = root / "distro.erofs" - scratch = root / "scratch.ext4" - work = root / "work" - tools = root / "tools" - layer.write_bytes(b"distro") - scratch.write_bytes(b"scratch") - (work / "src").mkdir(parents=True) - tools.mkdir() - args = nvx.parse_args( - [ - "sandbox", - "--layer", - f"distro,{layer},11111111-1111-1111-1111-111111111111", - "--scratch", - str(scratch), - "--mount", - f"/workspace,{work},rw", - "--mount-deny", - os.fspath(work), - "--mount-allow", - os.fspath(work / "src"), - "--mount", - f"/opt/hostedtoolcache,{tools}", - "--dry-run", - ] - ) - - def require(path: Path, _description: str) -> Path: - return path - - with ( - patch.object(nvx, "require_file", side_effect=require), - patch.object( - nvx, "_format_command", return_value="formatted" - ) as format_command, - ): - nvx.command_sandbox(args) - - command = format_command.call_args.args[0] - - def values(option: str) -> list[str]: - return [ - command[index + 1] - for index, value in enumerate(command) - if value == option - ] - - # OpenVMM hides the workspace's root behind its one allowed path. - workspace = sandbox.aggregate_child_name(0, "/workspace") - toolcache = sandbox.aggregate_child_name(1, "/opt/hostedtoolcache") - self.assertEqual( - values("--mount-child"), - [f"{workspace},{work},rw", f"{toolcache},{tools},ro"], - ) - self.assertEqual(values("--mount-deny"), [os.fspath(work)]) - self.assertEqual(values("--mount-allow"), [os.fspath(work / "src")]) - - def test_sandbox_command_attributes_mount_deny_to_the_preceding_mount(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - layer = root / "distro.erofs" - scratch = root / "scratch.ext4" - work = root / "work" - tools = root / "tools" - layer.write_bytes(b"distro") - scratch.write_bytes(b"scratch") - work.mkdir() - tools.mkdir() - common_args = [ - "sandbox", - "--layer", - f"distro,{layer},11111111-1111-1111-1111-111111111111", - "--scratch", - str(scratch), - ] - args = nvx.parse_args( - [ - *common_args, - "--mount", - f"/workspace,{work},rw", - "--mount-deny", - "secrets", - "--mount", - f"/opt/hostedtoolcache,{tools},ro", - "--mount-deny", - "credentials", - "--mount-deny", - "logs", - "--dry-run", - ] - ) - mounts = nvx._sandbox_mounts(args) - self.assertEqual( - [(mount.guest_target, mount.denied_paths) for mount in mounts], - [ - ("/workspace", ("secrets",)), - ("/opt/hostedtoolcache", ("credentials", "logs")), - ], - ) - - def require(path: Path, _description: str) -> Path: - return path - - with ( - patch.object(nvx, "require_file", side_effect=require), - patch.object( - nvx, "_format_command", return_value="formatted" - ) as format_command, - ): - nvx.command_sandbox(args) - - command = format_command.call_args.args[0] - workspace = sandbox.aggregate_child_name(0, "/workspace") - toolcache = sandbox.aggregate_child_name(1, "/opt/hostedtoolcache") - self.assertNotIn("--mount", command) - self.assertEqual( - [ - command[index + 1] - for index, value in enumerate(command) - if value in ("--mount-aggregate", "--mount-child") - ], - [ - "/run/nvx/shares", - f"{workspace},{work},rw", - f"{toolcache},{tools},ro", - ], - ) - self.assertEqual( - [ - command[index + 1] - for index, value in enumerate(command) - if value == "--mount-deny" - ], - [ - os.fspath(work / "secrets"), - os.fspath(tools / "credentials"), - os.fspath(tools / "logs"), - ], - ) - cmdline = command[command.index("--cmdline") + 1] - self.assertIn( - f"nvx_share={workspace},/workspace,rw " - f"nvx_share={toolcache},/opt/hostedtoolcache,ro", - cmdline, - ) - - # With one share, a --mount-deny may come first; with several, it - # must follow the share whose directory it hides. - single = nvx.parse_args( - [*common_args, "--mount-deny", "secrets", "--mount", f"/w,{work},rw"] - ) - self.assertEqual(nvx._sandbox_mounts(single)[0].denied_paths, ("secrets",)) - leading = nvx.parse_args( - [ - *common_args, - "--mount-deny", - "secrets", - "--mount", - f"/workspace,{work},rw", - "--mount", - f"/opt/hostedtoolcache,{tools}", - ] - ) - with self.assertRaisesRegex(common.ScriptError, "must follow the --mount"): - nvx.command_sandbox(leading) - # More shares are children of the same aggregate. - (root / "cache").mkdir() - three = nvx.parse_args( - [ - *common_args, - "--mount", - f"/a,{work}", - "--mount", - f"/b,{tools}", - "--mount", - f"/c,{root / 'cache'}", - "--dry-run", - ] - ) - with ( - patch.object(nvx, "require_file", side_effect=require), - patch.object( - nvx, "_format_command", return_value="formatted" - ) as format_command, - ): - nvx.command_sandbox(three) - command = format_command.call_args.args[0] - self.assertEqual(command.count("--mount-child"), 3) - - def test_run_attaches_several_mounts_as_an_aggregate(self): - args = nvx.parse_args( - [ - "run", - "--mount", - "/workspace,work,rw", - "--mount", - "/opt/hostedtoolcache,tools", - "--mount", - "/srv/cache,cache,ro", - "--cmdline", - "quiet", - "--dry-run", - ] - ) - with ( - patch.object(nvx, "require_file", return_value=Path("artifact")), - patch.object( - nvx, "_format_command", return_value="formatted" - ) as format_command, - ): - nvx.command_run(args) - command = format_command.call_args.args[0] - workspace, toolcache, cache = ( - sandbox.aggregate_child_name(index, target) - for index, target in enumerate( - ("/workspace", "/opt/hostedtoolcache", "/srv/cache") - ) - ) - booted = [ - "/run/nvx/shares", - f"{workspace},work,rw", - f"{toolcache},tools,ro", - f"{cache},cache,ro", - ] - self.assertNotIn("--mount", command) - self.assertEqual( - [ - command[index + 1] - for index, value in enumerate(command) - if value in ("--mount-aggregate", "--mount-child") - ], - booted, - ) - self.assertEqual( - command[command.index("--cmdline") + 1], - f"quiet nvx_share={workspace},/workspace,rw " - f"nvx_share={toolcache},/opt/hostedtoolcache,ro " - f"nvx_share={cache},/srv/cache,ro", - ) - - # One share stays a plain --mount without bind tokens. - single = nvx.parse_args( - ["run", "--mount", "/workspace,work,rw", "--dry-run"] - ) - nvx.command_run(single) - command = format_command.call_args.args[0] - self.assertEqual( - command[command.index("--mount") :], ["--mount", "/workspace,work,rw"] - ) - self.assertNotIn("--cmdline", command) - - # A restored guest keeps its binds, so only the devices are named, - # with the children's names of the boot: they pin the targets that - # the guest bound, and another target names another child. - restored = nvx.parse_args( - [ - "run", - "--restore-snapshot", - "snapshot", - "--mount", - "/workspace,work,rw", - "--mount", - "/opt/hostedtoolcache,tools", - "--mount", - "/srv/cache,cache,ro", - "--dry-run", - ] - ) - nvx.command_run(restored) - command = format_command.call_args.args[0] - self.assertEqual( - [ - command[index + 1] - for index, value in enumerate(command) - if value in ("--mount-aggregate", "--mount-child") - ], - booted, - ) - self.assertNotIn("--cmdline", command) - retargeted = nvx.parse_args( - [ - "run", - "--restore-snapshot", - "snapshot", - "--mount", - "/workspace,work,rw", - "--mount", - "/opt/tools,tools", - "--mount", - "/srv/cache,cache,ro", - "--dry-run", - ] - ) - nvx.command_run(retargeted) - command = format_command.call_args.args[0] - children = [ - command[index + 1] - for index, value in enumerate(command) - if value == "--mount-child" - ] - self.assertEqual(children[0::2], booted[1::2]) - self.assertNotEqual(children[1], booted[2]) - self.assertEqual(children[1].split(",", 1)[1], "tools,ro") - - for mounts, message in ( - (["/a,a,rx"], "unsupported --mount mode"), - (["/workspace,a", "/workspace/cache,b"], "overlap"), - (["/run/nvx,a", "/b,b"], "overlap"), - (["/a b,a", "/b,b"], "invalid --mount target"), - (["relative,a", "/b,b"], "invalid --mount target"), - ([f"/share/{index:02},host" for index in range(60)], "1024-byte"), - ): - with self.subTest(mounts=mounts[:2]): - invalid = nvx.parse_args( - ["run", *[f"--mount={mount}" for mount in mounts], "--dry-run"] - ) - with self.assertRaisesRegex(common.ScriptError, message): - nvx.command_run(invalid) - relative = nvx.parse_args( - [ - "run", - "--mount", - "/a,a", - "--mount", - "/b,b", - "--mount-deny", - "secrets", - "--dry-run", - ] - ) - with self.assertRaisesRegex(common.ScriptError, "absolute host path"): - nvx.command_run(relative) - - def test_managed_lifecycle_persists_and_replays_two_shares(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary).resolve() - layer_path = root / "distro.erofs" - scratch_path = root / "scratch.ext4" - work = root / "work" - tools = root / "tools" - layer_path.write_bytes(b"layer") - scratch_path.write_bytes(b"scratch") - work.mkdir() - tools.mkdir() - state = root / "state" - sandbox_lifecycle.provision( - state, - sandbox.SandboxLaunch( - layers=( - sandbox.SandboxLayer( - role="distro", - path=layer_path, - uuid="11111111-1111-1111-1111-111111111111", - ), - ), - scratch=scratch_path, - mounts=( - sandbox.SandboxMount.parse( - f"/workspace,{work},rw", ("secrets",) - ), - sandbox.SandboxMount.parse(f"/opt/hostedtoolcache,{tools}"), - ), - ), - hypervisor="whp", - memory_mib=256, - net=None, - network_profile=None, - network_egress=None, - network_ingress=None, - network_egress_allow=(), - network_egress_deny=(), - host_loopback=None, - network_proxy=None, - host_loopback_forward=(), - cmdline="quiet", - timeout=10, - ) - - config = json.loads( - (state / sandbox_lifecycle.CONFIG_NAME).read_text(encoding="utf-8") - ) - self.assertEqual( - config["format"], sandbox_lifecycle.MULTI_MOUNT_CONFIG_FORMAT - ) - # Older releases know only the single `mount`, and reject format 4. - self.assertNotIn("mount", config) - with self.assertRaisesRegex(common.ScriptError, "unsupported format"): - sandbox_lifecycle._read_json( - state / sandbox_lifecycle.CONFIG_NAME, - "sandbox configuration", - version=(1, 2, 3), - ) - self.assertEqual( - config["mounts"], - [ - { - "guest_target": "/workspace", - "host_path": os.fspath(work), - "access": "rw", - "denied_paths": ["secrets"], - "owner": "vmm", - }, - { - "guest_target": "/opt/hostedtoolcache", - "host_path": os.fspath(tools), - "access": "ro", - "denied_paths": [], - "owner": "vmm", - }, - ], - ) - - process = MagicMock() - process.pid = 123 - process.stdin = io.BytesIO() - - def require(path: Path, _description: str) -> Path: - return path - - with ( - patch.object(sandbox_lifecycle, "require_file", side_effect=require), - patch.object( - sandbox_lifecycle.subprocess, "Popen", return_value=process - ) as popen, - patch.object( - sandbox_lifecycle, "_process_start_time", return_value=456 - ), - patch.object( - sandbox_lifecycle.ControlSession, - "connect", - return_value=MagicMock(), - ), - ): - sandbox_lifecycle.start(state, 10) - - command = popen.call_args.args[0] - self.assertEqual( - [ - command[index + 1] - for index, value in enumerate(command) - if value in ("--mount-aggregate", "--mount-child", "--mount-deny") - ], - [ - "/run/nvx/shares", - f"{sandbox.aggregate_child_name(0, '/workspace')},{work},rw", - f"{sandbox.aggregate_child_name(1, '/opt/hostedtoolcache')}," - f"{tools},ro", - os.fspath(work / "secrets"), - ], - ) - - def test_managed_lifecycle_persists_and_replays_the_access_policy(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary).resolve() - layer_path = root / "distro.erofs" - scratch_path = root / "scratch.ext4" - work = root / "work" - layer_path.write_bytes(b"layer") - scratch_path.write_bytes(b"scratch") - work.mkdir() - state = root / "state" - sandbox_lifecycle.provision( - state, - sandbox.SandboxLaunch( - layers=( - sandbox.SandboxLayer( - role="distro", - path=layer_path, - uuid="11111111-1111-1111-1111-111111111111", - ), - ), - scratch=scratch_path, - mounts=( - sandbox.SandboxMount.parse( - f"/workspace,{work},rw", - ("logs",), - allowed_paths=("logs/payloads",), - writable_paths=("out",), - ), - ), - ), - hypervisor="whp", - memory_mib=256, - net=None, - network_profile=None, - network_egress=None, - network_ingress=None, - network_egress_allow=(), - network_egress_deny=(), - host_loopback=None, - network_proxy=None, - host_loopback_forward=(), - cmdline="quiet", - timeout=10, - ) - - config = json.loads( - (state / sandbox_lifecycle.CONFIG_NAME).read_text(encoding="utf-8") - ) - self.assertEqual(config["format"], sandbox_lifecycle.POLICY_CONFIG_FORMAT) - # Earlier releases read at most format 4, which would start the - # share without its policy, so they must reject format 5. - self.assertNotIn("mount", config) - with self.assertRaisesRegex(common.ScriptError, "unsupported format"): - sandbox_lifecycle._read_json( - state / sandbox_lifecycle.CONFIG_NAME, - "sandbox configuration", - version=(1, 2, 3, 4), - ) - self.assertEqual( - config["mounts"], - [ - { - "guest_target": "/workspace", - "host_path": os.fspath(work), - "access": "rw", - "denied_paths": ["logs"], - "allowed_paths": ["logs/payloads"], - "writable_paths": ["out"], - "owner": "vmm", - }, - ], - ) - - process = MagicMock() - process.pid = 123 - process.stdin = io.BytesIO() - - def require(path: Path, _description: str) -> Path: - return path - - with ( - patch.object(sandbox_lifecycle, "require_file", side_effect=require), - patch.object( - sandbox_lifecycle.subprocess, "Popen", return_value=process - ) as popen, - patch.object( - sandbox_lifecycle, "_process_start_time", return_value=456 - ), - patch.object( - sandbox_lifecycle.ControlSession, - "connect", - return_value=MagicMock(), - ), - ): - sandbox_lifecycle.start(state, 10) - - command = popen.call_args.args[0] - self.assertEqual( - [ - (value, command[index + 1]) - for index, value in enumerate(command) - if value.startswith("--mount") - ], - [ - ("--mount", f"/workspace,{work},rw"), - ("--mount-deny", "logs"), - ("--mount-allow", "logs/payloads"), - ("--mount-write", "out"), - ], - ) - - # Format 5 holds an access policy, and only format 5 does. - config["mounts"][0]["allowed_paths"] = [] - config["mounts"][0]["writable_paths"] = [] - with ( - patch.object(sandbox, "require_file", side_effect=require), - self.assertRaisesRegex(common.ScriptError, "does not match"), - ): - sandbox_lifecycle._deserialize_launch(config) - - def test_launch_contract_attaches_a_file_as_an_aggregate_child(self): - distro = sandbox.SandboxLayer.parse( - "distro,distro.erofs,11111111-1111-1111-1111-111111111111" - ) - settings = sandbox.SandboxMount.parse( - "/etc/app/settings.json,settings.json,ro", kind="file" - ) - launch = sandbox.SandboxLaunch( - layers=(distro,), scratch=Path("scratch.ext4"), mounts=(settings,) - ) - name = sandbox.aggregate_child_name(0, "/etc/app/settings.json") - # Even alone, a file is the child of an aggregate, because the guest - # mounts the device's root, which is a directory. - self.assertTrue(sandbox.mounts_use_aggregate(launch.mounts)) - arguments = launch.openvmm_arguments() - self.assertNotIn("--mount", arguments) - self.assertEqual( - arguments[arguments.index("--mount-aggregate") :], - [ - "--mount-aggregate", - "/run/nvx/shares", - "--mount-child", - f"{name},{os.fspath(Path('settings.json'))},ro,file", - ], - ) - self.assertEqual( - sandbox.mounts_command_line_fragment(launch.mounts), - " virtfs_dir=/run/nvx/shares virtfs_tag=microvm virtfs_mode=ro" - " virtfs_aggregate=1", - ) - self.assertTrue( - launch.kernel_command_line().endswith( - f" nvx_share={name},/etc/app/settings.json,ro,file" - ) - ) - - # Files and directories share one aggregate, and only a file's token - # is marked. - workspace = sandbox.SandboxMount.parse("/workspace,work,rw", ("secrets",)) - output = sandbox.SandboxMount.parse( - "/results/output.txt,output.txt,rw", kind="file" - ) - mixed = sandbox.SandboxLaunch( - layers=(distro,), - scratch=Path("scratch.ext4"), - mounts=(workspace, settings, output), - ) - names = [ - sandbox.aggregate_child_name(index, mount.guest_target) - for index, mount in enumerate(mixed.mounts) - ] - self.assertTrue( - mixed.kernel_command_line().endswith( - f" nvx_share={names[0]},/workspace,rw" - f" nvx_share={names[1]},/etc/app/settings.json,ro,file" - f" nvx_share={names[2]},/results/output.txt,rw,file" - ) - ) - arguments = mixed.openvmm_arguments() - self.assertEqual( - arguments[arguments.index("--mount-deny") + 1], - os.fspath(Path.cwd() / "work" / "secrets"), - ) - - # A file's mode applies to the whole file, so no policy path names - # one, and the runtime owns the machine ID's path. - for option in ("denied_paths", "allowed_paths", "writable_paths"): - policy: dict[str, Any] = {option: ("x",)} - with self.subTest(option=option): - with self.assertRaisesRegex(common.ScriptError, "shares a file"): - sandbox.SandboxMount( - guest_target="/results/output.txt", - host_path=Path("output.txt"), - access="rw", - kind="file", - **policy, - ) - with self.assertRaisesRegex( - common.ScriptError, "unsupported sandbox mount kind" - ): - sandbox.SandboxMount.parse("/a,a", kind="socket") - # The runtime replaces /etc/machine-id with a file, so nothing at or - # below it may be shared. - for target in ("/etc/machine-id", "/etc/machine-id/child"): - with self.subTest(target=target): - with self.assertRaisesRegex(common.ScriptError, "reserved"): - sandbox.SandboxMount.parse(f"{target},id", kind="file") - # Only /etc itself is reserved, not what lies below it. - sandbox.SandboxMount.parse("/etc/app,app") - - def test_launch_validation_requires_the_kind_of_each_share(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - config = root / "config" - config.mkdir() - settings = config / "settings.json" - settings.write_bytes(b"{}") - self.assertEqual(sandbox.mount_kind(settings), "file") - self.assertEqual(sandbox.mount_kind(config), "directory") - self.assertEqual(sandbox.mount_kind(root / "missing"), "directory") - share = sandbox.SandboxMount( - guest_target="/etc/app/settings.json", host_path=settings, kind="file" - ) - self.assertIs(share.validated(), share) - # A directory at the file's path is not shared in its place. - settings.unlink() - settings.mkdir() - with self.assertRaisesRegex(common.ScriptError, "not a regular file"): - share.validated() - settings.rmdir() - with self.assertRaisesRegex(common.ScriptError, "not a regular file"): - share.validated() - - # A file overlaps a shared directory that contains it, and another - # share of the same file, such as through a hard link, but not a - # file beside it. - settings.write_bytes(b"{}") - other = config / "other.json" - other.write_bytes(b"{}") - link = root / "link.json" - os.link(settings, link) - - def file_share(target: str, path: Path) -> sandbox.SandboxMount: - return sandbox.SandboxMount( - guest_target=target, host_path=path, kind="file" - ) - - sandbox.validate_mount_host_paths( - (file_share("/a.json", settings), file_share("/b.json", other)) - ) - for mounts, message in ( - ( - ( - sandbox.SandboxMount(guest_target="/config", host_path=config), - file_share("/a.json", settings), - ), - "overlap", - ), - ( - (file_share("/a.json", settings), file_share("/b.json", link)), - "same directory or file", - ), - ): - with self.subTest(message=message): - with self.assertRaisesRegex(common.ScriptError, message): - sandbox.validate_mount_host_paths(mounts) - - def test_managed_configuration_format_records_file_shares(self): - def entry(target: str, path: str, kind: str) -> dict[str, object]: - return { - "guest_target": target, - "host_path": path, - "access": "ro", - "denied_paths": [], - "owner": "vmm", - "allowed_paths": [], - "writable_paths": [], - "kind": kind, - } - - config: dict[str, object] = { - "format": sandbox_lifecycle.FILE_MOUNT_CONFIG_FORMAT, - "layers": [ - { - "role": "distro", - "path": "distro.erofs", - "uuid": "11111111-1111-1111-1111-111111111111", - } - ], - "scratch": "scratch.ext4", - "hostname": "nvx-sandbox", - "workload_uid": 65534, - "workload_gid": 65534, - "memory_max": None, - "pids_max": None, - "mounts": [entry("/etc/app/settings.json", "settings.json", "file")], - } - - def require(path: Path, _description: str) -> Path: - return path - - def unchanged(mount: sandbox.SandboxMount) -> sandbox.SandboxMount: - return mount - - with ( - patch.object(sandbox, "require_file", side_effect=require), - patch.object(sandbox.SandboxMount, "validated", unchanged), - ): - launch = sandbox_lifecycle._deserialize_launch(config) - self.assertEqual([mount.kind for mount in launch.mounts], ["file"]) - serialized = sandbox_lifecycle._serialize_launch( - launch, - hypervisor="kvm", - memory_mib=256, - net=None, - network_profile=None, - network_egress=None, - network_ingress=None, - network_egress_allow=(), - network_egress_deny=(), - host_loopback=None, - network_proxy=None, - host_loopback_forward=(), - cmdline="", - ) - self.assertEqual( - serialized["format"], sandbox_lifecycle.FILE_MOUNT_CONFIG_FORMAT - ) - self.assertEqual(serialized["mounts"][0]["kind"], "file") - # Only format 6 holds a file share, and every format-6 share - # records its kind. - config["mounts"] = [entry("/workspace", "work", "directory")] - with self.assertRaisesRegex(common.ScriptError, "does not match"): - sandbox_lifecycle._deserialize_launch(config) - config["mounts"] = [{**entry("/a.json", "a.json", "file"), "kind": None}] - with self.assertRaisesRegex(common.ScriptError, "kind"): - sandbox_lifecycle._deserialize_launch(config) - unmarked = entry("/a.json", "a.json", "file") - del unmarked["kind"] - config["mounts"] = [unmarked] - with self.assertRaisesRegex(common.ScriptError, "malformed"): - sandbox_lifecycle._deserialize_launch(config) - config["format"] = sandbox_lifecycle.POLICY_CONFIG_FORMAT - config["mounts"] = [entry("/a.json", "a.json", "file")] - with self.assertRaisesRegex(common.ScriptError, "does not match"): - sandbox_lifecycle._deserialize_launch(config) - - def test_managed_configuration_format_binds_several_mounts(self): - def entry(target: str, path: str, owner: str = "vmm") -> dict[str, object]: - return { - "guest_target": target, - "host_path": path, - "access": "rw", - "denied_paths": [], - "owner": owner, - } - - config: dict[str, object] = { - "format": sandbox_lifecycle.MULTI_MOUNT_CONFIG_FORMAT, - "layers": [ - { - "role": "distro", - "path": "distro.erofs", - "uuid": "11111111-1111-1111-1111-111111111111", - } - ], - "scratch": "scratch.ext4", - "hostname": "nvx-sandbox", - "workload_uid": 65534, - "workload_gid": 65534, - "memory_max": None, - "pids_max": None, - "mounts": [entry("/workspace", "work"), entry("/tools", "tools")], - } - - def require(path: Path, _description: str) -> Path: - return path - - def unchanged(mount: sandbox.SandboxMount) -> sandbox.SandboxMount: - return mount - - def identity(path: Path) -> tuple[int, int]: - return 0, hash(path) - - with ( - patch.object(sandbox, "require_file", side_effect=require), - patch.object(sandbox.SandboxMount, "validated", unchanged), - patch.object(sandbox, "_path_identity", side_effect=identity), - ): - launch = sandbox_lifecycle._deserialize_launch(config) - self.assertEqual( - [mount.guest_target for mount in launch.mounts], - ["/workspace", "/tools"], - ) - # Format 4 always holds several shares, and only format 4 does. - config["mounts"] = [entry("/workspace", "work")] - with self.assertRaisesRegex(common.ScriptError, "does not match"): - sandbox_lifecycle._deserialize_launch(config) - config["mounts"] = {"guest_target": "/workspace"} - with self.assertRaisesRegex(common.ScriptError, "malformed"): - sandbox_lifecycle._deserialize_launch(config) - config["mounts"] = [entry("/workspace", "work"), entry("/a", "a", "caller")] - with self.assertRaisesRegex(common.ScriptError, "same --mount-owner"): - sandbox_lifecycle._deserialize_launch(config) - config["mounts"] = [entry("/workspace", "work"), entry("/workspace", "b")] - with self.assertRaisesRegex(common.ScriptError, "overlap"): - sandbox_lifecycle._deserialize_launch(config) - config["format"] = sandbox_lifecycle.MOUNT_CONFIG_FORMAT - config["mounts"] = [entry("/workspace", "work"), entry("/tools", "tools")] - with self.assertRaisesRegex(common.ScriptError, "does not match"): - sandbox_lifecycle._deserialize_launch(config) - - def test_managed_lifecycle_persists_and_replays_mount(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary).resolve() - layer_path = root / "distro.erofs" - scratch_path = root / "scratch.ext4" - share = root / "share" - layer_path.write_bytes(b"layer") - scratch_path.write_bytes(b"scratch") - share.mkdir() - state = root / "state" - previous = Path.cwd() - os.chdir(root) - try: - sandbox_lifecycle.provision( - state, - sandbox.SandboxLaunch( - layers=( - sandbox.SandboxLayer( - role="distro", - path=layer_path, - uuid="11111111-1111-1111-1111-111111111111", - ), - ), - scratch=scratch_path, - mounts=( - sandbox.SandboxMount.parse( - "/workspace,share,rw", ("secrets",) - ), - ), - ), - hypervisor="whp", - memory_mib=256, - net=None, - network_profile=None, - network_egress=None, - network_ingress=None, - network_egress_allow=(), - network_egress_deny=(), - host_loopback=None, - network_proxy=None, - host_loopback_forward=(), - cmdline="quiet", - timeout=10, - ) - finally: - os.chdir(previous) - - config = json.loads( - (state / sandbox_lifecycle.CONFIG_NAME).read_text(encoding="utf-8") - ) - self.assertEqual(config["format"], sandbox_lifecycle.MOUNT_CONFIG_FORMAT) - with self.assertRaisesRegex(common.ScriptError, "unsupported format"): - sandbox_lifecycle._read_json( - state / sandbox_lifecycle.CONFIG_NAME, - "sandbox configuration", - version=1, - ) - self.assertEqual( - config["mount"], - { - "guest_target": "/workspace", - "host_path": os.fspath(share), - "access": "rw", - "denied_paths": ["secrets"], - "owner": "vmm", - }, - ) - - process = MagicMock() - process.pid = 123 - process.stdin = io.BytesIO() - context = MagicMock() - - def require(path: Path, _description: str) -> Path: - return path - - with ( - patch.object(sandbox_lifecycle, "require_file", side_effect=require), - patch.object( - sandbox_lifecycle.subprocess, "Popen", return_value=process - ) as popen, - patch.object( - sandbox_lifecycle, "_process_start_time", return_value=456 - ), - patch.object( - sandbox_lifecycle.ControlSession, "connect", return_value=context - ), - ): - sandbox_lifecycle.start(state, 10) - - command = popen.call_args.args[0] - self.assertEqual( - command[command.index("--mount") + 1], f"/workspace,{share},rw" - ) - self.assertEqual(command[command.index("--mount-deny") + 1], "secrets") - - def test_managed_lifecycle_accepts_configuration_without_mount(self): - config: dict[str, object] = { - "format": sandbox_lifecycle.CONFIG_FORMAT, - "layers": [ - { - "role": "distro", - "path": "distro.erofs", - "uuid": "11111111-1111-1111-1111-111111111111", - } - ], - "scratch": "scratch.ext4", - "hostname": "nvx-sandbox", - "workload_uid": 65534, - "workload_gid": 65534, - "memory_max": None, - "pids_max": None, - } - - def require(path: Path, _description: str) -> Path: - return path - - with ( - patch.object(sandbox, "require_file", side_effect=require), - ): - self.assertEqual(sandbox_lifecycle._deserialize_launch(config).mounts, ()) - config["format"] = sandbox_lifecycle.MOUNT_CONFIG_FORMAT - with self.assertRaisesRegex(common.ScriptError, "does not match"): - sandbox_lifecycle._deserialize_launch(config) - config["mount"] = {"guest_target": "/workspace"} - with self.assertRaisesRegex(common.ScriptError, "malformed"): - sandbox_lifecycle._deserialize_launch(config) - config["mount"] = { - "guest_target": "/proc", - "host_path": "share", - "access": "rw", - "denied_paths": [], - } - with self.assertRaisesRegex(common.ScriptError, "reserved"): - sandbox_lifecycle._deserialize_launch(config) - config["mount"] = { - "guest_target": "/workspace", - "host_path": "share", - "access": "rw", - "denied_paths": [], - } - config["format"] = sandbox_lifecycle.CONFIG_FORMAT - with self.assertRaisesRegex(common.ScriptError, "does not match"): - sandbox_lifecycle._deserialize_launch(config) - - def test_managed_configuration_format_binds_the_mount_owner(self): - mount: dict[str, object] = { - "guest_target": "/workspace", - "host_path": "share", - "access": "rw", - "denied_paths": [], - } - config: dict[str, object] = { - "format": sandbox_lifecycle.MOUNT_CONFIG_FORMAT, - "layers": [ - { - "role": "distro", - "path": "distro.erofs", - "uuid": "11111111-1111-1111-1111-111111111111", - } - ], - "scratch": "scratch.ext4", - "hostname": "nvx-sandbox", - "workload_uid": 65534, - "workload_gid": 65534, - "memory_max": None, - "pids_max": None, - "mount": mount, - } - - def require(path: Path, _description: str) -> Path: - return path - - def unchanged(mount: sandbox.SandboxMount) -> sandbox.SandboxMount: - return mount - - with ( - patch.object(sandbox, "require_file", side_effect=require), - patch.object(sandbox.SandboxMount, "validated", unchanged), - ): - # A share configured before ownership modes runs as the VMM. - launch = sandbox_lifecycle._deserialize_launch(config) - self.assertEqual(launch.mounts[0].owner, "vmm") - mount["owner"] = "vmm" - launch = sandbox_lifecycle._deserialize_launch(config) - self.assertEqual(launch.mounts[0].owner, "vmm") - # Only format 3, which older releases reject, carries caller ownership. - mount["owner"] = "caller" - with self.assertRaisesRegex(common.ScriptError, "does not match"): - sandbox_lifecycle._deserialize_launch(config) - config["format"] = sandbox_lifecycle.OWNER_CONFIG_FORMAT - launch = sandbox_lifecycle._deserialize_launch(config) - self.assertEqual(launch.mounts[0].owner, "caller") - mount["owner"] = "vmm" - with self.assertRaisesRegex(common.ScriptError, "does not match"): - sandbox_lifecycle._deserialize_launch(config) - mount["owner"] = "root" - with self.assertRaisesRegex( - common.ScriptError, "unsupported sandbox mount owner" - ): - sandbox_lifecycle._deserialize_launch(config) - - def test_managed_lifecycle_persists_and_replays_caller_owner(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary).resolve() - layer_path = root / "distro.erofs" - scratch_path = root / "scratch.ext4" - share = root / "share" - layer_path.write_bytes(b"layer") - scratch_path.write_bytes(b"scratch") - share.mkdir() - state = root / "state" - launch = sandbox.SandboxLaunch( - layers=( - sandbox.SandboxLayer( - role="distro", - path=layer_path, - uuid="11111111-1111-1111-1111-111111111111", - ), - ), - scratch=scratch_path, - mounts=( - sandbox.SandboxMount.parse(f"/workspace,{share},rw", (), "caller"), - ), - ) - - def provision() -> None: - sandbox_lifecycle.provision( - state, - launch, - hypervisor="whp" if sys.platform == "win32" else "kvm", - memory_mib=256, - net=None, - network_profile=None, - network_egress=None, - network_ingress=None, - network_egress_allow=(), - network_egress_deny=(), - host_loopback=None, - network_proxy=None, - host_loopback_forward=(), - cmdline="quiet", - timeout=10, - ) - - if sys.platform == "win32": - with self.assertRaisesRegex(common.ScriptError, "Linux host"): - provision() - self.assertFalse((state / sandbox_lifecycle.CONFIG_NAME).exists()) - return - if share.stat().st_uid == 0: - self.skipTest("caller ownership refuses a root-owned share") - provision() - - config = json.loads( - (state / sandbox_lifecycle.CONFIG_NAME).read_text(encoding="utf-8") - ) - self.assertEqual(config["format"], sandbox_lifecycle.OWNER_CONFIG_FORMAT) - self.assertEqual(config["mount"]["owner"], "caller") - # NVX releases that know only formats 1 and 2 refuse to start it. - with self.assertRaisesRegex(common.ScriptError, "unsupported format"): - sandbox_lifecycle._read_json( - state / sandbox_lifecycle.CONFIG_NAME, - "sandbox configuration", - version=( - sandbox_lifecycle.CONFIG_FORMAT, - sandbox_lifecycle.MOUNT_CONFIG_FORMAT, - ), - ) - - process = MagicMock() - process.pid = 123 - process.stdin = io.BytesIO() - - def require(path: Path, _description: str) -> Path: - return path - - with ( - patch.object(sandbox_lifecycle, "require_file", side_effect=require), - patch.object( - sandbox_lifecycle.subprocess, "Popen", return_value=process - ) as popen, - patch.object( - sandbox_lifecycle, "_process_start_time", return_value=456 - ), - patch.object( - sandbox_lifecycle.ControlSession, - "connect", - return_value=MagicMock(), - ), - ): - sandbox_lifecycle.start(state, 10) - - command = popen.call_args.args[0] - self.assertEqual(command[command.index("--mount-owner") + 1], "caller") - - def test_managed_lifecycle_provisions_and_deprovisions_state(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - layer_path = root / "distro.erofs" - scratch_path = root / "scratch.ext4" - layer_path.write_bytes(b"layer") - scratch_path.write_bytes(b"scratch") - launch = sandbox.SandboxLaunch( - layers=( - sandbox.SandboxLayer( - role="distro", - path=layer_path, - uuid="11111111-1111-1111-1111-111111111111", - ), - ), - scratch=scratch_path, - ) - state = root / "state" - - sandbox_lifecycle.provision( - state, - launch, - hypervisor="whp", - memory_mib=256, - net=None, - network_profile=None, - network_egress=None, - network_ingress=None, - network_egress_allow=(), - network_egress_deny=(), - host_loopback=None, - network_proxy=None, - host_loopback_forward=(), - cmdline="quiet", - timeout=10, - ) - - config = json.loads( - (state / sandbox_lifecycle.CONFIG_NAME).read_text(encoding="utf-8") - ) - self.assertEqual(config["format"], sandbox_lifecycle.CONFIG_FORMAT) - self.assertIsNone(config["mount"]) - self.assertEqual(config["workload_uid"], 65534) - self.assertEqual(config["hypervisor"], "whp") - self.assertFalse((state / sandbox_lifecycle.RUNTIME_NAME).exists()) - - config["network_egress_allow"] = None - (state / sandbox_lifecycle.CONFIG_NAME).write_text( - json.dumps(config), encoding="utf-8" - ) - - def require(path: Path, _description: str) -> Path: - return path - - with ( - patch.object( - sandbox_lifecycle, - "require_file", - side_effect=require, - ), - self.assertRaisesRegex( - common.ScriptError, "sandbox configuration is malformed" - ), - ): - sandbox_lifecycle.start(state, 10) - - with self.assertRaisesRegex(common.ScriptError, "already provisioned"): - sandbox_lifecycle.provision( - state, - launch, - hypervisor="whp", - memory_mib=256, - net=None, - network_profile=None, - network_egress=None, - network_ingress=None, - network_egress_allow=(), - network_egress_deny=(), - host_loopback=None, - network_proxy=None, - host_loopback_forward=(), - cmdline="quiet", - timeout=10, - ) - - sandbox_lifecycle.deprovision(state, 10) - self.assertFalse(state.exists()) - - def test_managed_lifecycle_start_requests_openvmm_outcome(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary).resolve() - layer_path = root / "distro.erofs" - scratch_path = root / "scratch.ext4" - layer_path.write_bytes(b"layer") - scratch_path.write_bytes(b"scratch") - state = root / "state" - sandbox_lifecycle.provision( - state, - sandbox.SandboxLaunch( - layers=( - sandbox.SandboxLayer( - role="distro", - path=layer_path, - uuid="11111111-1111-1111-1111-111111111111", - ), - ), - scratch=scratch_path, - ), - hypervisor="whp", - memory_mib=256, - net=None, - network_profile=None, - network_egress=None, - network_ingress=None, - network_egress_allow=(), - network_egress_deny=(), - host_loopback=None, - network_proxy=None, - host_loopback_forward=(), - cmdline="quiet", - timeout=10, - ) - process = MagicMock() - process.pid = 123 - process.stdin = io.BytesIO() - session = MagicMock() - context = MagicMock() - context.__enter__.return_value = session - - def require(path: Path, _description: str) -> Path: - return path - - with ( - patch.object( - sandbox_lifecycle, - "require_file", - side_effect=require, - ), - patch.object( - sandbox_lifecycle.subprocess, - "Popen", - return_value=process, - ) as popen, - patch.object( - sandbox_lifecycle, - "_process_start_time", - return_value=456, - ) as process_start_time, - patch.object( - sandbox_lifecycle.ControlSession, - "connect", - return_value=context, - ), - ): - sandbox_lifecycle.start(state, 10) - - command = popen.call_args.args[0] - self.assertEqual( - Path(command[command.index("--microvm-report") + 1]), - state / sandbox_lifecycle.OUTCOME_NAME, - ) - session.ping.assert_called_once_with(10) - process_start_time.assert_called_once_with(123) - runtime = json.loads( - (state / sandbox_lifecycle.RUNTIME_NAME).read_text(encoding="utf-8") - ) - self.assertEqual((runtime["pid"], runtime["start_time"]), (123, 456)) - - def test_versioned_json_reader_supports_custom_version_field(self): - with tempfile.TemporaryDirectory() as temporary: - path = Path(temporary) / "outcome.json" - payload = {"schema_version": sandbox_lifecycle.OUTCOME_SCHEMA_VERSION} - path.write_text(json.dumps(payload), encoding="utf-8") - - self.assertEqual( - sandbox_lifecycle._read_json( - path, - "OpenVMM outcome report", - version_field="schema_version", - version=sandbox_lifecycle.OUTCOME_SCHEMA_VERSION, - ), - payload, - ) - with self.assertRaisesRegex(common.ScriptError, "unsupported format"): - sandbox_lifecycle._read_json( - path, - "OpenVMM outcome report", - version_field="schema_version", - version=sandbox_lifecycle.OUTCOME_SCHEMA_VERSION + 1, - ) - - def test_versioned_json_reader_rejects_invalid_utf8(self): - with tempfile.TemporaryDirectory() as temporary: - path = Path(temporary) / "config.json" - path.write_bytes(b"\xff") - - with self.assertRaisesRegex(common.ScriptError, "failed to read"): - sandbox_lifecycle._read_json(path, "sandbox configuration") - - def test_managed_exec_outcome_excludes_workload_data(self): - with tempfile.TemporaryDirectory() as temporary: - path = Path(temporary) / "outcome.json" - sandbox_lifecycle.write_exec_outcome( - path, - sandbox_lifecycle.ManagedExecResult( - 124, - "timeout", - b"sensitive stdout", - b"sensitive stderr", - ), - ) - outcome = json.loads(path.read_text(encoding="utf-8")) - - self.assertEqual(outcome["schema_version"], 1) - self.assertEqual( - outcome["outcome"], - { - "operation": "exec", - "category": "timeout", - "status_code": 124, - }, - ) - self.assertEqual(len(outcome["operation_id"]), 32) - encoded = json.dumps(outcome) - self.assertNotIn("stdout", encoded) - self.assertNotIn("stderr", encoded) - self.assertNotIn("sensitive", encoded) - - def test_managed_exec_outcome_does_not_overwrite(self): - with tempfile.TemporaryDirectory() as temporary: - path = Path(temporary) / "outcome.json" - path.write_text("existing", encoding="utf-8") - with self.assertRaisesRegex(common.ScriptError, "already exists"): - sandbox_lifecycle.write_exec_outcome( - path, - sandbox_lifecycle.ManagedExecResult(0, "exit", b"", b""), - ) - - def test_managed_exec_outcome_rejects_existing_symlink(self): - with tempfile.TemporaryDirectory() as temporary: - directory = Path(temporary) - target = directory / "missing-target" - path = directory / "outcome.json" - try: - path.symlink_to(target) - except OSError as error: - self.skipTest(f"symlinks are unavailable: {error}") - with self.assertRaisesRegex(common.ScriptError, "already exists"): - sandbox_lifecycle.write_exec_outcome( - path, - sandbox_lifecycle.ManagedExecResult(0, "exit", b"", b""), - ) - - def test_managed_stop_cleans_runtime_state_if_report_is_invalid(self): - with tempfile.TemporaryDirectory() as temporary: - state = Path(temporary) - sandbox_lifecycle._write_json( - state / sandbox_lifecycle.RUNTIME_NAME, - { - "format": sandbox_lifecycle.STATE_FORMAT, - "pid": 123, - "control_endpoint": "control.sock", - }, - ) - (state / sandbox_lifecycle.CAPABILITY_NAME).write_bytes(b"x" * 32) - (state / sandbox_lifecycle.CONTROL_SOCKET_NAME).touch() - (state / sandbox_lifecycle.OUTCOME_NAME).write_text( - "invalid", - encoding="utf-8", - ) - context = MagicMock() - with ( - patch.object( - sandbox_lifecycle, - "_process_running", - side_effect=(True, False), - ), - patch.object( - sandbox_lifecycle.ControlSession, - "connect", - return_value=context, - ), - self.assertRaisesRegex(common.ScriptError, "outcome report"), - ): - sandbox_lifecycle.stop(state, 10) - - self.assertFalse((state / sandbox_lifecycle.RUNTIME_NAME).exists()) - self.assertFalse((state / sandbox_lifecycle.CAPABILITY_NAME).exists()) - self.assertFalse((state / sandbox_lifecycle.CONTROL_SOCKET_NAME).exists()) - self.assertTrue((state / sandbox_lifecycle.OUTCOME_NAME).exists()) - - def test_process_identity_parses_linux_process_status(self): - def status(state: bytes, start_time: bytes = b"98765") -> bytes: - fields = b" ".join([state, *(b"0" for _ in range(18)), start_time, b"0"]) - # The command name contains a space and parentheses. - return b"42 (sh) x (y) " + fields + b"\n" - - self.assertEqual(sandbox_lifecycle._linux_start_time(status(b"S")), 98765) - self.assertEqual(sandbox_lifecycle._linux_start_time(status(b"D")), 98765) - for state in (b"Z", b"X"): - self.assertIsNone(sandbox_lifecycle._linux_start_time(status(state))) - for malformed in ( - b"42 (sh", - b"42 (sh) S 1 2\n", - status(b"S", b"-1"), - status(b"S", b"tick"), - ): - with self.assertRaisesRegex(ValueError, "unexpected format"): - sandbox_lifecycle._linux_start_time(malformed) - - def test_process_identity_requires_the_recorded_start_time(self): - if os.name != "nt" and sys.platform != "linux": - self.skipTest("process identity requires a Linux or Windows host") - pid = os.getpid() - start_time = sandbox_lifecycle._process_start_time(pid) - assert isinstance(start_time, int) - - self.assertTrue(sandbox_lifecycle._process_running(pid, start_time)) - self.assertFalse(sandbox_lifecycle._process_running(pid, start_time + 1)) - # Records from earlier NVX versions identify OpenVMM by process ID alone. - self.assertTrue(sandbox_lifecycle._process_running(pid, None)) - - def test_runtime_records_report_whether_their_process_runs(self): - if os.name != "nt" and sys.platform != "linux": - self.skipTest("process identity requires a Linux or Windows host") - pid = os.getpid() - start_time = sandbox_lifecycle._process_start_time(pid) - assert isinstance(start_time, int) - - self.assertTrue( - sandbox_lifecycle.runtime_process_running( - {"pid": pid, "start_time": start_time} - ) - ) - self.assertFalse( - sandbox_lifecycle.runtime_process_running( - {"pid": pid, "start_time": start_time + 1} - ) - ) - with self.assertRaisesRegex(common.ScriptError, "invalid process ID"): - sandbox_lifecycle.runtime_process_running({"pid": "openvmm"}) - with self.assertRaisesRegex(common.ScriptError, "invalid process start"): - sandbox_lifecycle.runtime_process_running({"pid": pid, "start_time": -1}) - - def test_process_identity_counts_an_exited_process_as_stopped(self): - if os.name != "nt" and sys.platform != "linux": - self.skipTest("process identity requires a Linux or Windows host") - child = _stdin_bound_child() - try: - start_time = sandbox_lifecycle._process_start_time(child.pid) - assert isinstance(start_time, int) - self.assertTrue(sandbox_lifecycle._process_running(child.pid, start_time)) - finally: - _finish_child(child) - - # On Windows, the handle that Popen holds keeps the exited process's ID. - self.assertFalse(sandbox_lifecycle._process_running(child.pid, start_time)) - self.assertFalse(sandbox_lifecycle._process_running(child.pid, None)) - - def test_process_identity_counts_a_zombie_as_stopped(self): - if sys.platform != "linux": - self.skipTest("zombie processes require a Linux host") - child = _stdin_bound_child() - try: - start_time = sandbox_lifecycle._process_start_time(child.pid) - assert isinstance(start_time, int) - _exit_unreaped(child) - stat = Path(f"/proc/{child.pid}/stat").read_bytes() - self.assertEqual(stat[stat.rfind(b")") + 2 :][:1], b"Z") - - self.assertIsNone(sandbox_lifecycle._process_start_time(child.pid)) - self.assertFalse(sandbox_lifecycle._process_running(child.pid, start_time)) - self.assertFalse(sandbox_lifecycle._process_running(child.pid, None)) - finally: - _finish_child(child) - - def test_managed_stop_completes_while_openvmm_remains_unreaped(self): - if sys.platform != "linux": - self.skipTest("zombie processes require a Linux host") - child = _stdin_bound_child() - try: - with tempfile.TemporaryDirectory() as temporary: - state = Path(temporary) - _write_managed_runtime( - state, - child.pid, - sandbox_lifecycle._process_start_time(child.pid), - ) - outcome = { - "schema_version": sandbox_lifecycle.OUTCOME_SCHEMA_VERSION, - "outcome": {"category": "stopped"}, - "network_policy": {}, - "teardown": {}, - } - (state / sandbox_lifecycle.OUTCOME_NAME).write_text( - json.dumps(outcome), encoding="utf-8" - ) - session = MagicMock() - context = MagicMock() - context.__enter__.return_value = session - - def acknowledge_stop(_timeout: float) -> None: - # OpenVMM exits after the guest acknowledges STOP, but its - # parent, such as a container init without --init, never - # reaps it. - _exit_unreaped(child) - - session.stop.side_effect = acknowledge_stop - with patch.object( - sandbox_lifecycle.ControlSession, - "connect", - return_value=context, - ): - self.assertEqual(sandbox_lifecycle.stop(state, 10), outcome) - - session.stop.assert_called_once_with(10) - self.assertFalse((state / sandbox_lifecycle.RUNTIME_NAME).exists()) - self.assertFalse((state / sandbox_lifecycle.CAPABILITY_NAME).exists()) - finally: - _finish_child(child) - - def test_managed_lifecycle_treats_a_reused_process_id_as_stopped(self): - if os.name != "nt" and sys.platform != "linux": - self.skipTest("process identity requires a Linux or Windows host") - with tempfile.TemporaryDirectory() as temporary: - state = Path(temporary) / "state" - state.mkdir() - start_time = sandbox_lifecycle._process_start_time(os.getpid()) - assert isinstance(start_time, int) - # OpenVMM is gone, and this test process now has its process ID. - _write_managed_runtime(state, os.getpid(), start_time + 1) - (state / sandbox_lifecycle.CONFIG_NAME).write_text("{}", encoding="utf-8") - - with patch.object(sandbox_lifecycle.ControlSession, "connect") as connect: - with self.assertRaisesRegex( - common.ScriptError, "runtime state is stale" - ): - sandbox_lifecycle.stop(state, 10) - with self.assertRaisesRegex( - common.ScriptError, "runtime state is stale" - ): - sandbox_lifecycle.exec_workload( - state, ("/bin/true",), timeout_ms=0, response_timeout=10 - ) - connect.assert_not_called() - - sandbox_lifecycle.deprovision(state, 10) - self.assertFalse(state.exists()) - - def test_managed_lifecycle_identifies_legacy_records_by_process_id(self): - if os.name != "nt" and sys.platform != "linux": - self.skipTest("process identity requires a Linux or Windows host") - with tempfile.TemporaryDirectory() as temporary: - state = Path(temporary) - _write_managed_runtime(state, os.getpid(), None) - - with self.assertRaisesRegex(common.ScriptError, "must be stopped"): - sandbox_lifecycle.deprovision(state, 10) - self.assertTrue((state / sandbox_lifecycle.RUNTIME_NAME).exists()) - - def test_managed_lifecycle_rejects_invalid_process_start_times(self): - with tempfile.TemporaryDirectory() as temporary: - state = Path(temporary) - for start_time in ("1", True, -1, 1.5): - with self.subTest(start_time=start_time): - sandbox_lifecycle._write_json( - state / sandbox_lifecycle.RUNTIME_NAME, - { - "format": sandbox_lifecycle.STATE_FORMAT, - "pid": os.getpid(), - "start_time": start_time, - "control_endpoint": "control.sock", - }, - ) - for operation in ( - lambda: sandbox_lifecycle.stop(state, 10), - lambda: sandbox_lifecycle.deprovision(state, 10), - ): - with self.assertRaisesRegex( - common.ScriptError, "invalid process start time" - ): - operation() - - def test_managed_start_fails_closed_without_openvmm_identity(self): - def require(path: Path, _description: str) -> Path: - return path - - # A side effect that yields None reports that no live process has the ID. - for identity, exit_status, message in ( - ((None,), 1, "OpenVMM exited during startup"), - ((None,), None, "cannot identify the OpenVMM process$"), - (OSError("denied"), None, "cannot identify the OpenVMM process: denied"), - ): - with ( - self.subTest(message=message), - tempfile.TemporaryDirectory() as temporary, - ): - state = _provision_managed_sandbox(Path(temporary).resolve()) - process = MagicMock() - process.pid = 123 - process.stdin = io.BytesIO() - process.poll.return_value = exit_status - with ( - patch.object( - sandbox_lifecycle, "require_file", side_effect=require - ), - patch.object( - sandbox_lifecycle.subprocess, "Popen", return_value=process - ), - patch.object( - sandbox_lifecycle, "_process_start_time", side_effect=identity - ), - patch.object( - sandbox_lifecycle.ControlSession, "connect" - ) as connect, - self.assertRaisesRegex(common.ScriptError, message), - ): - sandbox_lifecycle.start(state, 10) - - connect.assert_not_called() - self.assertEqual(process.terminate.called, exit_status is None) - self.assertFalse((state / sandbox_lifecycle.RUNTIME_NAME).exists()) - self.assertFalse((state / sandbox_lifecycle.CAPABILITY_NAME).exists()) - self.assertTrue((state / sandbox_lifecycle.CONFIG_NAME).exists()) - - def test_managed_start_lets_exiting_openvmm_publish_its_outcome(self): - # A guest that exits during startup, such as one that refuses the workload - # identity, makes OpenVMM close the control endpoint before it publishes - # its outcome report. Terminating OpenVMM then left the report's staging - # file, which deprovision refuses to remove (issue #438). - if os.name != "nt" and sys.platform != "linux": - self.skipTest("managed sandboxes require a Linux or Windows host") - temporary = tempfile.TemporaryDirectory() - self.addCleanup(temporary.cleanup) - state = _provision_managed_sandbox(Path(temporary.name).resolve()) - popen = subprocess.Popen - launched: list[subprocess.Popen[bytes]] = [] - - def launch(command: list[str], **options: Any) -> subprocess.Popen[bytes]: - attach_size = control_session.OUTER_HEADER.size + 32 - # With unknown options, the type checker cannot tell that start - # requests binary streams. - process = cast( - "subprocess.Popen[bytes]", - popen( - [ - sys.executable, - "-c", - _EXITING_OPENVMM, - str(attach_size), - *command[1:], - ], - **options, - ), - ) - self.addCleanup(_end_child, process) - launched.append(process) - return process - - def require(path: Path, _description: str) -> Path: - return path - - with ( - patch.object(sandbox_lifecycle, "require_file", side_effect=require), - patch.object(sandbox_lifecycle.subprocess, "Popen", side_effect=launch), - self.assertRaisesRegex( - common.ScriptError, - "^OpenVMM exited during startup: guest-exit status 125$", - ), - ): - sandbox_lifecycle.start(state, 30) - - log = (state / sandbox_lifecycle.LOG_NAME).read_text( - encoding="utf-8", errors="replace" - ) - # OpenVMM exited on its own, after it published the report. - self.assertEqual([process.returncode for process in launched], [125], log) - self.assertEqual( - sorted(path.name for path in state.iterdir()), - [ - sandbox_lifecycle.CONFIG_NAME, - sandbox_lifecycle.LOCK_NAME, - sandbox_lifecycle.LOG_NAME, - sandbox_lifecycle.OUTCOME_NAME, - ], - log, - ) - sandbox_lifecycle.deprovision(state, 10) - self.assertFalse(state.exists()) - - def test_managed_start_waits_only_for_openvmm_that_closed_its_endpoint(self): - def require(path: Path, _description: str) -> Path: - return path - - closed = control_session.ControlEndpointClosed() - # Each case holds the control failure, the status with which OpenVMM exits - # on its own or None if it runs until terminated, the waits for it, and the - # error that start raises. - for failure, status, waits, raised, message in ( - # OpenVMM closed the endpoint and exits within the grace period. - ( - closed, - 7, - [call(timeout=10)], - common.ScriptError, - "^OpenVMM exited during startup with status 7$", - ), - # It closed the endpoint but outlives the grace period. - ( - closed, - None, - [call(timeout=10), call(timeout=5)], - control_session.ControlEndpointClosed, - "^managed control endpoint closed$", - ), - # It still serves the endpoint, so it gets no grace period. - ( - TimeoutError("managed control response timed out"), - None, - [call(timeout=5)], - TimeoutError, - "^managed control response timed out$", - ), - # A session reset also comes from an OpenVMM that serves the endpoint. - ( - ConnectionError("managed control session was reset"), - None, - [call(timeout=5)], - ConnectionError, - "^managed control session was reset$", - ), - # An interrupt propagates even if OpenVMM exited. - (KeyboardInterrupt(), 0, [], KeyboardInterrupt, ""), - ): - with ( - self.subTest(failure=failure, status=status), - tempfile.TemporaryDirectory() as temporary, - ): - state = _provision_managed_sandbox(Path(temporary).resolve()) - process = _openvmm_mock(status) - session = MagicMock() - session.ping.side_effect = failure - context = MagicMock() - context.__enter__.return_value = session - context.__exit__.return_value = False - with ( - patch.object( - sandbox_lifecycle, "require_file", side_effect=require - ), - patch.object( - sandbox_lifecycle.subprocess, "Popen", return_value=process - ), - patch.object( - sandbox_lifecycle, "_process_start_time", return_value=456 - ), - patch.object( - sandbox_lifecycle.ControlSession, - "connect", - return_value=context, - ), - self.assertRaisesRegex(raised, message), - ): - sandbox_lifecycle.start(state, 10) - - self.assertEqual(process.wait.call_args_list, waits) - self.assertEqual(process.terminate.called, status is None) - self.assertEqual( - sorted(path.name for path in state.iterdir()), - [ - sandbox_lifecycle.CONFIG_NAME, - sandbox_lifecycle.LOCK_NAME, - sandbox_lifecycle.LOG_NAME, - ], - ) - - def test_managed_start_spawns_openvmm_with_a_prepared_capability_pipe(self): - # OpenVMM reads its capability as soon as it starts, so start hands it a - # pipe that already holds the capability and the end of file (#440). - def require(path: Path, _description: str) -> Path: - return path - - capability = bytes(range(1, 33)) - prepared: list[int] = [] - spawned: list[object] = [] - - def prepare(value: bytes) -> int: - self.assertEqual(value, capability) - descriptor = control_session.capability_pipe(value) - prepared.append(descriptor) - return descriptor - - process = MagicMock() - process.pid = 123 - process.stdin = None - - def launch(_command: list[str], *, stdin: object, **_options: object) -> Any: - # The pipe stays open until OpenVMM has started with it. - self.assertIsInstance(stdin, int) - os.fstat(cast(int, stdin)) - spawned.append(stdin) - return process - - session = MagicMock() - context = MagicMock() - context.__enter__.return_value = session - context.__exit__.return_value = False - with tempfile.TemporaryDirectory() as temporary: - state = _provision_managed_sandbox(Path(temporary).resolve()) - with ( - patch.object(sandbox_lifecycle, "require_file", side_effect=require), - patch.object( - sandbox_lifecycle.secrets, "token_bytes", return_value=capability - ), - patch.object(sandbox_lifecycle, "capability_pipe", side_effect=prepare), - patch.object(sandbox_lifecycle.subprocess, "Popen", side_effect=launch), - patch.object( - sandbox_lifecycle, "_process_start_time", return_value=456 - ), - patch.object( - sandbox_lifecycle.ControlSession, "connect", return_value=context - ) as connect, - ): - sandbox_lifecycle.start(state, 10) - - self.assertEqual( - (state / sandbox_lifecycle.CAPABILITY_NAME).read_bytes(), capability - ) - self.assertEqual(len(prepared), 1) - self.assertEqual(spawned, prepared) - # start closes its copy of the pipe once OpenVMM has it. - with self.assertRaises(OSError): - os.fstat(prepared[0]) - self.assertEqual(connect.call_args.args[1], capability) - session.ping.assert_called_once_with(10) - - def test_managed_start_launches_one_openvmm_for_overlapping_starts(self): - # Overlapping starts of one sandbox each launched OpenVMM. The runtime - # record and capability could then name different VMs, so that stop - # ended one VM or neither, and a failed start removed the other's files - # (#464). - if os.name != "nt" and sys.platform != "linux": - self.skipTest("managed sandboxes require a Linux or Windows host") - temporary = tempfile.TemporaryDirectory() - self.addCleanup(temporary.cleanup) - state = _provision_managed_sandbox(Path(temporary.name).resolve()) - popen = subprocess.Popen - launched: list[subprocess.Popen[bytes]] = [] - attempts: list[None] = [] - attempts_lock = threading.Lock() - overtaken = threading.Event() - - def launch(command: list[str], **options: Any) -> subprocess.Popen[bytes]: - with attempts_lock: - first = not attempts - attempts.append(None) - if first: - # Gives the other start time to launch its own OpenVMM before - # this one records its own, as it could without the lock. - overtaken.wait(0.5) - else: - overtaken.set() - # With unknown options, the type checker cannot tell that start - # requests binary streams. - process = cast( - "subprocess.Popen[bytes]", - popen( - [sys.executable, "-c", _SERVING_OPENVMM, *command[1:]], **options - ), - ) - self.addCleanup(_end_child, process) - launched.append(process) - return process - - def require(path: Path, _description: str) -> Path: - return path - - barrier = threading.Barrier(2) - results: list[str] = [] - - def contend() -> None: - barrier.wait() - try: - sandbox_lifecycle.start(state, 30) - results.append("started") - except common.ScriptError as error: - results.append(str(error)) - - with ( - patch.object(sandbox_lifecycle, "require_file", side_effect=require), - patch.object(sandbox_lifecycle.subprocess, "Popen", side_effect=launch), - ): - contenders = [ - threading.Thread(target=contend, daemon=True) for _ in range(2) - ] - for contender in contenders: - contender.start() - for contender in contenders: - contender.join(120) - self.assertFalse(any(contender.is_alive() for contender in contenders)) - - self.assertEqual(len(launched), 1) - self.assertEqual( - sorted(results), - ["sandbox is already running or has stale runtime state", "started"], - ) - runtime = json.loads( - (state / sandbox_lifecycle.RUNTIME_NAME).read_text(encoding="utf-8") - ) - self.assertEqual(runtime["pid"], launched[0].pid) - - # OpenVMM does not inherit the lock, so the next transition can take it - # while the VM runs. - relocked = threading.Event() - - def relock() -> None: - with sandbox_lifecycle._LifecycleLock(state, 30): - relocked.set() - - threading.Thread(target=relock, daemon=True).start() - self.assertTrue(relocked.wait(30)) - - # The capability on record authenticates stop to the only VM, which ends. - outcome = sandbox_lifecycle.stop(state, 30) - self.assertEqual(outcome["outcome"]["category"], "success") - self.assertEqual(launched[0].wait(timeout=30), 0) - sandbox_lifecycle.deprovision(state, 10) - self.assertFalse(state.exists()) - - def test_managed_start_refuses_runtime_files_without_a_record(self): - # A start that was killed can leave its capability or control socket, but - # no runtime record, while its OpenVMM still runs. Start and deprovision - # refuse them rather than launch a second VM for the sandbox or remove - # the files of a VM that may still run. - def require(path: Path, _description: str) -> Path: - return path - - for name in ( - sandbox_lifecycle.CAPABILITY_NAME, - sandbox_lifecycle.CONTROL_SOCKET_NAME, - ): - with ( - self.subTest(name=name), - tempfile.TemporaryDirectory() as temporary, - ): - state = _provision_managed_sandbox(Path(temporary).resolve()) - leftover = state / name - leftover.write_bytes(b"leftover") - with ( - patch.object( - sandbox_lifecycle, "require_file", side_effect=require - ), - patch.object(sandbox_lifecycle.subprocess, "Popen") as popen, - self.assertRaisesRegex( - common.ScriptError, - "^sandbox is already running or has stale runtime state$", - ), - ): - sandbox_lifecycle.start(state, 10) - - popen.assert_not_called() - self.assertEqual(leftover.read_bytes(), b"leftover") - self.assertFalse((state / sandbox_lifecycle.RUNTIME_NAME).exists()) - - with self.assertRaisesRegex( - common.ScriptError, - "^sandbox has runtime files but no runtime record, so its " - "OpenVMM process may still run", - ): - sandbox_lifecycle.deprovision(state, 10) - self.assertEqual(leftover.read_bytes(), b"leftover") - self.assertEqual( - sorted(path.name for path in state.iterdir()), - sorted( - ( - sandbox_lifecycle.CONFIG_NAME, - sandbox_lifecycle.LOCK_NAME, - name, - ) - ), - ) - - # A start creates its capability only if no other holds one. - with tempfile.TemporaryDirectory() as temporary: - capability = Path(temporary) / sandbox_lifecycle.CAPABILITY_NAME - capability.write_bytes(b"leftover") - with self.assertRaisesRegex( - common.ScriptError, - "^sandbox is already running or has stale runtime state$", - ): - sandbox_lifecycle._create_capability(capability, bytes(range(1, 33))) - self.assertEqual(capability.read_bytes(), b"leftover") - - def test_managed_cleanups_remove_the_runtime_record_last(self): - # An interrupted cleanup then leaves a record that shows OpenVMM gone, - # which deprovision accepts, rather than runtime files that it refuses. - unlink = Path.unlink - removed: list[str] = [] - expected = [ - sandbox_lifecycle.CONTROL_SOCKET_NAME, - sandbox_lifecycle.CAPABILITY_NAME, - sandbox_lifecycle.RUNTIME_NAME, - ] - - def tracked_unlink(path: Path, missing_ok: bool = False) -> None: - if path.name in sandbox_lifecycle.RUNTIME_FILE_NAMES: - removed.append(path.name) - unlink(path, missing_ok=missing_ok) - - with tempfile.TemporaryDirectory() as temporary: - state = Path(temporary) - _write_managed_runtime(state, 123, 456) - (state / sandbox_lifecycle.OUTCOME_NAME).write_text( - json.dumps( - { - "schema_version": sandbox_lifecycle.OUTCOME_SCHEMA_VERSION, - "outcome": {"category": "success"}, - "network_policy": {}, - "teardown": {}, - } - ), - encoding="utf-8", - ) - with ( - patch.object( - sandbox_lifecycle, "_process_running", side_effect=(True, False) - ), - patch.object( - sandbox_lifecycle.ControlSession, - "connect", - return_value=MagicMock(), - ), - patch.object(Path, "unlink", autospec=True, side_effect=tracked_unlink), - ): - sandbox_lifecycle.stop(state, 10) - self.assertEqual(removed, expected) - - def require(path: Path, _description: str) -> Path: - return path - - removed.clear() - session = MagicMock() - session.ping.side_effect = TimeoutError("managed control response timed out") - context = MagicMock() - context.__enter__.return_value = session - context.__exit__.return_value = False - with tempfile.TemporaryDirectory() as temporary: - state = _provision_managed_sandbox(Path(temporary).resolve()) - with ( - patch.object(sandbox_lifecycle, "require_file", side_effect=require), - patch.object( - sandbox_lifecycle.subprocess, - "Popen", - return_value=_openvmm_mock(None), - ), - patch.object( - sandbox_lifecycle, "_process_start_time", return_value=456 - ), - patch.object( - sandbox_lifecycle.ControlSession, "connect", return_value=context - ), - patch.object(Path, "unlink", autospec=True, side_effect=tracked_unlink), - self.assertRaisesRegex(TimeoutError, "response timed out"), - ): - sandbox_lifecycle.start(state, 10) - self.assertEqual(removed, expected) - - def test_managed_start_keeps_runtime_files_while_openvmm_may_run(self): - # A failed start that cannot end its OpenVMM process leaves the runtime - # files in place rather than make the sandbox look stopped, so start and - # deprovision refuse it while OpenVMM may still run. - def require(path: Path, _description: str) -> Path: - return path - - process = _openvmm_mock(None) - process.wait.side_effect = subprocess.TimeoutExpired("openvmm", 5) - session = MagicMock() - session.ping.side_effect = TimeoutError("managed control response timed out") - context = MagicMock() - context.__enter__.return_value = session - context.__exit__.return_value = False - with tempfile.TemporaryDirectory() as temporary: - state = _provision_managed_sandbox(Path(temporary).resolve()) - with ( - patch.object(sandbox_lifecycle, "require_file", side_effect=require), - patch.object( - sandbox_lifecycle.subprocess, "Popen", return_value=process - ), - patch.object( - sandbox_lifecycle, "_process_start_time", return_value=456 - ), - patch.object( - sandbox_lifecycle.ControlSession, "connect", return_value=context - ), - self.assertRaises(subprocess.TimeoutExpired), - ): - sandbox_lifecycle.start(state, 10) - - process.kill.assert_called_once_with() - self.assertTrue((state / sandbox_lifecycle.RUNTIME_NAME).is_file()) - self.assertTrue((state / sandbox_lifecycle.CAPABILITY_NAME).is_file()) - with self.assertRaisesRegex( - common.ScriptError, - "^sandbox is already running or has stale runtime state$", - ): - sandbox_lifecycle.start(state, 10) - with ( - patch.object(sandbox_lifecycle, "_process_running", return_value=True), - self.assertRaisesRegex( - common.ScriptError, "^sandbox must be stopped before deprovision$" - ), - ): - sandbox_lifecycle.deprovision(state, 10) - - def test_lifecycle_lock_excludes_other_holders(self): - with tempfile.TemporaryDirectory() as temporary: - state = _provision_managed_sandbox(Path(temporary).resolve()) - acquired = threading.Event() - - def wait_for_lock() -> None: - with sandbox_lifecycle._LifecycleLock(state, 30): - acquired.set() - - waiter = threading.Thread(target=wait_for_lock, daemon=True) - with sandbox_lifecycle._LifecycleLock(state, 30): - waiter.start() - self.assertFalse(acquired.wait(0.5)) - self.assertTrue(acquired.wait(30)) - waiter.join(30) - self.assertFalse(waiter.is_alive()) - self.assertTrue((state / sandbox_lifecycle.LOCK_NAME).is_file()) - - def test_lifecycle_lock_waits_at_most_the_timeout(self): - # A transition queued behind another that does not end fails once its - # own timeout passes, and changes nothing. - with tempfile.TemporaryDirectory() as temporary: - state = _provision_managed_sandbox(Path(temporary).resolve()) - config = (state / sandbox_lifecycle.CONFIG_NAME).read_bytes() - with ( - sandbox_lifecycle._LifecycleLock(state, 30), - patch.object(sandbox_lifecycle.subprocess, "Popen") as popen, - ): - for operation in ( - lambda: sandbox_lifecycle.start(state, 0.2), - lambda: sandbox_lifecycle.stop(state, 0.2), - lambda: sandbox_lifecycle.deprovision(state, 0.2), - ): - began = time.monotonic() - with self.assertRaisesRegex( - TimeoutError, - "^another lifecycle transition of the sandbox is still " - "in progress$", - ): - operation() - self.assertLess(time.monotonic() - began, 10) - popen.assert_not_called() - self.assertEqual( - sorted(path.name for path in state.iterdir()), - [sandbox_lifecycle.CONFIG_NAME, sandbox_lifecycle.LOCK_NAME], - ) - self.assertEqual( - (state / sandbox_lifecycle.CONFIG_NAME).read_bytes(), config - ) - - def test_lifecycle_lock_waiter_fails_after_deprovision(self): - # A transition that waits while deprovision removes the sandbox fails as - # for an unprovisioned sandbox, without opening the lock file again. - if os.name != "nt" and sys.platform != "linux": - self.skipTest("process identity requires a Linux or Windows host") - temporary = tempfile.TemporaryDirectory() - self.addCleanup(temporary.cleanup) - state = _provision_managed_sandbox(Path(temporary.name).resolve()) - # A stale record makes deprovision check its process while it holds the - # lock. - _write_managed_runtime(state, os.getpid(), None) - try_lock_file = sandbox_lifecycle._try_lock_file - open_lock_file = sandbox_lifecycle._open_lock_file - waiting = threading.Event() - waiter_opens: list[Path] = [] - errors: list[BaseException] = [] - - def waiter_thread() -> bool: - return threading.current_thread() is not threading.main_thread() - - def try_lock(descriptor: int) -> bool: - if waiter_thread(): - waiting.set() - return try_lock_file(descriptor) - - def open_lock(path: Path) -> int: - if waiter_thread(): - waiter_opens.append(path) - return open_lock_file(path) - - def stop() -> None: - try: - sandbox_lifecycle.stop(state, 10) - except BaseException as error: # noqa: BLE001 - errors.append(error) - - waiter = threading.Thread(target=stop, daemon=True) - - def process_running(_pid: int, _start_time: int | None) -> bool: - waiter.start() - self.assertTrue(waiting.wait(30)) - return False - - with ( - patch.object(sandbox_lifecycle, "_try_lock_file", side_effect=try_lock), - patch.object(sandbox_lifecycle, "_open_lock_file", side_effect=open_lock), - patch.object( - sandbox_lifecycle, "_process_running", side_effect=process_running - ), - ): - sandbox_lifecycle.deprovision(state, 10) - waiter.join(30) - - self.assertFalse(waiter.is_alive()) - self.assertEqual(len(errors), 1) - self.assertIsInstance(errors[0], common.ScriptError) - self.assertEqual(str(errors[0]), "sandbox is not provisioned") - self.assertEqual(waiter_opens, [state / sandbox_lifecycle.LOCK_NAME]) - self.assertFalse(state.exists()) - - def test_managed_deprovision_releases_its_lock_if_the_lock_file_lingers(self): - # NFS, and Windows file systems without POSIX deletion such as ReFS, keep - # a removed file in its directory while it is open, so the directory - # empties only once deprovision releases the lock. - with tempfile.TemporaryDirectory() as temporary: - state = _provision_managed_sandbox(Path(temporary).resolve()) - rmdir = Path.rmdir - release = sandbox_lifecycle._LifecycleLock.release - events: list[str] = [] - - def lingering_rmdir(path: Path) -> None: - events.append("rmdir") - if events.count("rmdir") == 1: - raise OSError(errno.ENOTEMPTY, "Directory not empty", str(path)) - rmdir(path) - - def tracked_release(lock: sandbox_lifecycle._LifecycleLock) -> None: - events.append("release") - release(lock) - - with ( - patch.object(Path, "rmdir", autospec=True, side_effect=lingering_rmdir), - patch.object( - sandbox_lifecycle._LifecycleLock, - "release", - autospec=True, - side_effect=tracked_release, - ), - ): - sandbox_lifecycle.deprovision(state, 10) - - self.assertEqual(events, ["rmdir", "release", "rmdir", "release"]) - self.assertFalse(state.exists()) - - def test_managed_deprovision_retries_only_while_lock_files_linger(self): - # Deprovision retries removing the emptied directory only while it is not - # empty, at once if the last lingering lock file is gone by then. Any - # other failure, such as a parent that denies the removal, it reports at - # once rather than after its timeout. - def check( - failure: int, failures: int | None, attempts: int, pauses: int - ) -> None: - with tempfile.TemporaryDirectory() as temporary: - state = _provision_managed_sandbox(Path(temporary).resolve()) - rmdir = Path.rmdir - removals: list[Path] = [] - - def failing_rmdir(path: Path) -> None: - if path == state: - removals.append(path) - if failures is None or len(removals) <= failures: - raise OSError(failure, os.strerror(failure), str(path)) - rmdir(path) - - with ( - patch.object( - Path, "rmdir", autospec=True, side_effect=failing_rmdir - ), - patch.object( - sandbox_lifecycle, "_pause", wraps=sandbox_lifecycle._pause - ) as pause, - ): - if failures is None: - with self.assertRaises(PermissionError): - sandbox_lifecycle.deprovision(state, 1) - else: - sandbox_lifecycle.deprovision(state, 30) - - self.assertEqual(len(removals), attempts) - self.assertEqual(pause.call_count, pauses) - if failures is None: - self.assertEqual(list(state.iterdir()), []) - else: - self.assertFalse(state.exists()) - - for name, failure, failures, attempts, pauses in ( - ("last lock file gone", errno.ENOTEMPTY, 2, 3, 0), - ("hidden lock file", errno.ENOTEMPTY, 4, 5, 1), - ("removal denied", errno.EACCES, None, 1, 0), - ): - with self.subTest(name): - check(failure, failures, attempts, pauses) - - def test_managed_deprovision_outlasts_waiters_that_keep_the_lock_file(self): - # On NFS and on Windows file systems without POSIX deletion, each waiter's - # handle keeps the removed lock file in the directory until the waiter - # finds the sandbox gone. Deprovision then retries until they have - # closed it, and no waiter creates a new lock file. - if os.name != "nt" and sys.platform != "linux": - self.skipTest("process identity requires a Linux or Windows host") - temporary = tempfile.TemporaryDirectory() - self.addCleanup(temporary.cleanup) - state = _provision_managed_sandbox(Path(temporary.name).resolve()) - _write_managed_runtime(state, os.getpid(), None) - rmdir = Path.rmdir - try_lock_file = sandbox_lifecycle._try_lock_file - polling: set[int] = set() - polling_lock = threading.Lock() - both_polling = threading.Event() - errors: list[BaseException] = [] - - def try_lock(descriptor: int) -> bool: - if threading.current_thread() is not threading.main_thread(): - with polling_lock: - polling.add(threading.get_ident()) - if len(polling) == 2: - both_polling.set() - return try_lock_file(descriptor) - - def stop() -> None: - try: - sandbox_lifecycle.stop(state, 30) - except BaseException as error: # noqa: BLE001 - errors.append(error) - - waiters = [threading.Thread(target=stop, daemon=True) for _ in range(2)] - - def process_running(_pid: int, _start_time: int | None) -> bool: - for waiter in waiters: - waiter.start() - self.assertTrue(both_polling.wait(30)) - return False - - def lingering_rmdir(path: Path) -> None: - if path == state and any(waiter.is_alive() for waiter in waiters): - raise OSError(errno.ENOTEMPTY, "Directory not empty", str(path)) - rmdir(path) - - with ( - patch.object(sandbox_lifecycle, "_try_lock_file", side_effect=try_lock), - patch.object( - sandbox_lifecycle, "_process_running", side_effect=process_running - ), - patch.object(Path, "rmdir", autospec=True, side_effect=lingering_rmdir), - ): - sandbox_lifecycle.deprovision(state, 30) - for waiter in waiters: - waiter.join(30) - - self.assertFalse(any(waiter.is_alive() for waiter in waiters)) - self.assertEqual( - [str(error) for error in errors], ["sandbox is not provisioned"] * 2 - ) - self.assertFalse(state.exists()) - - def test_managed_deprovision_outlasts_a_transition_that_starts_meanwhile(self): - # A transition that starts after deprovision removed the lock file holds - # a new one in the directory until it finds the sandbox gone, which must - # not make deprovision fail. - with tempfile.TemporaryDirectory() as temporary: - state = _provision_managed_sandbox(Path(temporary).resolve()) - rmdir = Path.rmdir - arrived = threading.Event() - leave = threading.Event() - attempts: list[bool] = [] - - def arrive() -> None: - with sandbox_lifecycle._LifecycleLock(state, 30): - arrived.set() - leave.wait(30) - - arrival = threading.Thread(target=arrive, daemon=True) - - def contended_rmdir(path: Path) -> None: - if path != state: - rmdir(path) - return - if not attempts: - attempts.append(False) - arrival.start() - self.assertTrue(arrived.wait(30)) - raise OSError(errno.ENOTEMPTY, "Directory not empty", str(path)) - try: - rmdir(path) - except OSError: - attempts.append(False) - leave.set() - raise - attempts.append(True) - - with patch.object( - Path, "rmdir", autospec=True, side_effect=contended_rmdir - ): - sandbox_lifecycle.deprovision(state, 30) - arrival.join(30) - - self.assertFalse(arrival.is_alive()) - self.assertEqual(attempts[0], False) - self.assertIn(False, attempts[1:]) - self.assertEqual(attempts[-1], True) - self.assertFalse(state.exists()) - - def test_provision_waiting_on_deprovision_provisions_a_new_directory(self): - # A provision that waits while deprovision removes the sandbox creates - # nothing in the directory being removed. Once deprovision has removed - # it, the provision creates the directory again, and both succeed. - if os.name != "nt" and sys.platform != "linux": - self.skipTest("process identity requires a Linux or Windows host") - temporary = tempfile.TemporaryDirectory() - self.addCleanup(temporary.cleanup) - root = Path(temporary.name).resolve() - state = _provision_managed_sandbox(root) - _write_managed_runtime(state, os.getpid(), None) - rmdir = Path.rmdir - try_lock_file = sandbox_lifecycle._try_lock_file - polling = threading.Event() - removals: list[bool] = [] - errors: list[BaseException] = [] - - def try_lock(descriptor: int) -> bool: - if threading.current_thread() is not threading.main_thread(): - polling.set() - return try_lock_file(descriptor) - - def provision() -> None: - try: - _provision_managed_sandbox(root) - except BaseException as error: # noqa: BLE001 - errors.append(error) - - waiter = threading.Thread(target=provision, daemon=True) - - def process_running(_pid: int, _start_time: int | None) -> bool: - waiter.start() - self.assertTrue(polling.wait(30)) - return False - - def lingering_rmdir(path: Path) -> None: - if path == state and not removals: - # As on NFS, the waiter's handle keeps the removed lock file. - removals.append(False) - raise OSError(errno.ENOTEMPTY, "Directory not empty", str(path)) - rmdir(path) - if path == state: - removals.append(True) - - with ( - patch.object(sandbox_lifecycle, "_try_lock_file", side_effect=try_lock), - patch.object( - sandbox_lifecycle, "_process_running", side_effect=process_running - ), - patch.object(Path, "rmdir", autospec=True, side_effect=lingering_rmdir), - ): - sandbox_lifecycle.deprovision(state, 30) - waiter.join(30) - - self.assertFalse(waiter.is_alive()) - self.assertEqual(errors, []) - self.assertEqual(removals[0], False) - self.assertIn(True, removals[1:]) - self.assertEqual( - sorted(path.name for path in state.iterdir()), - [sandbox_lifecycle.CONFIG_NAME, sandbox_lifecycle.LOCK_NAME], - ) - - def test_provision_waiting_on_deprovision_retries_once_deprovision_ends(self): - # Deprovision clears the mark of the removed lock file once it has removed - # the directory, or failed to remove it or the lock file, so a provision - # that waited for it retries at once. It then finds the sandbox that - # another provision created in between, or provisions the directory that - # deprovision left. - if os.name != "nt" and sys.platform != "linux": - self.skipTest("process identity requires a Linux or Windows host") - - def check(ending: str) -> None: - temporary = tempfile.TemporaryDirectory() - self.addCleanup(temporary.cleanup) - root = Path(temporary.name).resolve() - state = _provision_managed_sandbox(root) - _write_managed_runtime(state, os.getpid(), None) - rmdir = Path.rmdir - unlink = Path.unlink - try_lock_file = sandbox_lifecycle._try_lock_file - polling = threading.Event() - denials: list[Path] = [] - errors: list[BaseException] = [] - - def deny(path: Path) -> None: - denials.append(path) - raise PermissionError( - errno.EACCES, os.strerror(errno.EACCES), str(path) - ) - - def try_lock(descriptor: int) -> bool: - if threading.current_thread() is not threading.main_thread(): - polling.set() - return try_lock_file(descriptor) - - def provision() -> None: - try: - _provision_managed_sandbox(root) - except BaseException as error: # noqa: BLE001 - errors.append(error) - - waiter = threading.Thread(target=provision, daemon=True) - - def process_running(_pid: int, _start_time: int | None) -> bool: - waiter.start() - self.assertTrue(polling.wait(30)) - return False - - def ending_rmdir(path: Path) -> None: - if path == state and ending == "removal denied": - deny(path) - rmdir(path) - if path == state and ending == "replaced": - # Another provision recreates the directory before the waiter - # could find it gone. - _provision_managed_sandbox(root) - - def ending_unlink(path: Path, missing_ok: bool = False) -> None: - # Only the first removal of the lock file fails, so the release - # that follows removes it. - lock = path == state / sandbox_lifecycle.LOCK_NAME - if lock and ending == "unlink denied" and not denials: - deny(path) - unlink(path, missing_ok=missing_ok) - - with ( - patch.object(sandbox_lifecycle, "_try_lock_file", side_effect=try_lock), - patch.object( - sandbox_lifecycle, "_process_running", side_effect=process_running - ), - patch.object(Path, "rmdir", autospec=True, side_effect=ending_rmdir), - patch.object(Path, "unlink", autospec=True, side_effect=ending_unlink), - ): - if ending == "replaced": - sandbox_lifecycle.deprovision(state, 30) - else: - with self.assertRaises(PermissionError): - sandbox_lifecycle.deprovision(state, 30) - waiter.join(30) - - self.assertFalse(waiter.is_alive()) - self.assertEqual(len(denials), 0 if ending == "replaced" else 1) - self.assertEqual( - [str(error) for error in errors], - ["sandbox is already provisioned"] if ending == "replaced" else [], - ) - self.assertEqual( - sorted(path.name for path in state.iterdir()), - [sandbox_lifecycle.CONFIG_NAME, sandbox_lifecycle.LOCK_NAME], - ) - - for ending in ("replaced", "removal denied", "unlink denied"): - with self.subTest(ending): - check(ending) - - def test_provision_handoff_completes_or_ends_the_removal(self): - # While only lingering lock files keep the directory that deprovision - # removes, a provision waits. It stops once the path names no directory or - # another one, or the directory holds anything else, and once nothing - # keeps the directory, it removes it itself, in case deprovision has given - # up. A removal that fails for another reason leaves it to provision the - # directory in place. - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary).resolve() - state = root / "state" - state.mkdir() - lingering = state / sandbox_lifecycle.LOCK_NAME - lingering.write_bytes(b"") - removing = os.lstat(state) - with self.assertRaisesRegex( - TimeoutError, - "^another lifecycle transition of the sandbox is still in progress$", - ): - sandbox_lifecycle._wait_for_handoff( - state, removing, time.monotonic() + 0.1 - ) - - def wait() -> None: - sandbox_lifecycle._wait_for_handoff( - state, removing, time.monotonic() + 30 - ) - - with patch.object(sandbox_lifecycle, "_pause", return_value=False) as pause: - for name in (sandbox_lifecycle.CONFIG_NAME, "foreign"): - (state / name).write_text("{}", encoding="utf-8") - wait() - (state / name).unlink() - # The old directory keeps its identity under another name, so the - # new one cannot reuse it. - state.rename(root / "old") - state.mkdir() - wait() - self.assertTrue(state.is_dir()) - state.rmdir() - (root / "old").rename(state) - lingering.unlink() - with patch.object( - Path, - "rmdir", - autospec=True, - side_effect=PermissionError(errno.EACCES, "denied"), - ): - wait() - self.assertTrue(state.is_dir()) - wait() - self.assertFalse(state.exists()) - wait() - pause.assert_not_called() - - def test_provision_completes_a_removal_that_deprovision_gave_up(self): - # Once a lingering lock file makes deprovision release the lock before the - # directory is gone, a provision that waits for the removal completes it - # itself if deprovision gives up, and then provisions the path again. - if os.name != "nt" and sys.platform != "linux": - self.skipTest("process identity requires a Linux or Windows host") - temporary = tempfile.TemporaryDirectory() - self.addCleanup(temporary.cleanup) - root = Path(temporary.name).resolve() - state = _provision_managed_sandbox(root) - _write_managed_runtime(state, os.getpid(), None) - rmdir = Path.rmdir - try_lock_file = sandbox_lifecycle._try_lock_file - wait_for_handoff = sandbox_lifecycle._wait_for_handoff - polling = threading.Event() - deprovisioned = threading.Event() - attempts: list[Path] = [] - handoffs: list[Path] = [] - errors: list[BaseException] = [] - - def try_lock(descriptor: int) -> bool: - if threading.current_thread() is not threading.main_thread(): - polling.set() - return try_lock_file(descriptor) - - def provision() -> None: - try: - _provision_managed_sandbox(root) - except BaseException as error: # noqa: BLE001 - errors.append(error) - - waiter = threading.Thread(target=provision, daemon=True) - - def process_running(_pid: int, _start_time: int | None) -> bool: - waiter.start() - self.assertTrue(polling.wait(30)) - return False - - def abandoned_rmdir(path: Path) -> None: - # A lock file lingers at deprovision's first attempt, and its retry is - # denied, so that it gives up. - if path == state and threading.current_thread() is threading.main_thread(): - attempts.append(path) - if len(attempts) == 1: - raise OSError(errno.ENOTEMPTY, os.strerror(errno.ENOTEMPTY)) - raise PermissionError(errno.EACCES, os.strerror(errno.EACCES)) - rmdir(path) - - def handoff(state_dir: Path, removing: os.stat_result, deadline: float) -> None: - handoffs.append(state_dir) - if not deprovisioned.wait(30): - raise AssertionError("deprovision did not give up") - wait_for_handoff(state_dir, removing, deadline) - - with ( - patch.object(sandbox_lifecycle, "_try_lock_file", side_effect=try_lock), - patch.object( - sandbox_lifecycle, "_process_running", side_effect=process_running - ), - patch.object(Path, "rmdir", autospec=True, side_effect=abandoned_rmdir), - patch.object(sandbox_lifecycle, "_wait_for_handoff", side_effect=handoff), - ): - with self.assertRaises(PermissionError): - sandbox_lifecycle.deprovision(state, 30) - deprovisioned.set() - waiter.join(30) - - self.assertFalse(waiter.is_alive()) - self.assertEqual(errors, []) - self.assertEqual(attempts, [state, state]) - self.assertEqual(handoffs, [state]) - self.assertEqual( - sorted(path.name for path in state.iterdir()), - [sandbox_lifecycle.CONFIG_NAME, sandbox_lifecycle.LOCK_NAME], - ) - - def test_managed_deprovision_accepts_a_removal_that_a_provision_completed(self): - # A provision that waits for deprovision to remove the directory may - # remove it first and provision the path again. Deprovision then succeeds - # rather than report the new sandbox as provisioned again, which it tells - # apart even where a new directory could reuse the old inode number. - if os.name != "nt" and sys.platform != "linux": - self.skipTest("process identity requires a Linux or Windows host") - temporary = tempfile.TemporaryDirectory() - self.addCleanup(temporary.cleanup) - root = Path(temporary.name).resolve() - state = _provision_managed_sandbox(root) - _write_managed_runtime(state, os.getpid(), None) - rmdir = Path.rmdir - try_lock_file = sandbox_lifecycle._try_lock_file - polling = threading.Event() - provisioned = threading.Event() - attempts: list[Path] = [] - errors: list[BaseException] = [] - - def try_lock(descriptor: int) -> bool: - if threading.current_thread() is not threading.main_thread(): - polling.set() - return try_lock_file(descriptor) - - def provision() -> None: - try: - _provision_managed_sandbox(root) - except BaseException as error: # noqa: BLE001 - errors.append(error) - finally: - provisioned.set() - - waiter = threading.Thread(target=provision, daemon=True) - - def process_running(_pid: int, _start_time: int | None) -> bool: - waiter.start() - self.assertTrue(polling.wait(30)) - return False - - def contended_rmdir(path: Path) -> None: - if path == state and threading.current_thread() is threading.main_thread(): - attempts.append(path) - if len(attempts) == 1: - raise OSError(errno.ENOTEMPTY, os.strerror(errno.ENOTEMPTY)) - # Deprovision retries only once the waiting provision has removed - # the directory and provisioned the path again. - self.assertTrue(provisioned.wait(30)) - rmdir(path) - - with ( - patch.object(sandbox_lifecycle, "_try_lock_file", side_effect=try_lock), - patch.object( - sandbox_lifecycle, "_process_running", side_effect=process_running - ), - patch.object(Path, "rmdir", autospec=True, side_effect=contended_rmdir), - ): - sandbox_lifecycle.deprovision(state, 30) - waiter.join(30) - - self.assertFalse(waiter.is_alive()) - self.assertEqual(errors, []) - self.assertEqual(attempts, [state, state]) - self.assertEqual( - sorted(path.name for path in state.iterdir()), - [sandbox_lifecycle.CONFIG_NAME, sandbox_lifecycle.LOCK_NAME], - ) - - def test_lifecycle_lock_file_transfers_complete_despite_short_counts(self): - # The tombstone must be complete before deprovision removes the lock file, - # and a waiter must read all of it, even if os.write() or os.read() - # transfers only part of it at a time. - with tempfile.TemporaryDirectory() as temporary: - descriptor = sandbox_lifecycle._open_lock_file( - Path(temporary) / sandbox_lifecycle.LOCK_NAME - ) - try: - self._check_lock_file_transfers(descriptor) - finally: - os.close(descriptor) - - def _check_lock_file_transfers(self, descriptor: int) -> None: - write = os.write - read = os.read - tombstone = sandbox_lifecycle.LOCK_TOMBSTONE - - def short_write(fd: int, data: memoryview) -> int: - return write(fd, bytes(data[:3])) - - def short_read(fd: int, size: int) -> bytes: - return read(fd, min(size, 3)) - - with ( - patch.object( - sandbox_lifecycle.os, "write", side_effect=short_write - ) as writes, - patch.object(sandbox_lifecycle.os, "read", side_effect=short_read), - ): - sandbox_lifecycle._write_lock_file(descriptor, tombstone) - self.assertEqual( - sandbox_lifecycle._read_lock_file(descriptor, len(tombstone)), - tombstone, - ) - self.assertEqual(writes.call_count, -(-len(tombstone) // 3)) - self.assertEqual(os.lseek(descriptor, 0, os.SEEK_CUR), 0) - with ( - patch.object(sandbox_lifecycle.os, "write", return_value=0), - self.assertRaisesRegex(OSError, "made no progress"), - ): - sandbox_lifecycle._write_lock_file(descriptor, tombstone) - - def test_managed_deprovision_reports_a_provision_that_starts_meanwhile(self): - # A provision that starts after deprovision removed the lock file, and - # before it removed the directory, provisions the directory again first. - # Deprovision then reports that rather than remove the new sandbox. - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary).resolve() - state = _provision_managed_sandbox(root) - rmdir = Path.rmdir - provisioned: list[Path] = [] - - def contended_rmdir(path: Path) -> None: - if path == state and not provisioned: - arrival = threading.Thread( - target=lambda: provisioned.append( - _provision_managed_sandbox(root) - ), - daemon=True, - ) - arrival.start() - arrival.join(30) - rmdir(path) - - with ( - patch.object(Path, "rmdir", autospec=True, side_effect=contended_rmdir), - self.assertRaisesRegex( - common.ScriptError, - "^sandbox was provisioned again while deprovision removed it$", - ), - ): - sandbox_lifecycle.deprovision(state, 10) - - self.assertEqual(provisioned, [state]) - self.assertEqual( - sorted(path.name for path in state.iterdir()), - [sandbox_lifecycle.CONFIG_NAME, sandbox_lifecycle.LOCK_NAME], - ) - - def test_lifecycle_deadlines_reject_retries_after_they_pass(self): - # Each wait sleeps at most until its deadline, and no retry starts after - # it, so neither a directory that deprovision removes only then nor a lock - # file that another transition replaces only then counts. - self.assertFalse(sandbox_lifecycle._pause(time.monotonic() - 1)) - with tempfile.TemporaryDirectory() as temporary: - directory = Path(temporary) / "going" - directory.mkdir() - # A lock file lingers in the directory until the deadline. - lingering = directory / sandbox_lifecycle.LOCK_NAME - lingering.write_bytes(b"") - deadline = 3 / 64 - clock = [0.0] - sleeps: list[float] = [] - - def sleep(seconds: float) -> None: - sleeps.append(seconds) - clock[0] += seconds - if clock[0] >= deadline and directory.exists(): - lingering.unlink() - directory.rmdir() - - with ( - patch.object(sandbox_lifecycle, "LOCK_RETRY_INTERVAL", 1 / 32), - patch.object( - sandbox_lifecycle.time, "monotonic", side_effect=lambda: clock[0] - ), - patch.object(sandbox_lifecycle.time, "sleep", side_effect=sleep), - self.assertRaisesRegex( - TimeoutError, - "^another lifecycle transition of the sandbox is still in " - "progress$", - ), - ): - sandbox_lifecycle._wait_for_handoff( - directory, os.lstat(directory), deadline - ) - - self.assertEqual(sleeps, [1 / 32, 1 / 64]) - self.assertFalse(directory.exists()) - - with tempfile.TemporaryDirectory() as temporary: - state = _provision_managed_sandbox(Path(temporary).resolve()) - names_open_file = sandbox_lifecycle._names_open_file - clock = [0.0] - checks: list[Path] = [] - - def replaced_late(path: Path, descriptor: int) -> bool: - checks.append(path) - if len(checks) > 1: - return names_open_file(path, descriptor) - clock[0] = 1.0 - return False - - with ( - patch.object( - sandbox_lifecycle.time, "monotonic", side_effect=lambda: clock[0] - ), - patch.object( - sandbox_lifecycle, "_names_open_file", side_effect=replaced_late - ), - self.assertRaisesRegex( - TimeoutError, - "^another lifecycle transition of the sandbox is still in " - "progress$", - ), - sandbox_lifecycle._LifecycleLock(state, 1), - ): - self.fail("the lock was acquired after its deadline") - - self.assertEqual(checks, [state / sandbox_lifecycle.LOCK_NAME]) - self.assertEqual( - sorted(path.name for path in state.iterdir()), - [sandbox_lifecycle.CONFIG_NAME, sandbox_lifecycle.LOCK_NAME], - ) - - def test_provision_recreates_a_state_directory_that_deprovision_removed(self): - # A deprovision can remove the state directory after provision prepared - # it and before provision opens the lock file in it. The provision then - # creates the directory again instead of failing. - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary).resolve() - state = root / "state" - open_lock_file = sandbox_lifecycle._open_lock_file - opened: list[Path] = [] - - def open_after_deprovision(path: Path) -> int: - if not opened: - shutil.rmtree(state) - opened.append(path) - return open_lock_file(path) - - with patch.object( - sandbox_lifecycle, "_open_lock_file", side_effect=open_after_deprovision - ): - self.assertEqual(_provision_managed_sandbox(root), state) - - self.assertEqual(len(opened), 2) - self.assertEqual( - sorted(path.name for path in state.iterdir()), - [sandbox_lifecycle.CONFIG_NAME, sandbox_lifecycle.LOCK_NAME], - ) - - def test_provision_waits_out_a_lock_file_pending_deletion(self): - # Without POSIX deletion semantics, Windows denies opening a lock file - # that deprovision removed until its last handle closes, so a provision - # retries. Elsewhere, such a denial is a real error. - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary).resolve() - open_lock_path = sandbox_lifecycle._open_lock_path - attempts: list[Path] = [] - - def pending_deletion(path: Path) -> int: - attempts.append(path) - if len(attempts) == 1: - raise PermissionError(errno.EACCES, "Access is denied", str(path)) - return open_lock_path(path) - - with patch.object( - sandbox_lifecycle, "_open_lock_path", side_effect=pending_deletion - ): - if os.name == "nt": - _provision_managed_sandbox(root) - self.assertEqual(len(attempts), 2) - self.assertTrue( - (root / "state" / sandbox_lifecycle.CONFIG_NAME).is_file() - ) - else: - with self.assertRaises(PermissionError): - _provision_managed_sandbox(root) - self.assertEqual(len(attempts), 1) - - def test_lifecycle_lock_refuses_a_linked_lock_file(self): - # Deprovision writes to the lock file, so a link to a file elsewhere must - # not stand in for it. - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary).resolve() - state = _provision_managed_sandbox(root) - target = root / "precious" - target.write_bytes(b"precious data") - lock = state / sandbox_lifecycle.LOCK_NAME - links: dict[str, Callable[[], None]] = { - "hard link": lambda: os.link(target, lock), - "symbolic link": lambda: os.symlink(target, lock), - } - for kind, link in links.items(): - with self.subTest(kind=kind): - lock.unlink(missing_ok=True) - try: - link() - except OSError as error: - self.skipTest(f"cannot create a {kind}: {error}") - with self.assertRaisesRegex( - common.ScriptError, - "^sandbox lifecycle lock is not a standalone regular file", - ): - sandbox_lifecycle.deprovision(state, 10) - self.assertEqual(target.read_bytes(), b"precious data") - self.assertTrue((state / sandbox_lifecycle.CONFIG_NAME).is_file()) - - def test_lifecycle_lock_rejects_an_unbounded_timeout(self): - with tempfile.TemporaryDirectory() as temporary: - state = _provision_managed_sandbox(Path(temporary).resolve()) - for timeout in (0.0, -1.0, float("nan"), float("inf")): - with ( - self.subTest(timeout=timeout), - self.assertRaisesRegex( - ValueError, - "^lifecycle timeout must be finite and greater than 0$", - ), - ): - sandbox_lifecycle.stop(state, timeout) - - def test_lifecycle_lock_file_stays_only_in_a_provisioned_sandbox(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary).resolve() - unprovisioned = root / "unprovisioned" - unprovisioned.mkdir() - with self.assertRaisesRegex(common.ScriptError, "sandbox configuration"): - sandbox_lifecycle.start(unprovisioned, 10) - with self.assertRaisesRegex(common.ScriptError, "sandbox is not running"): - sandbox_lifecycle.stop(unprovisioned, 10) - self.assertEqual(list(unprovisioned.iterdir()), []) - - state = _provision_managed_sandbox(root) - self.assertEqual( - sorted(path.name for path in state.iterdir()), - [sandbox_lifecycle.CONFIG_NAME, sandbox_lifecycle.LOCK_NAME], - ) - # A sandbox that an earlier NVX version provisioned gains a lock file. - (state / sandbox_lifecycle.LOCK_NAME).unlink() - with self.assertRaisesRegex(common.ScriptError, "sandbox is not running"): - sandbox_lifecycle.stop(state, 10) - self.assertTrue((state / sandbox_lifecycle.LOCK_NAME).is_file()) - - # Deprovision removes every NVX file, the lock file included, even when - # a file of another owner keeps the directory. - foreign = state / "operator-note.txt" - foreign.write_text("operator data\n", encoding="utf-8") - with self.assertRaisesRegex(common.ScriptError, "operator-note.txt"): - sandbox_lifecycle.deprovision(state, 10) - self.assertEqual(list(state.iterdir()), [foreign]) - foreign.unlink() - sandbox_lifecycle.deprovision(state, 10) - self.assertFalse(state.exists()) - - def test_launch_contract_rejects_disk_option_delimiters(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - layer_path = root / "layer;create=1G" - scratch = root / "scratch.ext4" - layer_path.touch() - scratch.touch() - layer = sandbox.SandboxLayer( - role="distro", - path=layer_path, - uuid="11111111-1111-1111-1111-111111111111", - ) - - with self.assertRaisesRegex(common.ScriptError, "semicolons"): - sandbox.SandboxLaunch( - layers=(layer,), - scratch=scratch, - ).validated() - - -class BenchmarkTests(unittest.TestCase): - def test_kvm_worker_result_decoding(self): - completed = subprocess.CompletedProcess( - ["worker"], - 0, - stdout='noise\nOPENVMM_KVM_E2E_RESULT={"p50_ms": 1.5}\n', - stderr="", - ) - with patch.object(benchmark.subprocess, "run", return_value=completed): - result = benchmark._run_kvm_worker(["worker"], "e2e") - - self.assertEqual(result, {"p50_ms": 1.5}) - - def test_kvm_workers_forward_translated_scratch_directory(self): - with tempfile.TemporaryDirectory() as temporary: - scratch = Path(temporary).resolve() - args = argparse.Namespace( - scratch_dir=scratch, - warmups=1, - runs=2, - memory_mib=128, - processors=4, - host_cpu_reserve=1, - cpus="0-3", - timeout=1.0, - teardown_mode="guest-exit", - net=None, - network_profile=None, - keep_kvm_stage=True, - ) - workers = ( - (benchmark.benchmark_kvm, ""), - (benchmark.benchmark_e2e_kvm, "e2e"), - (benchmark.benchmark_snapshot_restore_kvm, "restore"), - (benchmark.benchmark_snapshot_kvm, "snapshot"), - ) - - def translate(path: Path) -> str: - if path == benchmark.NVX_SCRIPT: - return "/workspace/scripts/nvx.py" - self.assertEqual(path, scratch) - return "/mnt/data/nvx-benchmark-scratch" - - with ( - patch.object(benchmark, "stage_kvm"), - patch.object( - benchmark, - "windows_to_wsl", - side_effect=translate, - ) as translate_path, - patch.object( - benchmark, - "_run_kvm_worker", - return_value={}, - ) as run_worker, - ): - for worker, result_kind in workers: - with self.subTest(result_kind=result_kind or "boot"): - translate_path.reset_mock() - run_worker.reset_mock() - worker( - args, - Path("openvmm"), - Path("kernel"), - Path("initrd"), - ) - - command = run_worker.call_args.args[0] - scratch_index = command.index("--scratch-dir") - self.assertEqual( - command[scratch_index : scratch_index + 2], - [ - "--scratch-dir", - "/mnt/data/nvx-benchmark-scratch", - ], - ) - self.assertEqual( - translate_path.call_args_list, - [call(benchmark.NVX_SCRIPT), call(scratch)], - ) - self.assertEqual( - run_worker.call_args.args[1], - result_kind, - ) - - def test_kvm_worker_defaults_to_translated_system_temporary_directory(self): - with tempfile.TemporaryDirectory() as temporary: - system_temporary = Path(temporary).resolve() - args = argparse.Namespace(scratch_dir=None) - with ( - patch.object( - benchmark.tempfile, - "gettempdir", - return_value=str(system_temporary), - ), - patch.object( - benchmark, - "windows_to_wsl", - return_value="/mnt/c/system-temp", - ) as translate, - ): - self.assertEqual( - benchmark._kvm_worker_scratch_arguments(args), - ["--scratch-dir", "/mnt/c/system-temp"], - ) - - translate.assert_called_once_with(system_temporary) - - def test_require_file_preserves_resolved_path_error(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - missing = root / "nested" / ".." / "missing" - resolved_missing = missing.resolve() - - with self.assertRaises(FileNotFoundError) as raised: - benchmark.require_file(missing, "benchmark artifact") - - self.assertEqual( - str(raised.exception), - f"benchmark artifact not found: {resolved_missing}", - ) - - def test_snapshot_profile_records_are_parsed_and_summarized(self): - record = benchmark.parse_snapshot_profile_line( - b"OPENVMM_SNAPSHOT_PROFILE_V1 operation=restore " - b"phase=artifact_open exclusive=1 duration_ns=1500000 " - b"process_elapsed_ns=2000000 pid=42 logical_bytes=1024 " - b"allocated_bytes=512\r" - ) - - self.assertIsNotNone(record) - assert record is not None - self.assertEqual(record["operation"], "restore") - self.assertEqual(record["duration_ns"], 1_500_000) - self.assertTrue(record["exclusive"]) - summary = benchmark.summarize_lifecycle_profiles( - [{"records": [record, {**record, "duration_ns": 2_500_000}]}] - ) - metric = summary["phases"]["restore.artifact_open"] - self.assertEqual(metric["samples_ms"], [1.5, 2.5]) - self.assertEqual(metric["p50_ms"], 2.0) - self.assertEqual(metric["p95_ms"], 2.5) - self.assertTrue(metric["exclusive"]) - - def test_snapshot_profile_rejects_incomplete_records(self): - with self.assertRaisesRegex(ValueError, "missing"): - benchmark.parse_snapshot_profile_line( - b"OPENVMM_SNAPSHOT_PROFILE_V1 operation=restore phase=open" - ) - - def test_snapshot_profile_records_console_input_dispatch(self): - collector = benchmark.SnapshotProfileCollector(42, 100) - - with patch.object(benchmark, "process_resource_counters", return_value={}): - collector.feed( - b"OPENVMM_SNAPSHOT_PROFILE_V1 operation=capture " - b"phase=input_gate exclusive=1 duration_ns=20 " - b"process_elapsed_ns=120 pid=42\n" - b"OPENVMM_SNAPSHOT_PROFILE_V1 operation=capture " - b"phase=publication_commit exclusive=1 duration_ns=10 " - b"process_elapsed_ns=310 pid=42\n" - ) - sample = collector.finish_capture(200, 260, 300, 400, 450, 4096) - - records = cast(list[dict[str, object]], sample["records"]) - by_phase = {str(record["phase"]): record for record in records} - generation = by_phase["snapshot_generation"] - self.assertEqual(generation["duration_ns"], 210) - self.assertEqual(generation["source"], "openvmm_clock") - self.assertEqual(sample["generation_duration_ns"], 210) - dispatch = by_phase["console_input_dispatch"] - self.assertEqual(dispatch["operation"], "capture") - self.assertEqual(dispatch["phase"], "console_input_dispatch") - self.assertEqual(dispatch["duration_ns"], 60) - self.assertEqual(dispatch["observer_elapsed_ns"], 160) - self.assertEqual(dispatch["logical_bytes"], 4096) - self.assertTrue(dispatch["exclusive"]) - round_trip = by_phase["console_command_round_trip"] - self.assertEqual(round_trip["phase"], "console_command_round_trip") - self.assertEqual(round_trip["duration_ns"], 100) - guest_to_publication = by_phase["guest_dispatch_to_publication"] - self.assertEqual(guest_to_publication["phase"], "guest_dispatch_to_publication") - self.assertEqual(guest_to_publication["duration_ns"], 100) - self.assertEqual(by_phase["request_to_publication"]["duration_ns"], 200) - - def test_snapshot_generation_requires_valid_process_clock_boundaries(self): - gate: dict[str, object] = { - "operation": "capture", - "phase": "input_gate", - "duration_ns": 20, - "process_elapsed_ns": 120, - "pid": 42, - } - publication: dict[str, object] = { - "operation": "capture", - "phase": "publication_commit", - "duration_ns": 10, - "process_elapsed_ns": 310, - "pid": 42, - } - self.assertEqual( - benchmark.snapshot_generation_duration_ns([gate, publication], 42), 210 - ) - for records, error in ( - ([publication], "exactly one 'input_gate'"), - ([gate], "exactly one 'publication_commit'"), - ([gate, gate, publication], "exactly one 'input_gate'"), - ([gate, publication, publication], "exactly one 'publication_commit'"), - ([gate, {**publication, "pid": 43}], "different process"), - ([{**gate, "duration_ns": -1}, publication], "not monotonic"), - ([{**gate, "process_elapsed_ns": 10}, publication], "not monotonic"), - ([gate, {**publication, "process_elapsed_ns": 125}], "not monotonic"), - ): - with self.subTest(records=records): - with self.assertRaisesRegex(ValueError, error): - benchmark.snapshot_generation_duration_ns(records, 42) - - def test_capture_excludes_polling_input_and_matches_profile_generation(self): - for delay_ms in (0, 500, 2000): - for profiled in (False, True): - with self.subTest(delay_ms=delay_ms, profiled=profiled): - self.check_capture_timing(delay_ms, profiled) - - def check_capture_timing(self, delay_ms: int, profiled: bool) -> None: - clock_ns = 0 - writes: list[bytes] = [] - - class FakeProcess: - pid = 42 - returncode = 0 - - def poll(self) -> int: - return 0 - - def wait(self) -> int: - nonlocal clock_ns - clock_ns += 1_000_000 - return 0 - - class FakeInteraction: - process = FakeProcess() - - def read_output(self, _chunks: object) -> None: - pass - - def read_stderr(self, _chunks: object) -> None: - pass - - def write_input(self, data: bytes) -> None: - nonlocal clock_ns - writes.append(data) - if data == b"nvx-snapshot\n": - clock_ns += 20_000_000 - - def close(self) -> None: - pass - - chunks = iter( - ( - ("console", benchmark.BOOT_MARKER + b"\n", 0, False), - ("console", benchmark.SMP_PROBE_COMPLETION_MARKER + b"\n", 1, False), - ( - "console", - benchmark.SNAPSHOT_GUEST_DISPATCH_MARKER + b"\n", - delay_ms, - False, - ), - ( - "stderr", - b"OPENVMM_SNAPSHOT_PROFILE_V1 operation=capture " - b"phase=input_gate exclusive=1 duration_ns=500000 " - b"process_elapsed_ns=500000000 pid=42\n" - b"OPENVMM_SNAPSHOT_PROFILE_V1 operation=capture " - b"phase=publication_commit exclusive=1 duration_ns=100000 " - b"process_elapsed_ns=502500000 pid=42\n", - 3, - True, - ), - ("console", None, 0, False), - ("stderr", None, 0, False), - ) - ) - profiles: list[dict[str, object]] = [] - with tempfile.TemporaryDirectory() as temporary: - snapshot = Path(temporary) / "snapshot" - - def read_chunk(*, timeout: float) -> tuple[str, bytes | None]: - nonlocal clock_ns - del timeout - stream, chunk, advance_ms, publish = next(chunks) - clock_ns += advance_ms * 1_000_000 - if publish: - snapshot.mkdir() - return stream, chunk - - with ( - patch.object( - benchmark, "InteractiveProcess", return_value=FakeInteraction() - ) as interaction, - patch.object(benchmark.threading, "Thread"), - patch.object(benchmark.queue, "Queue") as queues, - patch.object( - benchmark.time, "perf_counter_ns", side_effect=lambda: clock_ns - ), - patch.object(benchmark, "_try_peak_rss", return_value=1024), - patch.object( - benchmark, "process_resource_counters", return_value={} - ) as counters, - ): - queues.return_value.get.side_effect = read_chunk - result = benchmark.capture_snapshot( - ["openvmm"], - snapshot, - processors=1, - timeout=5, - snapshot_profile=profiled, - profile_sink=profiles, - ) - - self.assertEqual(result, (3.0, delay_ms + 23.0, 1.0, 1024)) - self.assertEqual(interaction.call_args.args[0], ["openvmm"]) - self.assertEqual( - interaction.call_args.args[1][benchmark.SNAPSHOT_PROFILE_ENV], "1" - ) - self.assertIs(interaction.call_args.kwargs["separate_stderr"], True) - self.assertEqual(writes[1:], [b"nvx-snapshot\n"]) - self.assertIn( - b"echo OPENVMM-SNAPSHOT-RESTORE-OK\nnvx-exit 0\n", - writes[0], - ) - if profiled: - phases = benchmark.summarize_lifecycle_profiles(profiles)["phases"] - self.assertEqual(phases["capture.snapshot_generation"]["p50_ms"], result[0]) - self.assertEqual( - phases["capture.console_command_round_trip"]["p50_ms"], - delay_ms + 20.0, - ) - else: - counters.assert_not_called() - self.assertEqual(profiles, []) - - # OpenVMM's capture profile records from a Linux/MSHV acceptance run, in - # which the records reached the shared terminal in the middle of the - # guest's dispatch marker line. - INCIDENT_PID = 621736 - INCIDENT_CAPTURE_PROFILE = ( - ("input_gate", 1, 263787, 1422094434, ""), - ("vp_stop_at_io_boundary", 1, 120994, 1422235827, ""), - ("quiesce", 1, 393680, 1422716902, ""), - ("save_state", 1, 647567, 1423376968, ""), - ("mapped_memory_flush", 1, 2700, 1423391067, ""), - ("memory_handle_flush", 1, 1400, 1423507161, " logical_bytes=134217728"), - ( - "publication_state", - 1, - 21499, - 1423577158, - " logical_bytes=2455 allocated_bytes=4096", - ), - ( - "publication_manifest", - 1, - 8899, - 1423643854, - " logical_bytes=1454 allocated_bytes=4096", - ), - ( - "publication_memory", - 1, - 39798, - 1423694652, - " logical_bytes=134217728 allocated_bytes=65011712", - ), - ("publication_staging_sync", 1, 3000, 1423709951, ""), - ("publication_commit", 1, 12700, 1423734150, ""), - ("publication_parent_sync", 1, 2700, 1423748449, ""), - ("publication", 0, 235088, 1423760848, " logical_bytes=134217728"), - ) - - @classmethod - def incident_capture_records(cls) -> bytes: - return b"".join( - ( - f"OPENVMM_SNAPSHOT_PROFILE_V1 operation=capture phase={phase} " - f"exclusive={exclusive} duration_ns={duration} " - f"process_elapsed_ns={elapsed} pid={cls.INCIDENT_PID}{extra}\n" - ).encode() - for phase, exclusive, duration, elapsed, extra in ( - cls.INCIDENT_CAPTURE_PROFILE - ) - ) - - def test_capture_observes_dispatch_marker_split_by_profile_records(self): - records = self.incident_capture_records() - committed = b"INFO microVM snapshot committed; terminating source process\n" - snapshot_requested = threading.Event() - console_split = threading.Event() - stderr_written = threading.Event() - - class FakeProcess: - pid = BenchmarkTests.INCIDENT_PID - returncode = 0 - - def poll(self) -> None: - return None - - def wait(self) -> int: - return 0 - - class FakeInteraction: - def __init__(self, snapshot: Path) -> None: - self.process = FakeProcess() - self.snapshot = snapshot - self.writes: list[bytes] = [] - - def read_output(self, chunks: queue.Queue[bytes | None]) -> None: - chunks.put(benchmark.BOOT_MARKER + b"\r\n") - chunks.put(benchmark.SMP_PROBE_COMPLETION_MARKER + b"\r\n") - if snapshot_requested.wait(5): - # The console relay delivers the dispatch line in two - # writes, around OpenVMM's capture records on stderr. - chunks.put(b"nvx-snapshot\r\nNVX-S") - console_split.set() - if stderr_written.wait(5): - chunks.put(b"NAPSHOT-DISPATCHED\r\n") - chunks.put(None) - - def read_stderr(self, chunks: queue.Queue[bytes | None]) -> None: - if console_split.wait(5): - chunks.put(records) - self.snapshot.mkdir() - chunks.put(committed) - stderr_written.set() - chunks.put(None) - - def write_input(self, data: bytes) -> None: - self.writes.append(data) - if data == b"nvx-snapshot\n": - snapshot_requested.set() - - def close(self) -> None: - pass - - profiles: list[dict[str, object]] = [] - with tempfile.TemporaryDirectory() as temporary: - snapshot = Path(temporary) / "snapshot" - log_path = Path(temporary) / "capture.log" - interaction = FakeInteraction(snapshot) - with ( - patch.object( - benchmark, "InteractiveProcess", return_value=interaction - ) as spawn, - patch.object(benchmark, "_try_peak_rss", return_value=1024), - patch.object(benchmark, "process_resource_counters", return_value={}), - patch.object(benchmark, "terminate"), - ): - result = benchmark.capture_snapshot( - ["openvmm"], - snapshot, - processors=1, - timeout=10, - snapshot_profile=True, - profile_sink=profiles, - log_path=log_path, - ) - log = log_path.read_bytes() - - self.assertIs(spawn.call_args.kwargs["separate_stderr"], True) - self.assertEqual(result[0], 1_903_503 / 1_000_000) - self.assertEqual(result[3], 1024) - self.assertEqual(interaction.writes[1:], [b"nvx-snapshot\n"]) - sample = cast(list[dict[str, object]], profiles[0]["records"]) - self.assertEqual( - [record["phase"] for record in sample if record["source"] == "openvmm"], - [phase for phase, *_ in self.INCIDENT_CAPTURE_PROFILE], - ) - self.assertTrue( - benchmark.contains_output_line( - log, benchmark.SNAPSHOT_GUEST_DISPATCH_MARKER - ) - ) - for line in [*records.splitlines(), committed.removesuffix(b"\n")]: - self.assertTrue(benchmark.contains_output_line(log, line), line) - - def test_capture_samples_storage_from_the_request_through_source_exit(self): - records = self.incident_capture_records() - snapshot_requested = threading.Event() - - class FakeProcess: - pid = BenchmarkTests.INCIDENT_PID - returncode = 0 - - def poll(self) -> None: - return None - - def wait(self) -> int: - return 0 - - class FakeInteraction: - def __init__(self, snapshot: Path) -> None: - self.process = FakeProcess() - self.snapshot = snapshot - self.writes: list[bytes] = [] - - def read_output(self, chunks: queue.Queue[bytes | None]) -> None: - chunks.put(benchmark.BOOT_MARKER + b"\r\n") - chunks.put(benchmark.SMP_PROBE_COMPLETION_MARKER + b"\r\n") - if snapshot_requested.wait(5): - chunks.put( - b"nvx-snapshot\r\n" - + benchmark.SNAPSHOT_GUEST_DISPATCH_MARKER - + b"\r\n" - ) - chunks.put(None) - - def read_stderr(self, chunks: queue.Queue[bytes | None]) -> None: - if snapshot_requested.wait(5): - chunks.put(records) - self.snapshot.mkdir() - chunks.put(None) - - def write_input(self, data: bytes) -> None: - self.writes.append(data) - if data == b"nvx-snapshot\n": - snapshot_requested.set() - - def close(self) -> None: - pass - - class FakeTelemetry: - def __init__(self, interaction: FakeInteraction) -> None: - self.interaction = interaction - self.events: list[str] = [] - self.windows: dict[str, tuple[int, int]] = {} - - def start(self) -> None: - self.events.append(f"start after {len(self.interaction.writes)}") - - def stop(self) -> None: - self.events.append("stop") - - def result(self, windows: dict[str, tuple[int, int]]) -> dict[str, object]: - self.events.append("result") - self.windows = windows - return {"phases": {}} - - for retained in (True, False): - with ( - self.subTest(retained=retained), - tempfile.TemporaryDirectory() as temporary, - ): - snapshot_requested.clear() - snapshot = Path(temporary) / "snapshot" - interaction = FakeInteraction(snapshot) - telemetry = FakeTelemetry(interaction) - profiles: list[dict[str, object]] = [] - with ( - patch.object( - benchmark, "InteractiveProcess", return_value=interaction - ), - patch.object(benchmark, "_try_peak_rss", return_value=1024), - patch.object( - benchmark, "process_resource_counters", return_value={} - ), - patch.object(benchmark, "terminate"), - patch.object( - benchmark.host_telemetry, - "create_storage_telemetry", - return_value=telemetry, - ) as create, - ): - benchmark.capture_snapshot( - ["openvmm"], - snapshot, - processors=1, - timeout=10, - snapshot_profile=True, - profile_sink=profiles if retained else None, - ) - - if not retained: - # Warmups keep no profile, so they sample nothing. - create.assert_not_called() - continue - volumes, origin = create.call_args.args - self.assertEqual(volumes["scratch"], Path(temporary)) - self.assertEqual(volumes["workspace"], BuildConstants.REPO_ROOT) - self.assertIsInstance(origin, int) - # Sampling starts after the SMP probe script, before the request. - self.assertEqual( - telemetry.events, ["start after 1", "stop", "result", "stop"] - ) - self.assertEqual(interaction.writes[1:], [b"nvx-snapshot\n"]) - flush_start, flush_end = telemetry.windows[ - "capture.mapped_memory_flush" - ] - self.assertEqual(flush_end - flush_start, 2700) - generation_start, generation_end = telemetry.windows[ - "capture.snapshot_generation" - ] - self.assertEqual(generation_end - generation_start, 1_903_503) - self.assertEqual(profiles[0]["storage_telemetry"], {"phases": {}}) - - def test_profile_phase_windows_use_the_tightest_clock_offset(self): - records: list[dict[str, object]] = [ - { - "operation": "capture", - "phase": "save_state", - "duration_ns": 1_000, - "process_elapsed_ns": 5_000, - "observer_elapsed_ns": 20_500, - "source": "openvmm", - }, - { - "operation": "capture", - "phase": "mapped_memory_flush", - "duration_ns": 900_000, - "process_elapsed_ns": 905_000, - "observer_elapsed_ns": 920_000, - "source": "openvmm", - }, - { - "operation": "capture", - "phase": "snapshot_generation", - "duration_ns": 950_000, - "process_elapsed_ns": 955_000, - "source": "openvmm_clock", - }, - { - "operation": "capture", - "phase": "source_teardown", - "duration_ns": 9_000, - "observer_elapsed_ns": 990_000, - "source": "benchmark_observer", - }, - ] - - # The save_state record arrived 0.5 us late, so the flush record's - # 15 us offset bounds OpenVMM's clock. - self.assertEqual( - benchmark.snapshot_profile_phase_windows(records), - { - "capture.save_state": (19_000, 20_000), - "capture.mapped_memory_flush": (20_000, 920_000), - "capture.snapshot_generation": (20_000, 970_000), - "capture.source_teardown": (981_000, 990_000), - }, - ) - - def test_capture_telemetry_line_describes_the_ram_flush_window(self): - records = [ - { - "operation": "capture", - "phase": "mapped_memory_flush", - "duration_ns": 1_165_314_800, - } - ] - sample: dict[str, object] = { - "records": records, - "storage_telemetry": { - "volumes": {"F:": ["scratch"]}, - "phases": { - "capture.mapped_memory_flush": { - "volumes": { - "F:": { - "write_mib_per_second": 128.44, - "average_write_latency_ms": 31.2, - "average_queue_length": 3.94, - "busy_percent": 99.4, - } - }, - "cpu": {"busy_percent": 6.2}, - } - }, - }, - } - - self.assertEqual( - benchmark.describe_capture_telemetry(sample), - "mapped_memory_flush 1165.315 ms: scratch F: write 128.4 MiB/s at " - "31.2 ms, queue 3.9, 99% busy; CPU 6% busy", - ) - self.assertIsNone(benchmark.describe_capture_telemetry({"records": records})) - - def test_result_documents_record_host_provenance(self): - args = nvx.parse_args(["benchmark"]) - with ( - patch.object(benchmark, "_git_revision", return_value="revision"), - patch.object( - benchmark, - "benchmark_host_provenance", - return_value={"runner_name": "azure-windows-3"}, - ) as host, - ): - document = benchmark.result_document(args, None, None) - - host.assert_called_once_with(args) - self.assertEqual(document["host"], {"runner_name": "azure-windows-3"}) - - def test_host_provenance_names_workspace_temporary_and_scratch_volumes(self): - with tempfile.TemporaryDirectory() as temporary: - scratch = Path(temporary).resolve() / "scratch" - scratch.mkdir() - args = nvx.parse_args(["benchmark", "--scratch-dir", str(scratch)]) - system_temporary = Path(tempfile.gettempdir()).resolve() - with ( - benchmark.benchmark_scratch_directory(args), - patch.object( - benchmark.host_telemetry, - "host_provenance", - return_value={"schema_version": 1}, - ) as provenance, - ): - host = benchmark.benchmark_host_provenance(args) - - self.assertEqual(host, {"schema_version": 1}) - provenance.assert_called_once_with( - { - "workspace": BuildConstants.REPO_ROOT, - "temporary": system_temporary, - "scratch": scratch, - } - ) - - def test_device_io_resume_ignores_host_provenance(self): - args = nvx.parse_args( - [ - "benchmark", - "--suite", - "device-io", - "--backend", - "whp", - "--platform", - "windows-whp-baremetal", - ] - ) - benchmark.apply_benchmark_suite_defaults(args) - hosts = iter(({"machine_name": "first"}, {"machine_name": "second"})) - - def host_provenance(_: argparse.Namespace) -> dict[str, str]: - return next(hosts, {"machine_name": "third"}) - - with ( - tempfile.TemporaryDirectory() as temporary, - patch.object( - benchmark, "benchmark_host_provenance", side_effect=host_provenance - ), - patch.object(benchmark, "sha256_file", return_value="0" * 64), - patch.object( - benchmark, - "_device_io_helper_provenance", - return_value={"source": "helper"}, - ), - patch.object(benchmark, "_git_revision", return_value="revision"), - patch.object(benchmark, "_git_status", return_value=[]), - ): - output = Path(temporary) - - def write() -> Path: - return benchmark.write_benchmark_metadata( - args, - output, - Path("openvmm.exe"), - Path("vmlinux"), - Path("initramfs.cpio.gz"), - "whp", - ) - - path = write() - written = path.read_text(encoding="utf-8") - self.assertEqual(json.loads(written)["host"], {"machine_name": "first"}) - self.assertEqual(write(), path) - self.assertEqual(path.read_text(encoding="utf-8"), written) - args.device_io_port += 1 - with self.assertRaisesRegex( - ValueError, "resume controls or provenance do not match" - ): - write() - - def test_measure_once_observes_restore_marker_split_by_profile_record(self): - record = ( - b"OPENVMM_SNAPSHOT_PROFILE_V1 operation=restore " - b"phase=guest_repair_gate exclusive=0 duration_ns=1500 " - b"process_elapsed_ns=90000000 pid=4321\n" - ) - console_split = threading.Event() - stderr_written = threading.Event() - - class FakeInteraction: - def __init__(self) -> None: - self.process = MagicMock(pid=4321) - self.process.poll.return_value = None - - def read_output(self, chunks: queue.Queue[bytes | None]) -> None: - chunks.put(b"OPENVMM-SNAP") - console_split.set() - if stderr_written.wait(5): - chunks.put(b"SHOT-RESTORE-OK\r\n") - chunks.put(None) - - def read_stderr(self, chunks: queue.Queue[bytes | None]) -> None: - if console_split.wait(5): - chunks.put(record) - stderr_written.set() - chunks.put(None) - - def write_input(self, data: bytes) -> None: - raise AssertionError(f"unexpected input: {data!r}") - - def close(self) -> None: - pass - - profiles: list[dict[str, object]] = [] - with tempfile.TemporaryDirectory() as temporary: - log_path = Path(temporary) / "restore.log" - with ( - patch.object( - benchmark, "InteractiveProcess", return_value=FakeInteraction() - ) as spawn, - patch.object(benchmark, "live_peak_rss_bytes", return_value=1024), - patch.object(benchmark, "wait_for_process_exit", return_value=0), - patch.object(benchmark, "process_resource_counters", return_value={}), - ): - result = benchmark.measure_once( - ["openvmm"], - environment={}, - timeout=10, - marker=benchmark.RESTORE_MARKER, - marker_must_be_line=True, - guest_exit_prequeued=True, - snapshot_profile=True, - profile_sink=profiles, - log_path=log_path, - ) - log = log_path.read_bytes() - - self.assertIs(spawn.call_args.kwargs["separate_stderr"], True) - self.assertEqual(result[1], 1024) - records = cast(list[dict[str, object]], profiles[0]["records"]) - self.assertIn("guest_repair_gate", [item["phase"] for item in records]) - self.assertTrue(benchmark.contains_output_line(log, benchmark.RESTORE_MARKER)) - self.assertTrue(benchmark.contains_output_line(log, record.removesuffix(b"\n"))) - - def test_measure_once_keeps_profile_records_observed_after_the_marker(self): - records = ( - b"OPENVMM_SNAPSHOT_PROFILE_V1 operation=startup " - b"phase=worker_launch exclusive=0 duration_ns=2000000 " - b"process_elapsed_ns=5000000 pid=4321\n" - b"OPENVMM_SNAPSHOT_PROFILE_V1 operation=restore " - b"phase=device_start exclusive=1 duration_ns=300000 " - b"process_elapsed_ns=80000000 pid=4321\n" - b"OPENVMM_SNAPSHOT_PROFILE_V1 operation=restore " - b"phase=guest_repair_gate exclusive=0 duration_ns=1500 " - b"process_elapsed_ns=90000000 pid=4321\n" - ) - torn_down = threading.Event() - - class FakeInteraction: - def __init__(self) -> None: - self.process = MagicMock(pid=4321) - self.process.poll.return_value = None - - def read_output(self, chunks: queue.Queue[bytes | None]) -> None: - chunks.put(benchmark.RESTORE_MARKER + b"\r\n") - chunks.put(None) - - def read_stderr(self, chunks: queue.Queue[bytes | None]) -> None: - # The stderr reader falls behind: OpenVMM's records arrive - # only once the guest marker has been handled. - if torn_down.wait(5): - chunks.put(records[:100]) - chunks.put(records[100:]) - chunks.put(None) - - def write_input(self, data: bytes) -> None: - raise AssertionError(f"unexpected input: {data!r}") - - def close(self) -> None: - pass - - def counters(_pid: int) -> dict[str, int]: - return {} if torn_down.is_set() else {"rss_bytes": 7} - - def wait_for_exit(_process: object, _timeout: float) -> int: - torn_down.set() - return 0 - - profiles: list[dict[str, object]] = [] - with ( - patch.object( - benchmark, "InteractiveProcess", return_value=FakeInteraction() - ), - patch.object(benchmark, "live_peak_rss_bytes", return_value=1024), - patch.object(benchmark, "wait_for_process_exit", side_effect=wait_for_exit), - patch.object(benchmark, "process_resource_counters", side_effect=counters), - ): - benchmark.measure_once( - ["openvmm"], - environment={}, - timeout=10, - marker=benchmark.RESTORE_MARKER, - marker_must_be_line=True, - guest_exit_prequeued=True, - snapshot_profile=True, - profile_sink=profiles, - ) - - by_phase = { - str(record["phase"]): record - for record in cast(list[dict[str, object]], profiles[0]["records"]) - } - self.assertLessEqual( - {"process_startup", "worker_launch", "device_start", "guest_repair_gate"}, - set(by_phase), - ) - self.assertEqual(by_phase["process_startup"]["duration_ns"], 3_000_000) - # The device_start observation followed the marker, so it cannot bound - # the interval to readiness. - self.assertNotIn("resume_to_readiness", by_phase) - readiness = by_phase["process_launch_to_readiness"] - self.assertEqual(readiness["host_counters"], {"rss_bytes": 7}) - - def test_measure_once_reads_output_to_its_end_before_finishing(self): - record = ( - b"OPENVMM_SNAPSHOT_PROFILE_V1 operation=restore " - b"phase=guest_repair_gate exclusive=0 duration_ns=1500 " - b"process_elapsed_ns=90000000 pid=4321\n" - ) - split = record.index(b"duration_ns=") + len(b"duration_ns=") - - class FakeInteraction: - def __init__(self, exited: threading.Event) -> None: - self.process = MagicMock(pid=4321) - self.process.poll.return_value = None - self.exited = exited - - def read_output(self, chunks: queue.Queue[bytes | None]) -> None: - chunks.put(benchmark.RESTORE_MARKER + b"\r\n") - chunks.put(None) - - def read_stderr(self, chunks: queue.Queue[bytes | None]) -> None: - if self.exited.wait(5): - chunks.put(record[:split]) - # The rest of the record outlasts the bounded drain that - # error paths use. - time.sleep(0.2) - chunks.put(record[split:]) - chunks.put(None) - - def write_input(self, data: bytes) -> None: - raise AssertionError(f"unexpected input: {data!r}") - - def close(self) -> None: - pass - - for profiled in (False, True): - with self.subTest(profiled=profiled): - exited = threading.Event() - - def wait_for_exit( - _process: object, _timeout: float, exited: threading.Event = exited - ) -> int: - exited.set() - return 0 - - profiles: list[dict[str, object]] = [] - with tempfile.TemporaryDirectory() as temporary: - log_path = Path(temporary) / "restore.log" - with ( - patch.object( - benchmark, - "InteractiveProcess", - return_value=FakeInteraction(exited), - ), - patch.object(benchmark, "OUTPUT_DRAIN_TIMEOUT_SECONDS", 0.05), - patch.object( - benchmark, "live_peak_rss_bytes", return_value=1024 - ), - patch.object( - benchmark, - "wait_for_process_exit", - side_effect=wait_for_exit, - ), - patch.object( - benchmark, "process_resource_counters", return_value={} - ), - ): - benchmark.measure_once( - ["openvmm"], - environment={}, - timeout=10, - marker=benchmark.RESTORE_MARKER, - marker_must_be_line=True, - guest_exit_prequeued=True, - snapshot_profile=profiled, - profile_sink=profiles if profiled else None, - log_path=log_path, - ) - log = log_path.read_bytes() - - self.assertTrue( - benchmark.contains_output_line(log, record.removesuffix(b"\n")) - ) - if profiled: - records = cast(list[dict[str, object]], profiles[0]["records"]) - repair = next( - item for item in records if item["phase"] == "guest_repair_gate" - ) - self.assertEqual(repair["duration_ns"], 1500) - self.assertEqual(repair["process_elapsed_ns"], 90_000_000) - else: - self.assertEqual(profiles, []) - - def test_measure_once_reports_output_that_does_not_reach_eof(self): - release = threading.Event() - self.addCleanup(release.set) - - class FakeInteraction: - def __init__(self) -> None: - self.process = MagicMock(pid=4321) - self.process.poll.return_value = None - - def read_output(self, chunks: queue.Queue[bytes | None]) -> None: - chunks.put(benchmark.RESTORE_MARKER + b"\r\n") - chunks.put(None) - - def read_stderr(self, chunks: queue.Queue[bytes | None]) -> None: - chunks.put( - b"OPENVMM_SNAPSHOT_PROFILE_V1 operation=restore " - b"phase=guest_repair_gate exclusive=0 duration_ns=" - ) - release.wait(10) - chunks.put(None) - - def write_input(self, data: bytes) -> None: - raise AssertionError(f"unexpected input: {data!r}") - - def close(self) -> None: - pass - - profiles: list[dict[str, object]] = [] - with ( - patch.object( - benchmark, "InteractiveProcess", return_value=FakeInteraction() - ), - patch.object(benchmark, "OUTPUT_DRAIN_TIMEOUT_SECONDS", 0.05), - patch.object(benchmark, "live_peak_rss_bytes", return_value=1024), - patch.object(benchmark, "wait_for_process_exit", return_value=0), - patch.object(benchmark, "process_resource_counters", return_value={}), - patch.object(benchmark, "terminate") as terminate, - self.assertRaisesRegex( - RuntimeError, r"did not reach EOF within 0\.5s" - ) as raised, - ): - benchmark.measure_once( - ["openvmm"], - environment={}, - timeout=0.5, - marker=benchmark.RESTORE_MARKER, - marker_must_be_line=True, - guest_exit_prequeued=True, - snapshot_profile=True, - profile_sink=profiles, - ) - - self.assertEqual(profiles, []) - terminate.assert_called_once() - self.assertIn("phase=guest_repair_gate", str(raised.exception)) - - def test_measure_once_returns_when_a_descendant_keeps_the_output_open(self): - # The descendant inherits the console output and stderr and outlives - # the launched process, so neither stream reaches EOF. - child = ( - "import subprocess, sys\n" - "holder = subprocess.Popen(\n" - " [sys.executable, '-c', 'import time; time.sleep(60)'],\n" - " stdin=subprocess.DEVNULL,\n" - ")\n" - "with open(sys.argv[1], 'w') as stream:\n" - " stream.write(str(holder.pid))\n" - "print('NVX-OUTPUT-HOLDER-READY', flush=True)\n" - ) - with tempfile.TemporaryDirectory() as temporary: - holder_pid = Path(temporary) / "holder.pid" - - def kill_holder() -> None: - try: - os.kill(int(holder_pid.read_text()), signal.SIGTERM) - except (OSError, ValueError): - pass - - started = time.monotonic() - try: - with ( - patch.object(benchmark, "OUTPUT_DRAIN_TIMEOUT_SECONDS", 0.05), - self.assertRaisesRegex( - RuntimeError, r"did not reach EOF within 3s of its exit" - ), - ): - benchmark.measure_once( - [sys.executable, "-I", "-c", child, str(holder_pid)], - environment=dict(os.environ), - timeout=3, - marker=b"NVX-OUTPUT-HOLDER-READY", - marker_must_be_line=True, - guest_exit_prequeued=True, - log_path=Path(temporary) / "launch.log", - ) - elapsed = time.monotonic() - started - finally: - kill_holder() - - # Closing a pipe that a reader is blocked on would wait for the - # descendant to exit. - self.assertLess(elapsed, 20) - - def test_output_reader_reads_and_stops_above_fd_setsize(self): - if sys.platform != "linux": - self.skipTest("select() limits descriptors to FD_SETSIZE on POSIX") - import fcntl - import resource - - floor = 1100 - limit = floor + 64 - soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) - if soft < limit: - if hard != resource.RLIM_INFINITY and hard < limit: - self.skipTest(f"the RLIMIT_NOFILE hard limit {hard} is too low") - resource.setrlimit(resource.RLIMIT_NOFILE, (limit, hard)) - self.addCleanup(resource.setrlimit, resource.RLIMIT_NOFILE, (soft, hard)) - read_fd, write_fd = os.pipe() - self.addCleanup(os.close, write_fd) - # F_DUPFD takes the lowest free descriptor at or above the floor, so it - # cannot replace a descriptor that something else owns. - high = fcntl.fcntl(read_fd, fcntl.F_DUPFD, floor) - os.close(read_fd) - os.set_inheritable(high, False) - self.addCleanup(os.close, high) - self.assertGreaterEqual(high, floor) - - reader = benchmark._OutputReader(high) - chunks: queue.Queue[bytes | None] = queue.Queue() - thread = threading.Thread(target=reader.run, args=(chunks,), daemon=True) - thread.start() - os.write(write_fd, b"console output") - self.assertEqual(chunks.get(timeout=5), b"console output") - # The write end stays open, so only stop() can end the read. - self.assertTrue(reader.stop(5)) - self.assertIsNone(chunks.get(timeout=5)) - thread.join(5) - self.assertFalse(thread.is_alive()) - - def test_guest_failure_report_includes_stderr_written_after_the_marker(self): - terminated = threading.Event() - - class FakeInteraction: - def __init__(self) -> None: - self.process = MagicMock(pid=4321) - self.process.poll.return_value = None - - def read_output(self, chunks: queue.Queue[bytes | None]) -> None: - chunks.put(b"NVX-RESTORE-PROCESSORS-FAIL unstable-tsc\r\n") - chunks.put(None) - - def read_stderr(self, chunks: queue.Queue[bytes | None]) -> None: - if terminated.wait(5): - chunks.put(b"WARN host diagnostics after the guest failure\n") - chunks.put(None) - - def write_input(self, data: bytes) -> None: - raise AssertionError(f"unexpected input: {data!r}") - - def close(self) -> None: - pass - - def terminate_openvmm(_process: object) -> None: - terminated.set() - - with ( - patch.object( - benchmark, "InteractiveProcess", return_value=FakeInteraction() - ), - patch.object( - benchmark, "terminate", side_effect=terminate_openvmm - ) as terminate, - self.assertRaises(benchmark.GuestFailureReported) as raised, - ): - benchmark.measure_once( - ["openvmm"], - environment={}, - timeout=10, - marker=b"NVX-RESTORE-PROCESSORS-OK count=8", - marker_must_be_line=True, - guest_exit_prequeued=True, - failure_marker=b"NVX-RESTORE-PROCESSORS-FAIL", - ) - - terminate.assert_called_once() - self.assertEqual( - raised.exception.line, "NVX-RESTORE-PROCESSORS-FAIL unstable-tsc" - ) - self.assertIn("host diagnostics after the guest failure", str(raised.exception)) - - def test_separated_output_keeps_process_output_lines_whole(self): - record = ( - b"OPENVMM_SNAPSHOT_PROFILE_V1 operation=capture phase=input_gate " - b"exclusive=1 duration_ns=263787 process_elapsed_ns=1422094434 pid=1\n" - ) - child = ( - "import os, time; " - "os.write(1, b'NVX-S'); time.sleep(0.05); " - f"os.write(2, {record!r}); time.sleep(0.05); " - "os.write(1, b'NAPSHOT-DISPATCHED\\n~ # ')" - ) - interaction = benchmark.InteractiveProcess( - [sys.executable, "-I", "-c", child], - dict(os.environ), - separate_stderr=True, - ) - stderr = bytearray() - try: - output = benchmark.SeparatedOutput(interaction) - deadline = time.monotonic() + 30 - while not output.closed: - stream, chunk = output.get(max(0.0, deadline - time.monotonic())) - if stream == "stderr" and chunk is not None: - stderr.extend(chunk) - self.assertEqual(interaction.process.wait(timeout=30), 0) - finally: - benchmark.terminate(interaction.process) - interaction.close() - - marker = benchmark.SNAPSHOT_GUEST_DISPATCH_MARKER - self.assertTrue(benchmark.contains_output_line(output.console, marker)) - self.assertNotIn(b"OPENVMM_SNAPSHOT_PROFILE_V1", output.console) - self.assertIn(record, bytes(stderr)) - contents = output.contents() - self.assertTrue(benchmark.contains_output_line(contents, marker)) - self.assertTrue( - benchmark.contains_output_line(contents, record.removesuffix(b"\n")) - ) - self.assertTrue(contents.endswith(b"~ # ")) - - def test_interactive_process_gives_stderr_its_own_pipe_on_request(self): - for platform in ("linux", "win32"): - for separate_stderr in (False, True): - with self.subTest(platform=platform, separate=separate_stderr): - terminal: list[int] = [] - - def open_terminal( - terminal: list[int] = terminal, - ) -> tuple[int, int]: - terminal.extend(os.pipe()) - return terminal[0], terminal[1] - - process = MagicMock(pid=4321) - if not separate_stderr: - process.stderr = None - with ( - patch.object(benchmark.sys, "platform", platform), - patch.object( - benchmark.os, - "openpty", - side_effect=open_terminal, - create=True, - ), - patch.object( - benchmark.subprocess, "Popen", return_value=process - ) as popen, - ): - interaction = benchmark.InteractiveProcess( - ["openvmm"], {}, separate_stderr=separate_stderr - ) - try: - stderr = popen.call_args.kwargs["stderr"] - if separate_stderr: - self.assertEqual(stderr, subprocess.PIPE) - elif platform == "linux": - self.assertEqual(stderr, terminal[1]) - else: - self.assertEqual(stderr, subprocess.STDOUT) - if not separate_stderr: - chunks: queue.Queue[bytes | None] = queue.Queue() - with self.assertRaisesRegex( - RuntimeError, "shares the console stream" - ): - interaction.read_stderr(chunks) - self.assertIsNone(chunks.get_nowait()) - finally: - interaction.close() - if separate_stderr: - process.stderr.close.assert_called_once_with() - - def test_warm_snapshot_cache_reads_every_artifact(self): - with tempfile.TemporaryDirectory() as temporary: - snapshot = Path(temporary) - for index, name in enumerate(benchmark.SNAPSHOT_FILENAMES): - (snapshot / name).write_bytes(bytes([index]) * 32) - - benchmark.warm_snapshot_artifacts(snapshot) - - def test_snapshot_artifact_validation_preserves_error_prefix(self): - with tempfile.TemporaryDirectory() as temporary: - snapshot = Path(temporary) - for prefix in ("snapshot", "console/active snapshot"): - with self.subTest(prefix=prefix): - with self.assertRaises(RuntimeError) as context: - benchmark._require_snapshot_artifacts(snapshot, prefix) - - self.assertEqual( - str(context.exception), - f"{prefix} did not publish {snapshot / 'manifest.bin'}", - ) - - def test_measure_once_does_not_resend_prequeued_guest_exit(self): - class FakeProcess: - pid = 123 - returncode = 0 - - def poll(self): - return None - - def terminate(self): - raise AssertionError("guest-exit teardown terminated the host process") - - class FakeInteraction: - def __init__(self): - self.process = FakeProcess() - self.writes: list[bytes] = [] - - def read_output(self, chunks: queue.Queue[bytes | None]): - chunks.put(b"/ # echo OPENVMM-SNAPSHOT-RESTORE-OK\r\n") - chunks.put(b"OPENVMM-SNAPSHOT-RESTORE-OK\r\n") - chunks.put(None) - - def read_stderr(self, chunks: queue.Queue[bytes | None]): - chunks.put(None) - - def write_input(self, data: bytes): - self.writes.append(data) - - def close(self): - pass - - interaction = FakeInteraction() - with ( - patch.object(benchmark, "InteractiveProcess", return_value=interaction), - patch.object(benchmark, "live_peak_rss_bytes", return_value=1024), - patch.object(benchmark, "wait_for_process_exit", return_value=0), - patch.object( - benchmark.time, - "perf_counter_ns", - side_effect=[0, 10_000_000, 17_000_000], - ), - ): - result = benchmark.measure_once( - ["openvmm"], - environment={}, - timeout=1, - marker=benchmark.RESTORE_MARKER, - marker_must_be_line=True, - guest_exit_prequeued=True, - ) - - self.assertEqual(result, (10.0, 1024, 7.0, 17.0)) - self.assertEqual(interaction.writes, []) - - def test_measure_once_reports_missing_rss_when_openvmm_exits_first(self): - class FakeProcess: - pid = 123 - - def __init__(self, status: int): - self.status = status - self.exited = False - self.returncode: int | None = None - - def poll(self): - if self.exited: - self.returncode = self.status - return self.returncode - - def wait(self, timeout: float | None = None) -> int: - del timeout - return self.status - - def terminate(self): - raise AssertionError("unexpected process termination") - - class FakeInteraction: - def __init__(self, status: int): - self.process = FakeProcess(status) - - def read_output(self, chunks: queue.Queue[bytes | None]): - self.process.exited = True - chunks.put(benchmark.RESTORE_MARKER + b"\n") - chunks.put(None) - - def read_stderr(self, chunks: queue.Queue[bytes | None]): - chunks.put(None) - - def write_input(self, data: bytes): - raise AssertionError(f"unexpected input: {data!r}") - - def close(self): - pass - - for status in (0, 1): - with self.subTest(status=status): - interaction = FakeInteraction(status) - - def linux_peak_rss_bytes( - pid: int, interaction: FakeInteraction = interaction - ) -> int | None: - self.assertEqual(pid, 123) - # A zombie's /proc status no longer reports VmHWM. - return None if interaction.process.exited else 1024 - - with ( - patch.object( - benchmark, "InteractiveProcess", return_value=interaction - ), - patch.object( - benchmark, - "_linux_live_peak_rss_bytes", - side_effect=linux_peak_rss_bytes, - ), - patch.object(benchmark, "windows_peak_rss_bytes", return_value=1), - patch.object( - benchmark, "wait_for_process_exit", return_value=status - ) as wait, - patch.object( - benchmark.time, - "perf_counter_ns", - side_effect=[0, 10_000_000, 17_000_000], - ), - ): - if status == 0: - result = benchmark.measure_once( - ["openvmm"], - environment={}, - timeout=1, - marker=benchmark.RESTORE_MARKER, - marker_must_be_line=True, - guest_exit_prequeued=True, - ) - self.assertEqual(result, (10.0, None, 7.0, 17.0)) - else: - with self.assertRaisesRegex( - RuntimeError, "status 1 during teardown" - ): - benchmark.measure_once( - ["openvmm"], - environment={}, - timeout=1, - marker=benchmark.RESTORE_MARKER, - marker_must_be_line=True, - guest_exit_prequeued=True, - ) - - wait.assert_called_once_with( - interaction.process, benchmark.TEARDOWN_TIMEOUT_SECONDS - ) - - def test_completed_output_line_with_prefix_requires_a_whole_line(self): - prefix = b"NVX-RESTORE-PROCESSORS-FAIL" - self.assertIsNone( - benchmark.completed_output_line_with_prefix( - b"NVX-RESTORE-PROCESSORS-FAIL unst", prefix - ) - ) - self.assertIsNone( - benchmark.completed_output_line_with_prefix( - b'> echo "NVX-RESTORE-PROCESSORS-FAIL count=$n"\r\n', prefix - ) - ) - self.assertEqual( - benchmark.completed_output_line_with_prefix( - b"guest\r\nNVX-RESTORE-PROCESSORS-FAIL unstable-tsc\r\nmore", prefix - ), - b"NVX-RESTORE-PROCESSORS-FAIL unstable-tsc", - ) - - def test_measure_once_stops_at_a_completed_guest_failure_marker(self): - class FakeInteraction: - def __init__(self): - self.process = MagicMock(pid=123) - self.process.poll.return_value = None - - def read_output(self, chunks: queue.Queue[bytes | None]): - chunks.put(b'> echo "NVX-RESTORE-PROCESSORS-FAIL count=$n"\r\n') - chunks.put(b"Measured 6 cycles TSC warp between CPUs\r\n") - chunks.put(b"NVX-RESTORE-PROCESSORS-FAIL unst") - chunks.put(b"able-tsc\r\n") - chunks.put(None) - - def read_stderr(self, chunks: queue.Queue[bytes | None]): - chunks.put(None) - - def write_input(self, data: bytes): - raise AssertionError(f"unexpected input: {data!r}") - - def close(self): - pass - - interaction = FakeInteraction() - with tempfile.TemporaryDirectory() as temporary: - log_path = Path(temporary) / "restore.log" - with ( - patch.object(benchmark, "InteractiveProcess", return_value=interaction), - patch.object(benchmark, "terminate") as terminate, - patch.object(benchmark, "wait_for_process_exit") as wait, - ): - with self.assertRaises(benchmark.GuestFailureReported) as raised: - benchmark.measure_once( - ["openvmm"], - environment={}, - timeout=60, - marker=b"NVX-RESTORE-PROCESSORS-OK count=8", - marker_must_be_line=True, - guest_exit_prequeued=True, - log_path=log_path, - failure_marker=b"NVX-RESTORE-PROCESSORS-FAIL", - ) - log = log_path.read_bytes() - - self.assertEqual( - raised.exception.line, "NVX-RESTORE-PROCESSORS-FAIL unstable-tsc" - ) - self.assertIn("TSC warp", raised.exception.output_tail) - self.assertIn("--- OpenVMM output ---", str(raised.exception)) - terminate.assert_called_once_with(interaction.process) - wait.assert_not_called() - self.assertTrue(log.endswith(b"NVX-RESTORE-PROCESSORS-FAIL unstable-tsc\r\n")) - - def test_guest_runner_preserves_primary_and_close_failures_with_bounded_tail(self): - class FakeProcess: - pid = 123 - - def poll(self): - return 7 - - def wait(self): - return 7 - - class FakeInteraction: - process = FakeProcess() - containment = None - - def read_output(self, chunks: queue.Queue[bytes | None]): - chunks.put(benchmark.BOOT_MARKER + b"\n" + b"x" * 5000 + b"\nDONE\n") - chunks.put(None) - - def write_input(self, _data: bytes): - pass - - def close(self): - raise OSError("close failed") - - with tempfile.TemporaryDirectory() as temporary: - log_path = Path(temporary) / "guest.log" - with ( - patch.object( - benchmark, "InteractiveProcess", return_value=FakeInteraction() - ), - patch.object(benchmark, "_try_peak_rss", return_value=1024), - patch.object(benchmark, "terminate") as terminate, - ): - with self.assertRaisesRegex( - RuntimeError, "OpenVMM exited with status 7" - ) as raised: - benchmark.run_guest_script( - ["openvmm"], - "guest command\n", - b"DONE", - timeout=1, - log_path=log_path, - ) - - error = raised.exception - cleanup_errors = cast( - tuple[BaseException, ...], - error.cleanup_errors, # type: ignore[attr-defined] - ) - self.assertEqual([str(item) for item in cleanup_errors], ["close failed"]) - self.assertIn("--- OpenVMM output ---", str(error)) - self.assertIn("--- cleanup failures ---", str(error)) - self.assertNotIn(benchmark.BOOT_MARKER.decode(), str(error)) - self.assertGreater(len(log_path.read_bytes()), 4096) - self.assertTrue(log_path.read_bytes().endswith(b"\nDONE\n")) - terminate.assert_called_once() - - def test_guest_runner_propagates_success_only_close_failure(self): - class FakeProcess: - pid = 123 - - def poll(self): - return 0 - - def wait(self): - return 0 - - class FakeInteraction: - process = FakeProcess() - containment = None - - def read_output(self, chunks: queue.Queue[bytes | None]): - chunks.put(benchmark.BOOT_MARKER + b"\nDONE\n") - chunks.put(None) - - def write_input(self, _data: bytes): - pass - - def close(self): - raise OSError("close after success") - - with ( - patch.object( - benchmark, "InteractiveProcess", return_value=FakeInteraction() - ), - patch.object(benchmark, "_try_peak_rss", return_value=1024), - ): - with self.assertRaisesRegex(OSError, "close after success"): - benchmark.run_guest_script( - ["openvmm"], - "guest command\n", - b"DONE", - timeout=1, - ) - - def test_live_peak_rss_samples_linux_process_without_reaping(self): - process = MagicMock(pid=123) - with ( - patch.object(benchmark.os, "name", "posix"), - patch.object(benchmark.sys, "platform", "linux"), - patch.object( - benchmark, "_linux_live_peak_rss_bytes", side_effect=[4096, None, 0] - ) as read, - ): - self.assertEqual(benchmark.live_peak_rss_bytes(process), 4096) - self.assertIsNone(benchmark.live_peak_rss_bytes(process)) - with self.assertRaisesRegex(RuntimeError, "peak RSS 0 bytes"): - benchmark.live_peak_rss_bytes(process) - - self.assertEqual(read.call_args_list, [call(123)] * 3) - process.poll.assert_not_called() - - def test_live_peak_rss_accepts_only_running_windows_samples(self): - running = MagicMock(pid=7) - running.poll.return_value = None - exited = MagicMock(pid=8) - exited.poll.return_value = 0 - racing = MagicMock(pid=9) - racing.poll.return_value = None - - def read_counters(pid: int) -> int: - if pid == racing.pid: - racing.poll.return_value = 0 - return 8192 - - with ( - patch.object(benchmark.os, "name", "nt"), - patch.object( - benchmark, "windows_peak_rss_bytes", side_effect=read_counters - ) as read, - ): - self.assertEqual(benchmark.live_peak_rss_bytes(running), 8192) - # A retained handle still reports counters, including teardown. - self.assertIsNone(benchmark.live_peak_rss_bytes(exited)) - self.assertIsNone(benchmark.live_peak_rss_bytes(racing)) - - self.assertEqual(read.call_args_list, [call(7), call(8), call(9)]) - with ( - patch.object(benchmark.os, "name", "nt"), - patch.object( - benchmark, "windows_peak_rss_bytes", side_effect=OSError("closed") - ), - ): - self.assertIsNone(benchmark.live_peak_rss_bytes(exited)) - with self.assertRaisesRegex(OSError, "closed"): - benchmark.live_peak_rss_bytes(running) - - def test_linux_live_peak_rss_requires_process_address_space(self): - running = "Name:\topenvmm\nState:\tS (sleeping)\nVmHWM:\t 59008 kB\n" - zombie = "Name:\topenvmm\nState:\tZ (zombie)\nThreads:\t1\n" - with patch.object( - Path, - "read_text", - autospec=True, - side_effect=[running, zombie, FileNotFoundError(), ProcessLookupError()], - ) as read: - self.assertEqual(benchmark._linux_live_peak_rss_bytes(123), 59008 * 1024) - for _ in range(3): - self.assertIsNone(benchmark._linux_live_peak_rss_bytes(123)) - - self.assertEqual(read.call_args.args[0], Path("/proc/123/status")) - with ( - patch.object(Path, "read_text", side_effect=PermissionError()), - self.assertRaises(PermissionError), - ): - benchmark._linux_live_peak_rss_bytes(123) - - def test_builds_isolated_workload_command(self): - command = benchmark.workload_boot_command( - Path("openvmm"), - "kvm", - Path("vmlinux"), - Path("initramfs.cpio.gz"), - 128, - "quiet loglevel=0 nokaslr", - processors=8, - command_prefix=("taskset", "-c", "2-3"), - network="10.0.0.2/24", - mount="/mnt/host,C:/work,rw", - ) - - self.assertEqual(command[:4], ["taskset", "-c", "2-3", "openvmm"]) - self.assertEqual( - command[4:10], - [ - "--single-process", - "--machine", - "microvm", - "--processors", - "8", - "--hypervisor", - ], - ) - self.assertIn("quiet loglevel=0 nokaslr", command) - self.assertEqual( - command[-6:], - [ - "--net", - "10.0.0.2/24", - "--network-profile", - "portable", - "--mount", - "/mnt/host,C:/work,rw", - ], - ) - - def test_benchmark_network_requires_explicit_profile(self): - args = nvx.parse_args( - [ - "benchmark", - "--net", - "10.0.0.2/24", - "--network-profile", - "portable", - ] - ) - self.assertEqual(args.network_profile, "portable") - - args.network_profile = None - with self.assertRaisesRegex(ValueError, "--net and --network-profile"): - benchmark.run(args) - - def test_benchmark_scratch_directory_routes_temporary_files(self): - with tempfile.TemporaryDirectory() as temporary: - scratch = Path(temporary).resolve() / "scratch" - scratch.mkdir() - args = nvx.parse_args(["benchmark", "--scratch-dir", str(scratch)]) - previous = tempfile.tempdir - - with ( - benchmark.benchmark_scratch_directory(args), - patch.object(benchmark, "_git_revision", return_value="revision"), - ): - self.assertEqual(Path(tempfile.gettempdir()), scratch) - with tempfile.TemporaryDirectory(prefix="openvmm-e2e-") as snapshot: - self.assertEqual(Path(snapshot).parent, scratch) - document = benchmark.result_document(args, None, None) - - self.assertEqual(tempfile.tempdir, previous) - self.assertEqual(args.scratch_dir, scratch) - self.assertEqual( - document["controls"]["scratch_directory"], - str(scratch), - ) - - def test_benchmark_scratch_directory_defaults_to_system_temporary(self): - args = nvx.parse_args(["benchmark"]) - previous = tempfile.tempdir - - with ( - benchmark.benchmark_scratch_directory(args), - patch.object(benchmark, "_git_revision", return_value="revision"), - ): - self.assertEqual(tempfile.tempdir, previous) - document = benchmark.result_document(args, None, None) - - self.assertIsNone(args.scratch_dir) - self.assertEqual( - document["controls"]["scratch_directory"], - str(Path(tempfile.gettempdir()).resolve()), - ) - - def test_benchmark_scratch_directory_must_exist(self): - with tempfile.TemporaryDirectory() as temporary: - missing = Path(temporary) / "missing" - args = nvx.parse_args(["benchmark", "--scratch-dir", str(missing)]) - previous = tempfile.tempdir - - with self.assertRaisesRegex(ValueError, "scratch directory does not exist"): - benchmark.run(args) - - self.assertEqual(tempfile.tempdir, previous) - - def test_network_snapshot_restore_selects_portable_profile(self): - command = benchmark.snapshot_restore_command( - Path("openvmm"), - "mshv", - Path("snapshot"), - processors=4, - network_profile="portable", - ) - - self.assertEqual( - command[2:8], - ["--machine", "microvm", "--processors", "4", "--hypervisor", "mshv"], - ) - - self.assertEqual( - command[-2:], - [ - "--network-profile", - "portable", - ], - ) - - def test_snapshot_restore_command_separates_capacity_from_online_target(self): - command = benchmark.snapshot_restore_command( - Path("openvmm"), - "kvm", - Path("snapshot"), - processors=8, - restore_processors=4, - ) - - self.assertEqual(command[command.index("--processors") + 1], "8") - self.assertEqual(command[command.index("--restore-processors") + 1], "4") - # Restore packet v4 gives every restore fresh entropy. - self.assertNotIn("--restore-entropy", command) - - def test_snapshot_restore_command_sets_memory_target_separately(self): - command = benchmark.snapshot_restore_command( - Path("openvmm"), - "whp", - Path("snapshot"), - restore_memory_mib=2048, - ) - - self.assertNotIn("--memory", command) - self.assertEqual(command[command.index("--restore-memory") + 1], "2048M") - - def test_memory_online_latency_parser_validates_added_bytes(self): - with tempfile.TemporaryDirectory() as temporary: - log = Path(temporary) / "restore.log" - log.write_bytes( - b"NVX-MEMORY-ONLINE-OK: added_bytes=536870912 " - b"memtotal_kib=1000000 elapsed_us=12345\n" - ) - self.assertEqual( - benchmark._memory_online_elapsed_ms(log, 536870912), - 12.345, - ) - with self.assertRaisesRegex(RuntimeError, "expected 1"): - benchmark._memory_online_elapsed_ms(log, 1) - - def test_benchmark_snapshot_restore_propagates_processors(self): - args = argparse.Namespace( - processors=4, - network_profile=None, - warmups=1, - runs=1, - timeout=1.0, - teardown_mode="guest-exit", - ) - result = cast(benchmark.BenchmarkResult, {"p50_ms": 10.0}) - with patch.object(benchmark, "benchmark", return_value=result) as run: - self.assertIs( - benchmark.benchmark_snapshot_restore( - args, - Path("openvmm"), - "kvm", - ["unused-cold-command"], - snapshot_path=Path("snapshot"), - ), - result, - ) - - command = run.call_args.args[0] - self.assertEqual( - command[2:8], - ["--machine", "microvm", "--processors", "4", "--hypervisor", "kvm"], - ) - self.assertTrue(run.call_args.kwargs["marker_must_be_line"]) - self.assertTrue(run.call_args.kwargs["guest_exit_prequeued"]) - - def test_parses_dd_rates_and_network_gateway(self): - output = """ -67108864 bytes copied, 0.25 s, 268.4 MB/s -67108864 bytes copied, 0.04 s, 1.6 GB/s -""" - - self.assertEqual(benchmark.parse_dd_rate(output, 1), 268.4) - self.assertEqual(benchmark.parse_dd_rate(output, 2), 1600.0) - self.assertEqual(benchmark.network_gateway("10.0.0.2/24"), "10.0.0.1") - - def test_benchmark_command_lines_carry_no_clock_tuning(self): - # Under time ABI v1 the hypervisor identity gives Linux its TSC and - # LAPIC rates and trust, so the lifecycle line tunes no clock and the - # cold-start clocksource variant selects tsc on every backend. - for token in ( - "tsc=", - "no_timer_check", - "clocksource=", - "tsc_early_khz=", - "lapic_timer_hz=", - ): - self.assertNotIn(token, benchmark.BASE_TUNING) - args = argparse.Namespace( - runs=1, - warmups=0, - memory_mib=128, - processors=1, - timeout=10.0, - teardown_mode="guest-exit", - ) - for backend in ("kvm", "mshv", "whp"): - commands: list[list[str]] = [] - - def fake_benchmark( - command: list[str], - commands: list[list[str]] = commands, - **_kwargs: object, - ): - commands.append(command) - return {"samples_ms": [100.0]} - - with ( - self.subTest(backend=backend), - patch.object(benchmark, "benchmark", side_effect=fake_benchmark), - contextlib.redirect_stdout(io.StringIO()) as stdout, - ): - benchmark.benchmark_cold_start_workload( - args, Path("openvmm"), Path("vmlinux"), Path("initrd"), backend - ) - cmdlines = [command[command.index("--cmdline") + 1] for command in commands] - self.assertIn("quiet loglevel=0 clocksource=tsc", cmdlines) - self.assertFalse(any("kvm-clock" in cmdline for cmdline in cmdlines)) - self.assertIn(" clocksource=tsc :", stdout.getvalue()) - - def test_smp_probe_uses_explicit_topology_and_worker_rendezvous(self): - script = benchmark.smp_probe_script(4) - - self.assertIn("getconf _NPROCESSORS_ONLN", script) - self.assertIn("physical_package_id", script) - self.assertIn("die_id", script) - self.assertIn("thread_siblings_list", script) - self.assertIn('taskset -c "$cpu"', script) - self.assertIn("for pid in $worker_pids", script) - self.assertIn("loc_before", script) - self.assertIn("SMP-LAPIC-FAIL", script) - self.assertIn("SMP-IPI-FAIL", script) - self.assertIn("read_loc_counter", script) - self.assertNotIn("while :; do", script) - self.assertIn("timer_attempts=10000", script) - self.assertIn('[ "$after" -gt "$before" ]', script) - self.assertIn('[ "$after" -ge "$current" ]', script) - self.assertNotIn("sleep 0.1", script) - self.assertIn("apic_ids=0,1,2,3 bsp=0 workers=$workers", script) - self.assertIn("NVX-SMP-PROBE-OK", script) - self.assertTrue(script.endswith("nvx-exit 0\n")) - network_script = benchmark.smp_probe_script( - 4, - network_gateway="10.0.0.1", - ioapic_irq=10, - ) - self.assertIn('ping -c 2 -W 1 "10.0.0.1"', network_script) - self.assertIn("SMP-IOAPIC-FAIL", network_script) - self.assertIn("SMP-IOAPIC-OK", network_script) - with self.assertRaises(ValueError): - benchmark.smp_probe_script(3) - with self.assertRaises(ValueError): - benchmark.smp_probe_script(4, network_gateway="10.0.0.1") - - def test_snapshot_request_triggers_waiting_capture_controller(self): - script = benchmark.snapshot_request_script( - 4, - teardown_mode="guest-exit", - ) - self.assertEqual(script, "nvx-snapshot\n") - - no_controller = benchmark.snapshot_request_script( - None, - teardown_mode="guest-exit", - ) - self.assertEqual( - no_controller, - "echo NVX-SNAPSHOT-DISPATCHED\n" - "nvx-snapshot\n" - "echo OPENVMM-SNAPSHOT-RESTORE-OK\n" - "nvx-exit 0\n", - ) - self.assertNotIn( - "nvx-exit", - benchmark.snapshot_request_script(None, teardown_mode="host-terminate"), - ) - - def test_prepare_snapshot_capture_stages_waiting_controller(self): - script = benchmark.prepare_snapshot_capture_script( - 4, - teardown_mode="guest-exit", - network_gateway="10.0.0.1", - ioapic_irq=10, - ) - - self.assertTrue( - script.startswith( - f"cat >{benchmark.SMP_PROBE_PATH} <<'NVX_SMP_PROBE_SCRIPT'\n" - ) - ) - self.assertIn('ping -c 2 -W 1 "10.0.0.1"', script) - self.assertIn("NVX-SMP-PROBE-OK", script) - self.assertIn( - f"cat >{benchmark.SNAPSHOT_CAPTURE_PATH} <<'NVX_SNAPSHOT_CAPTURE_SCRIPT'\n", - script, - ) - self.assertIn("IFS= read -r trigger\n", script) - self.assertIn("echo NVX-SNAPSHOT-DISPATCHED\n", script) - self.assertIn( - "/sbin/nvx-snapshot\necho OPENVMM-SNAPSHOT-RESTORE-OK\nnvx-exit 0\n", - script, - ) - host_terminated = benchmark.prepare_snapshot_capture_script( - 4, teardown_mode="host-terminate" - ) - self.assertNotIn("nvx-exit 0", host_terminated) - self.assertTrue( - script.endswith( - "NVX_SNAPSHOT_CAPTURE_SCRIPT\n" - f"chmod +x {benchmark.SMP_PROBE_PATH} " - f"{benchmark.SNAPSHOT_CAPTURE_PATH}\n" - f"{benchmark.SNAPSHOT_CAPTURE_PATH}\n" - ) - ) - - def test_capture_does_not_wait_for_a_clocksource(self): - # The time ABI registers the tsc clocksource at device_initcall on - # every backend, so no capture waits for Linux to leave tsc-early. - script = benchmark.prepare_snapshot_capture_script( - 1, teardown_mode="guest-exit" - ) - probe = script.split("<<'NVX_SMP_PROBE_SCRIPT'\n", 1)[1] - probe = probe.split("NVX_SMP_PROBE_SCRIPT\n", 1)[0] - self.assertEqual(probe, benchmark.smp_probe_script(1, exit_guest=False)) - for text in ("clocksource", "tsc-early", "clock_tries"): - self.assertNotIn(text, script) - self.assertFalse(hasattr(benchmark, "stable_clocksource_wait_script")) - - def test_output_marker_must_be_a_complete_line(self): - marker = benchmark.RESTORE_MARKER - self.assertFalse( - benchmark.contains_output_line( - b"/ # echo OPENVMM-SNAPSHOT-RESTORE-OK\r\n", - marker, - ) - ) - self.assertTrue( - benchmark.contains_output_line( - b"/ # echo OPENVMM-SNAPSHOT-RESTORE-OK\r\n" - b"OPENVMM-SNAPSHOT-RESTORE-OK\r\n", - marker, - ) - ) - - def test_device_restore_marker_and_trace_parsers(self): - marker = benchmark.parse_device_restore_marker( - "NVX-VIRTIO-RESTORE-PROBE phase=io-success device=net mode=deferred\r" - ) - self.assertEqual( - marker, - {"phase": "io-success", "device": "net", "mode": "deferred"}, - ) - self.assertIsNone(benchmark.parse_device_restore_marker("unrelated")) - - event = benchmark.parse_virtio_restore_event( - "DEBUG virtio_restore: virtio restore lifecycle " - 'event: "queue_start", device_type: 0x1a, trigger: "driver-ok", ' - "queue_index: 0, restored_progress: true, success: true" - ) - self.assertEqual( - event, - { - "event": "queue_start", - "device_type": 0x1A, - "trigger": "driver-ok", - "queue_index": 0, - "restored_progress": True, - "success": True, - }, - ) - self.assertIsNone(benchmark.parse_virtio_restore_event("unrelated")) - - def test_device_restore_commands_use_fixed_attachments(self): - common = ( - Path("openvmm"), - "whp", - Path("vmlinux"), - Path("initrd"), - 256, - ) - console_boot, console_restore = benchmark.device_restore_commands( - *common, "console", "deferred" - ) - self.assertIn( - "nvx_virtio_restore_probe=console,deferred", - console_boot[console_boot.index("--cmdline") + 1], - ) - self.assertEqual(console_boot[-2:], ["--virtio-console", "console"]) - self.assertEqual(console_restore[-2:], ["--virtio-console", "console"]) - - net_boot, net_restore = benchmark.device_restore_commands( - *common, "net", "active" - ) - self.assertIn("--network-profile", net_boot) - self.assertIn("--network-profile", net_restore) - - fs_boot, fs_restore = benchmark.device_restore_commands( - *common, - "virtiofs", - "deferred", - host_directory=Path("host-share"), - ) - self.assertIn("--mount", fs_boot) - self.assertIn("--mount", fs_restore) - - def test_device_restore_sample_rejects_premature_activation(self): - markers = [ - { - "phase": "restore-ready", - "device": "net", - "mode": "deferred", - "observer_elapsed_ms": "8.0", - }, - { - "phase": "trigger", - "device": "net", - "mode": "deferred", - "observer_elapsed_ms": "10.0", - }, - ] - events: list[dict[str, object]] = [ - { - "event": "restore_staged", - "device_type": 1, - "trigger": "inactive-start", - "queue_index": -1, - "restored_progress": False, - "success": True, - "observer_elapsed_ms": 5.0, - }, - { - "event": "private_state_apply", - "device_type": 1, - "trigger": "kick", - "queue_index": -1, - "restored_progress": False, - "success": True, - "observer_elapsed_ms": 9.0, - }, - { - "event": "kick_staged", - "device_type": 1, - "trigger": "kick", - "queue_index": 0, - "restored_progress": False, - "success": True, - "observer_elapsed_ms": 10.5, - }, - ] - for queue_index in range(2): - events.append( - { - "event": "queue_start", - "device_type": 1, - "trigger": "driver-ok", - "queue_index": queue_index, - "restored_progress": False, - "success": True, - "observer_elapsed_ms": 11.0, - } - ) - events.append( - { - "event": "kick_dispatch", - "device_type": 1, - "trigger": "driver-ok", - "queue_index": 0, - "restored_progress": False, - "success": True, - "observer_elapsed_ms": 12.0, - } - ) - sample = cast( - benchmark.DeviceRestoreSample, - { - "process_launch_to_ready_ms": 8.0, - "trigger_to_first_successful_io_ms": 2.0, - "peak_rss_bytes": 1, - "guest_markers": markers, - "events": events, - "queue_start_count": 0, - "staged_kick_dispatch_count": 0, - "stale_premature_callback_count": 0, - "log": "sample.log", - }, - ) - with self.assertRaisesRegex(RuntimeError, "premature"): - benchmark.validate_device_restore_sample(sample, "net", "deferred") - events[1]["observer_elapsed_ms"] = 10.25 - benchmark.validate_device_restore_sample(sample, "net", "deferred") - self.assertEqual(sample["queue_start_count"], 2) - self.assertEqual(sample["staged_kick_dispatch_count"], 1) - self.assertEqual(sample["stale_premature_callback_count"], 0) - - def test_device_restore_profile_writes_standalone_result(self): - args = argparse.Namespace( - processors=1, - restore_devices=["console"], - restore_modes=["active"], - net=None, - memory_mib=128, - timeout=1.0, - warmups=1, - runs=2, - platform="windows-whp-baremetal", - ) - - def sample() -> benchmark.DeviceRestoreSample: - return { - "process_launch_to_ready_ms": 10.0, - "trigger_to_first_successful_io_ms": 2.0, - "peak_rss_bytes": 1024, - "guest_markers": [ - { - "phase": "restore-ready", - "device": "console", - "mode": "active", - "observer_elapsed_ms": "10.0", - }, - { - "phase": "trigger", - "device": "console", - "mode": "active", - "observer_elapsed_ms": "11.0", - }, - ], - "events": [ - { - "event": "restore_staged", - "device_type": 3, - "trigger": "active-start", - "queue_index": -1, - "restored_progress": False, - "success": True, - }, - { - "event": "private_state_apply", - "device_type": 3, - "trigger": "active-start", - "queue_index": -1, - "restored_progress": False, - "success": True, - }, - { - "event": "kick_dispatch", - "device_type": 3, - "trigger": "kick", - "queue_index": 1, - "restored_progress": False, - "success": True, - }, - *[ - { - "event": "queue_start", - "device_type": 3, - "trigger": "active-start", - "queue_index": queue_index, - "restored_progress": True, - "success": True, - } - for queue_index in range(2) - ], - ], - "queue_start_count": 0, - "staged_kick_dispatch_count": 0, - "stale_premature_callback_count": 0, - "log": "", - } - - def restore_sample( - *_args: object, **_kwargs: object - ) -> benchmark.DeviceRestoreSample: - return sample() - - with tempfile.TemporaryDirectory() as temporary: - output_dir = Path(temporary) - with ( - patch.object( - benchmark, - "capture_device_restore_snapshot", - return_value=[ - { - "phase": "capture-ready", - "device": "console", - "mode": "active", - } - ], - ) as capture, - patch.object( - benchmark, - "run_device_restore_sample", - side_effect=restore_sample, - ) as restore, - ): - path = benchmark.benchmark_device_restore_profile( - args, - Path("openvmm"), - Path("vmlinux"), - Path("initrd"), - "whp", - output_dir, - ) - - self.assertEqual(capture.call_count, 1) - self.assertEqual(restore.call_count, 3) - document = json.loads(path.read_text(encoding="utf-8")) - self.assertTrue(document["non_canonical"]) - self.assertTrue(document["assertions"]["zero_stale_premature_callbacks"]) - scenario = document["scenarios"]["console-active"] - self.assertEqual( - scenario["process_launch_to_ready_ms"]["samples"], [10.0, 10.0] - ) - self.assertEqual(scenario["process_launch_to_ready_ms"]["p50"], 10.0) - self.assertEqual(scenario["queue_start_counts"], [2, 2]) - - def test_device_restore_profile_writes_lf_only_virtiofs_seed(self): - args = argparse.Namespace( - processors=1, - restore_devices=["virtiofs"], - restore_modes=["active"], - net=None, - memory_mib=128, - timeout=1.0, - warmups=1, - runs=1, - platform="windows-whp-baremetal", - ) - observed_seeds: list[bytes] = [] - host_directories: list[Path] = [] - - def commands(*_args: object, **kwargs: object) -> tuple[list[str], list[str]]: - host_directory = cast(Path, kwargs["host_directory"]) - host_directories.append(host_directory) - observed_seeds.append( - (host_directory / ".nvx-virtio-restore-host-seed").read_bytes() - ) - return ["boot"], ["restore"] - - sample = cast( - benchmark.DeviceRestoreSample, - { - "process_launch_to_ready_ms": 1.0, - "trigger_to_first_successful_io_ms": 1.0, - "peak_rss_bytes": 1, - "guest_markers": [], - "events": [], - "queue_start_count": 0, - "staged_kick_dispatch_count": 0, - "stale_premature_callback_count": 0, - "log": "sample.log", - }, - ) - - def run_sample( - *_args: object, **_kwargs: object - ) -> benchmark.DeviceRestoreSample: - (host_directories[-1] / ".nvx-virtio-restore-guest-result").write_bytes( - benchmark.VIRTFS_GUEST_TO_HOST - ) - return sample - - with tempfile.TemporaryDirectory() as temporary: - with ( - patch.object( - benchmark, "device_restore_commands", side_effect=commands - ), - patch.object( - benchmark, - "capture_device_restore_snapshot", - return_value=[], - ), - patch.object( - benchmark, - "run_device_restore_sample", - side_effect=run_sample, - ), - patch.object( - benchmark, - "summarize_device_restore_samples", - return_value={"stale_premature_callback_count": 0}, - ), - ): - benchmark.benchmark_device_restore_profile( - args, - Path("openvmm"), - Path("vmlinux"), - Path("initrd"), - "whp", - Path(temporary), - ) - - self.assertEqual(observed_seeds, [benchmark.VIRTFS_HOST_TO_GUEST]) - - def test_device_restore_profile_rejects_reused_output(self): - args = argparse.Namespace( - suite="device-restore-profile", - output=None, - output_dir=None, - platform="linux-kvm-baremetal", - ) - with tempfile.TemporaryDirectory() as temporary: - output_dir = Path(temporary) - (output_dir / "stale.log").write_text("stale", encoding="ascii") - args.output_dir = output_dir - with self.assertRaisesRegex(FileExistsError, "empty output directory"): - benchmark.run_workload_benchmarks( - args, - Path("openvmm"), - Path("vmlinux"), - Path("initrd"), - "kvm", - ) - - def test_benchmark_cpu_set_reserves_host_worker_capacity(self): - benchmark.validate_benchmark_cpu_set(set(range(10)), 8) - with self.assertRaisesRegex(ValueError, "requires at least 10"): - benchmark.validate_benchmark_cpu_set(set(range(9)), 8) - benchmark.validate_benchmark_cpu_set({0, 2, 4}, 1) - benchmark.validate_benchmark_cpu_set(set(range(8)), 8, 0) - with self.assertRaisesRegex(ValueError, "requires at least 8"): - benchmark.validate_benchmark_cpu_set(set(range(7)), 8, 0) - - def test_snapshot_capture_runs_smp_probe_for_selected_count(self): - args = argparse.Namespace( - warmups=0, - runs=1, - timeout=1.0, - processors=8, - teardown_mode="guest-exit", - ) - with patch.object( - benchmark, - "capture_snapshot", - return_value=(1.0, 1.0, 1.0, 1024), - ) as capture: - benchmark.benchmark_snapshot_capture(args, ["openvmm"]) - - self.assertNotIn("backend", capture.call_args.kwargs) - self.assertEqual(capture.call_args.kwargs["processors"], 8) - self.assertEqual(capture.call_args.kwargs["teardown_mode"], "guest-exit") - - def test_network_workload_runs_unmeasured_smp_interrupt_preflight(self): - args = argparse.Namespace( - net=None, - network_memory_mib=128, - processors=4, - warmups=0, - runs=1, - timeout=1.0, - teardown_mode="guest-exit", - ) - result: benchmark.BenchmarkResult = { - "samples_ms": [1.0], - "p50_ms": 1.0, - "p95_ms": 1.0, - "min_ms": 1.0, - "max_ms": 1.0, - "wall_samples_ms": [1.0], - "wall_p50_ms": 1.0, - "wall_p95_ms": 1.0, - "wall_min_ms": 1.0, - "wall_max_ms": 1.0, - "peak_rss_samples_bytes": [1], - "peak_rss_p50_bytes": 1, - "peak_rss_min_bytes": 1, - "peak_rss_max_bytes": 1, - "peak_rss_remeasured_count": 0, - "teardown_samples_ms": [1.0], - "teardown_completed_samples_ms": [1.0], - "teardown_timeout_count": 0, - "teardown_timeout_seconds": 5.0, - "teardown_p50_ms": 1.0, - "teardown_p95_ms": 1.0, - "teardown_min_ms": 1.0, - "teardown_max_ms": 1.0, - } - with ( - patch.object(benchmark, "run_guest_script") as probe, - patch.object(benchmark, "benchmark", return_value=result), - patch.object(benchmark, "capture_automatic_snapshot"), - ): - benchmark.benchmark_network_snapshot_workload( - args, - Path("openvmm"), - Path("vmlinux"), - Path("initrd"), - "kvm", - ) - - self.assertEqual(probe.call_count, 1) - self.assertIn("--network-profile", probe.call_args.args[0]) - self.assertIn("SMP-IOAPIC-OK", probe.call_args.args[1]) - self.assertEqual(probe.call_args.args[2], benchmark.SMP_PROBE_COMPLETION_MARKER) - - def test_shell_snapshot_restore_suite_only_measures_restore(self): - args = argparse.Namespace( - processors=8, - shell_memories=[512], - warmups=1, - runs=5, - timeout=40.0, - teardown_mode="guest-exit", - network_profile=None, - snapshot_profile=True, - ) - result = cast( - benchmark.BenchmarkResult, - { - "samples_ms": [10.0, 12.0], - "profile": { - "schema_version": 1, - "raw_samples": [], - "phases": { - "startup.partition_build": { - "exclusive": True, - "source": "openvmm", - "samples_ms": [1.0], - "p50_ms": 1.0, - "p95_ms": 1.0, - "min_ms": 1.0, - "max_ms": 1.0, - } - }, - }, - }, - ) - with ( - patch.object(benchmark, "capture_snapshot") as capture, - patch.object(benchmark, "benchmark", return_value=result) as run, - patch("sys.stdout", new_callable=io.StringIO) as output, - ): - benchmark.benchmark_shell_snapshot_restore_workload( - args, - Path("openvmm"), - Path("vmlinux"), - Path("initrd"), - "whp", - ) - - capture.assert_called_once() - capture_command = capture.call_args.args[0] - self.assertNotIn("shellsnap", capture_command) - self.assertEqual(capture.call_args.kwargs["processors"], 8) - self.assertEqual(capture.call_args.kwargs["teardown_mode"], "guest-exit") - run.assert_called_once() - restore_command = run.call_args.args[0] - self.assertIn("--restore-snapshot", restore_command) - self.assertEqual( - restore_command[restore_command.index("--processors") + 1], "8" - ) - self.assertIn("== 512 MiB ==", output.getvalue()) - self.assertIn("snapshot restore", output.getvalue()) - self.assertNotIn("cold boot", output.getvalue()) - self.assertIn('marker : "OPENVMM-SNAPSHOT-RESTORE-OK"', output.getvalue()) - self.assertTrue(run.call_args.kwargs["snapshot_profile"]) - self.assertEqual(run.call_args.kwargs["marker"], benchmark.RESTORE_MARKER) - self.assertTrue(run.call_args.kwargs["marker_must_be_line"]) - self.assertTrue(run.call_args.kwargs["guest_exit_prequeued"]) - self.assertIn( - "shell-snapshot-restore/whp/8vcpu/512-mib lifecycle phases:", - output.getvalue(), - ) - - def test_restore_vcpu_suite_reuses_one_capacity_snapshot_for_matrix(self): - args = argparse.Namespace( - processors=8, - memory_mib=512, - warmups=1, - runs=5, - timeout=40.0, - teardown_mode="guest-exit", - snapshot_profile=True, - ) - result = cast( - benchmark.BenchmarkResult, - { - "samples_ms": [10.0, 12.0], - "peak_rss_samples_bytes": [1024, 2048], - "profile": { - "schema_version": 1, - "raw_samples": [], - "phases": { - "restore.saved_state_restore": { - "exclusive": True, - "source": "openvmm", - "samples_ms": [2.0], - "p50_ms": 2.0, - "p95_ms": 2.0, - "min_ms": 2.0, - "max_ms": 2.0, - } - }, - }, - }, - ) - with ( - patch.object(benchmark, "capture_automatic_snapshot") as capture, - patch.object(benchmark, "benchmark", return_value=result) as run, - patch("sys.stdout", new_callable=io.StringIO) as output, - ): - benchmark.benchmark_snapshot_restore_vcpu_workload( - args, - Path("openvmm"), - Path("vmlinux"), - Path("initrd"), - "mshv", - ) - - capture.assert_called_once() - self.assertEqual(capture.call_args.kwargs["timeout"], args.timeout) - capture_command = capture.call_args.args[0] - self.assertEqual( - capture_command[capture_command.index("--processors") + 1], "8" - ) - self.assertTrue(any("maxcpus=1" in argument for argument in capture_command)) - self.assertEqual(run.call_count, 4) - targets: list[int] = [] - for invocation in run.call_args_list: - command = invocation.args[0] - self.assertEqual(invocation.kwargs["timeout"], args.timeout) - self.assertTrue(invocation.kwargs["snapshot_profile"]) - self.assertEqual(command[command.index("--processors") + 1], "8") - targets.append(int(command[command.index("--restore-processors") + 1])) - self.assertEqual(targets, [1, 2, 4, 8]) - self.assertIn("restore-online 8 vCPU", output.getvalue()) - self.assertIn( - "snapshot-restore-vcpu/mshv/capacity-8/online-1 lifecycle phases:", - output.getvalue(), - ) - - args.processors = 4 - with self.assertRaisesRegex(ValueError, "requires --processors 8"): - benchmark.benchmark_snapshot_restore_vcpu_workload( - args, - Path("openvmm"), - Path("vmlinux"), - Path("initrd"), - "kvm", - ) - - def test_benchmark_preserves_exact_process_wall_samples(self): - samples = [ - (10.0, 1024, 2.0, 13.5), - (12.0, 2048, 3.0, 16.5), - ] - with patch.object(benchmark, "measure_once", side_effect=samples): - result = benchmark.benchmark(["openvmm"], warmups=0, runs=2, timeout=1) - - self.assertEqual(result["samples_ms"], [10.0, 12.0]) - self.assertEqual(result["p95_ms"], 12.0) - self.assertEqual(result["wall_samples_ms"], [13.5, 16.5]) - self.assertEqual(result["wall_p50_ms"], 15.0) - self.assertEqual(result["wall_p95_ms"], 16.5) - self.assertEqual( - benchmark.SNAPSHOT_FILENAMES, - ("manifest.bin", "state.bin", "memory.bin"), - ) - - def test_benchmark_remeasures_attempt_without_marker_rss(self): - attempts = iter( - ( - (5.0, None, 1.0, 6.0), - (10.0, None, 7.0, 17.0), - (11.0, 2048, 7.0, 18.0), - (12.0, 4096, 8.0, 20.0), - ) - ) - - def measure_once( - *_args: object, - profile_sink: list[dict[str, object]] | None = None, - **_kwargs: object, - ) -> tuple[float, int | None, float | None, float]: - sample = next(attempts) - if profile_sink is not None: - profile_sink.append({"elapsed_ms": sample[0]}) - return sample - - before_each = MagicMock() - with ( - patch.object(benchmark, "measure_once", side_effect=measure_once), - patch.object( - benchmark, "summarize_lifecycle_profiles", return_value={} - ) as summarize, - patch("sys.stdout", new_callable=io.StringIO) as output, - ): - result = benchmark.benchmark( - ["openvmm"], - warmups=1, - runs=2, - timeout=1, - marker=benchmark.RESTORE_MARKER, - guest_exit_prequeued=True, - snapshot_profile=True, - before_each=before_each, - ) - - self.assertEqual(result["samples_ms"], [11.0, 12.0]) - self.assertEqual(result["peak_rss_samples_bytes"], [2048, 4096]) - self.assertEqual(result["peak_rss_min_bytes"], 2048) - self.assertEqual(result["peak_rss_remeasured_count"], 1) - self.assertEqual(result["teardown_samples_ms"], [7.0, 8.0]) - self.assertEqual(before_each.call_count, 4) - summarize.assert_called_once_with([{"elapsed_ms": 11.0}, {"elapsed_ms": 12.0}]) - self.assertIn("warmup 1/1: 5.000 ms, peak RSS=unavailable", output.getvalue()) - self.assertIn( - "sample 1/2: discarded attempt 1/3 (10.000 ms)", output.getvalue() - ) - - def test_benchmark_rejects_rss_missing_from_every_attempt(self): - with ( - patch.object( - benchmark, "measure_once", return_value=(10.0, None, 7.0, 17.0) - ) as measure, - patch("sys.stdout", new_callable=io.StringIO), - self.assertRaisesRegex( - RuntimeError, - f"{benchmark.PEAK_RSS_SAMPLE_ATTEMPTS} consecutive attempts " - "for sample 1/2", - ), - ): - benchmark.benchmark(["openvmm"], warmups=0, runs=2, timeout=1) - - self.assertEqual(measure.call_count, benchmark.PEAK_RSS_SAMPLE_ATTEMPTS) - - def test_nearest_rank_percentile(self): - self.assertEqual(benchmark.nearest_rank_percentile(range(1, 22), 95), 20) - self.assertEqual(benchmark.nearest_rank_percentile([7.0], 95), 7.0) - with self.assertRaisesRegex(ValueError, "without samples"): - benchmark.nearest_rank_percentile([], 95) - - def test_snapshot_capture_excludes_warmup_and_retains_last_snapshot(self): - with tempfile.TemporaryDirectory() as temporary: - retained = Path(temporary) / "retained" - observed_paths: list[Path] = [] - - def capture( - _command: object, - snapshot_path: Path, - **_kwargs: object, - ) -> tuple[float, float, float, int]: - snapshot_path.mkdir() - observed_paths.append(snapshot_path) - sample = float(len(observed_paths)) - return sample, sample, sample / 10, len(observed_paths) * 1024 - - args = argparse.Namespace( - warmups=1, - runs=2, - timeout=1.0, - processors=1, - teardown_mode="guest-exit", - ) - with patch.object(benchmark, "capture_snapshot", side_effect=capture): - result = benchmark.benchmark_snapshot_capture( - args, - ["openvmm"], - retained_snapshot_path=retained, - ) - - self.assertEqual(result["samples_ms"], [2.0, 3.0]) - self.assertEqual(result["peak_rss_samples_bytes"], [2048, 3072]) - self.assertEqual(result["peak_rss_p50_bytes"], 2560) - self.assertEqual(result["peak_rss_max_bytes"], 3072) - self.assertEqual(observed_paths[-1], retained) - self.assertTrue(retained.is_dir()) - - def test_snapshot_restore_reuses_provided_snapshot(self): - with tempfile.TemporaryDirectory() as temporary: - snapshot = Path(temporary) / "snapshot" - snapshot.mkdir() - args = argparse.Namespace( - warmups=1, - runs=2, - timeout=3.0, - teardown_mode="guest-exit", - network_profile=None, - processors=1, - ) - expected = {"p50_ms": 1.0} - with ( - patch.object(benchmark, "capture_snapshot") as capture, - patch.object(benchmark, "benchmark", return_value=expected) as run, - ): - result = benchmark.benchmark_snapshot_restore( - args, - Path("openvmm"), - "whp", - ["openvmm", "--kernel", "vmlinux"], - snapshot_path=snapshot, - ) - - self.assertIs(result, expected) - capture.assert_not_called() - self.assertIn(str(snapshot), run.call_args.args[0]) - self.assertEqual(run.call_args.kwargs["marker"], benchmark.RESTORE_MARKER) - self.assertTrue(run.call_args.kwargs["marker_must_be_line"]) - self.assertEqual(run.call_args.kwargs["teardown_mode"], "guest-exit") - self.assertTrue(run.call_args.kwargs["guest_exit_prequeued"]) - - def test_native_e2e_measures_and_reuses_snapshot_capture(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - openvmm = root / "openvmm" - executable = openvmm / "target" / "release" / "openvmm" - build_dir = root / "build" - executable.parent.mkdir(parents=True) - build_dir.mkdir() - (openvmm / "Cargo.toml").touch() - executable.touch() - (build_dir / "vmlinux").touch() - (build_dir / "initramfs.cpio.gz").touch() - output = root / "e2e.json" - args = nvx.parse_args( - [ - "benchmark", - "--suite", - "e2e", - "--backend", - "kvm", - "--openvmm-dir", - str(openvmm), - "--nvx-dir", - str(root), - "--cpus", - "0-2", - "--warmups", - "1", - "--runs", - "1", - "--skip-build", - "--output", - str(output), - ] - ) - run_result: benchmark.BenchmarkResult = { - "samples_ms": [10.0], - "p50_ms": 10.0, - "p95_ms": 10.0, - "min_ms": 10.0, - "max_ms": 10.0, - "wall_samples_ms": [12.0], - "wall_p50_ms": 12.0, - "wall_p95_ms": 12.0, - "wall_min_ms": 12.0, - "wall_max_ms": 12.0, - "peak_rss_samples_bytes": [1024], - "peak_rss_p50_bytes": 1024, - "peak_rss_min_bytes": 1024, - "peak_rss_max_bytes": 1024, - "peak_rss_remeasured_count": 0, - "teardown_samples_ms": [2.0], - "teardown_completed_samples_ms": [2.0], - "teardown_timeout_count": 0, - "teardown_timeout_seconds": 5.0, - "teardown_p50_ms": 2.0, - "teardown_p95_ms": 2.0, - "teardown_min_ms": 2.0, - "teardown_max_ms": 2.0, - } - capture_result: benchmark.SnapshotCaptureResult = { - "samples_ms": [3.0], - "p50_ms": 3.0, - "p95_ms": 3.0, - "min_ms": 3.0, - "max_ms": 3.0, - "request_to_publication_samples_ms": [3.0], - "request_to_publication_p50_ms": 3.0, - "request_to_publication_p95_ms": 3.0, - "request_to_publication_min_ms": 3.0, - "request_to_publication_max_ms": 3.0, - "post_publication_exit_samples_ms": [1.0], - "post_publication_exit_p50_ms": 1.0, - "post_publication_exit_p95_ms": 1.0, - "post_publication_exit_min_ms": 1.0, - "post_publication_exit_max_ms": 1.0, - "peak_rss_samples_bytes": [2048], - "peak_rss_p50_bytes": 2048, - "peak_rss_min_bytes": 2048, - "peak_rss_max_bytes": 2048, - } - with ( - patch.object(benchmark, "benchmark", return_value=run_result), - patch.object( - benchmark, - "benchmark_snapshot_capture", - return_value=capture_result, - ) as capture, - patch.object( - benchmark, - "benchmark_snapshot_restore", - return_value=run_result, - ) as restore, - ): - self.assertEqual(benchmark.run_native_linux(args), 0) - - retained = capture.call_args.kwargs["retained_snapshot_path"] - self.assertIsInstance(retained, Path) - self.assertEqual(restore.call_args.kwargs["snapshot_path"], retained) - document = json.loads(output.read_text(encoding="utf-8")) - self.assertEqual( - document["snapshot_capture"]["kvm"]["peak_rss_p50_bytes"], - 2048, - ) - - def test_kvm_worker_e2e_restores_the_measured_snapshot(self): - args = argparse.Namespace( - _stage_dir="/tmp/stage", - cpus="0", - memory_mib=128, - processors=4, - net=None, - suite="e2e", - warmups=1, - runs=1, - timeout=1.0, - teardown_mode="guest-exit", - ) - run_result = cast(benchmark.BenchmarkResult, {"p50_ms": 10.0}) - capture_result = cast(benchmark.SnapshotCaptureResult, {"p50_ms": 3.0}) - with ( - patch.object(benchmark, "benchmark", return_value=run_result), - patch.object( - benchmark, - "benchmark_snapshot_capture", - return_value=capture_result, - ) as capture, - patch.object( - benchmark, - "benchmark_snapshot_restore", - return_value=run_result, - ) as restore, - patch.object(benchmark, "print_summary"), - patch.object(benchmark, "print_snapshot_summary"), - ): - self.assertEqual(benchmark.run_kvm_worker(args), 0) - - retained = capture.call_args.kwargs["retained_snapshot_path"] - self.assertIsInstance(retained, Path) - self.assertEqual(restore.call_args.kwargs["snapshot_path"], retained) - - def test_virtfs_honors_host_termination_mode(self): - guest_exit = benchmark._virtfs_script(1, "guest-exit") - host_terminate = benchmark._virtfs_script(1, "host-terminate") - roundtrip = benchmark._virtfs_roundtrip_script("host-terminate") - - self.assertIn("nvx-exit 0", guest_exit) - self.assertNotIn("nvx-exit 0", host_terminate) - self.assertNotIn("nvx-exit 0", roundtrip) - - def test_virtfs_exchange_uses_lf_only_bytes(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - - def emulate_guest( - *_args: object, **_kwargs: object - ) -> benchmark.GuestCommandResult: - (root / "guest-visible").write_bytes(benchmark.VIRTFS_GUEST_TO_HOST) - deadline = benchmark.time.monotonic() + 1 - while benchmark.time.monotonic() < deadline: - if ( - root / "host-visible" - ).read_bytes() == benchmark.VIRTFS_HOST_TO_GUEST: - break - benchmark.time.sleep(0.001) - else: - self.fail("guest did not observe the byte-exact host update") - return {"text": "", "wall_ms": 1.0, "peak_rss_bytes": 1} - - with patch.object(benchmark, "run_guest_script", side_effect=emulate_guest): - benchmark._run_virtfs_roundtrip( - ["openvmm"], - root, - 1, - 1, - timeout=1, - windows_cpus=None, - teardown_mode="guest-exit", - ) - - self.assertEqual( - (root / "host-visible").read_bytes(), - b"host-to-guest\n", - ) - - def test_virtfs_excludes_declared_warmup(self): - args = argparse.Namespace( - virtfs_memory_mib=128, - payload_mib=1, - processors=1, - warmups=1, - timeout=1.0, - teardown_mode="guest-exit", - ) - io_result: benchmark.GuestCommandResult = { - "text": "1 byte copied, 1 s, 1 MB/s\n1 byte copied, 1 s, 2 MB/s\n", - "wall_ms": 1.0, - "peak_rss_bytes": 1, - } - roundtrip_result: benchmark.GuestCommandResult = { - "text": "", - "wall_ms": 2.0, - "peak_rss_bytes": 1, - } - with ( - patch.object(benchmark, "run_guest_script", return_value=io_result) as io, - patch.object( - benchmark, "_run_virtfs_roundtrip", return_value=roundtrip_result - ) as roundtrip, - ): - benchmark.benchmark_virtfs_workload( - args, - Path("openvmm"), - Path("vmlinux"), - Path("initrd"), - "kvm", - runs=3, - ) - - self.assertEqual(io.call_count, 4) - self.assertEqual(roundtrip.call_count, 4) - - def test_device_io_retains_failure_and_resumes_completed_attempts(self): - args = argparse.Namespace( - processors=1, - warmups=1, - runs=1, - device_io_duration_seconds=1.0, - device_io_size_mib=64, - device_io_port=5201, - net="192.0.2.2/24", - timeout=1.0, - teardown_mode="guest-exit", - ) - - def guest_result(command: list[str], *_args: object, **_kwargs: object): - if "--net" in command: - payload = ( - 'NVX_DEVICE_IO_GUEST_RESULT={"device":"network",' - '"operation":"roundtrip","operations":100,' - '"bytes_per_operation":64,"elapsed_ns":1000000000}\n' - ) - else: - payload = ( - 'NVX_DEVICE_IO_GUEST_RESULT={"device":"file",' - '"operation":"write","operations":100,' - '"bytes_per_operation":4096,"elapsed_ns":1000000000}\n' - 'NVX_DEVICE_IO_GUEST_RESULT={"device":"file",' - '"operation":"read","operations":100,' - '"bytes_per_operation":4096,"elapsed_ns":1000000000}\n' - ) - return {"text": payload, "wall_ms": 1.0, "peak_rss_bytes": 1} - - with tempfile.TemporaryDirectory() as temporary: - output = Path(temporary) / "device-io.log" - with ( - patch.object(benchmark, "host_ipv4_address", return_value="192.0.2.1"), - patch.object(benchmark, "UdpEchoServer") as echo_server, - patch.object( - benchmark, - "run_guest_script", - side_effect=guest_result, - ) as run_guest, - ): - echo_server.return_value.__enter__.return_value = None - benchmark.benchmark_device_io_workload( - args, - Path("openvmm"), - Path("vmlinux"), - Path("initramfs"), - "mshv", - output_path=output, - ) - self.assertEqual(run_guest.call_count, 6) - network_commands = [ - call.args[0] - for call in run_guest.call_args_list - if "--net" in call.args[0] - ] - self.assertEqual(len(network_commands), 2) - self.assertIn("192.0.2.2/24", network_commands[0]) - self.assertIn("microvm", network_commands[0]) - self.assertNotIn("microvm-v2", network_commands[0]) - block_commands = [ - call.args[0] - for call in run_guest.call_args_list - if "--microvm-sandbox-block" in call.args[0] - ] - self.assertEqual(len(block_commands), 2) - self.assertNotIn("--virtio-blk", block_commands[0]) - block_argument = block_commands[0][ - block_commands[0].index("--microvm-sandbox-block") + 1 - ] - self.assertTrue(block_argument.startswith("scratch:file:")) - - benchmark.benchmark_device_io_workload( - args, - Path("openvmm"), - Path("vmlinux"), - Path("initramfs"), - "mshv", - output_path=output, - ) - self.assertEqual(run_guest.call_count, 6) - - records = benchmark._read_device_io_records(output) - self.assertEqual(len(records), 6) - retained = [record for record in records if not record["warmup"]] - self.assertEqual([record["sample_index"] for record in retained], [0, 0, 0]) - - def test_device_io_missing_guest_result_becomes_failure(self): - result: benchmark.GuestCommandResult = { - "text": ( - 'NVX_DEVICE_IO_GUEST_RESULT={"device":"file",' - '"operation":"read","operations":1,' - '"bytes_per_operation":4096,"elapsed_ns":1}\n' - ), - "wall_ms": 1.0, - "peak_rss_bytes": 1, - } - - record = benchmark._device_io_attempt_record("virtio-blk", 0, 0, result, None) - - self.assertEqual(record["status"], "failure") - self.assertIn("missing", str(record["error"])) - - def test_managed_tap_cleanup_uses_openvmm_pid(self): - query: subprocess.CompletedProcess[str] = subprocess.CompletedProcess( - ["ip"], 0, "", "" - ) - removed: subprocess.CompletedProcess[str] = subprocess.CompletedProcess( - ["sudo", "-n", "ip"], 0, "", "" - ) - with ( - patch.object(benchmark.sys, "platform", "linux"), - patch.object(benchmark.os, "geteuid", return_value=1000, create=True), - patch.object( - benchmark.subprocess, "run", side_effect=(query, removed) - ) as run, - ): - benchmark.cleanup_managed_tap(1234) - - self.assertEqual( - run.call_args_list[1].args[0], - [ - "sudo", - "-n", - "ip", - "tuntap", - "del", - "dev", - "ovm1234", - "mode", - "tap", - ], - ) - - def test_performance_suite_writes_four_canonical_logs(self): - with tempfile.TemporaryDirectory() as temporary: - output = Path(temporary) - (output / "snapshot.log").write_text("stale", encoding="ascii") - (output / "snapshot-hello.log").write_text("stale", encoding="ascii") - args = nvx.parse_args( - [ - "benchmark", - "--suite", - "performance", - "--backend", - "whp", - "--platform", - "windows-whp-baremetal", - "--processors", - "8", - "--output-dir", - str(output), - ] - ) - with ( - patch.object(benchmark, "benchmark_cold_start_workload") as cold, - patch.object(benchmark, "benchmark_virtfs_workload") as virtfs, - patch.object(benchmark, "benchmark_shell_snapshot_workload") as shell, - patch.object( - benchmark, "benchmark_network_snapshot_workload" - ) as network, - ): - result = benchmark.run_workload_benchmarks( - args, - Path("openvmm.exe"), - Path("vmlinux"), - Path("initramfs.cpio.gz"), - "whp", - ) - - self.assertEqual(result, 0) - self.assertEqual( - {path.name for path in output.iterdir()}, - { - "benchmark-metadata.json", - "cold-start.log", - "virtfs.log", - "shell-snapshot.log", - "network.log", - }, - ) - metadata = json.loads( - (output / "benchmark-metadata.json").read_text(encoding="utf-8") - ) - self.assertEqual(metadata["platform"], "windows-whp-baremetal") - self.assertEqual(metadata["backend"], "whp") - self.assertEqual(metadata["microvm_abi_version"], 2) - self.assertEqual(metadata["processors"], 8) - self.assertEqual(metadata["host_affinity_set"], args.cpus) - self.assertEqual(metadata["host_cpu_reserve"], args.host_cpu_reserve) - self.assertEqual( - metadata["scratch_directory"], - str(Path(tempfile.gettempdir()).resolve()), - ) - cold.assert_called_once() - self.assertEqual(virtfs.call_args.kwargs["runs"], 3) - shell.assert_called_once() - network.assert_called_once() - - def test_restore_only_suite_writes_only_its_canonical_log(self): - with tempfile.TemporaryDirectory() as temporary: - output = Path(temporary) - (output / "cold-start.log").write_text("stale", encoding="ascii") - (output / "shell-snapshot.log").write_text("stale", encoding="ascii") - (output / "acceptance.json").write_text("stale", encoding="ascii") - args = nvx.parse_args( - [ - "benchmark", - "--suite", - "shell-snapshot-restore", - "--backend", - "whp", - "--platform", - "windows-whp-virtual-machine", - "--processors", - "8", - "--shell-memories", - "512", - "--output-dir", - str(output), - ] - ) - with patch.object( - benchmark, "benchmark_shell_snapshot_restore_workload" - ) as restore: - result = benchmark.run_workload_benchmarks( - args, - Path("openvmm.exe"), - Path("vmlinux"), - Path("initramfs.cpio.gz"), - "whp", - ) - - self.assertEqual(result, 0) - self.assertEqual( - {path.name for path in output.iterdir()}, - {"benchmark-metadata.json", "shell-snapshot-restore.log"}, - ) - restore.assert_called_once() - - def test_mshv_performance_suite_includes_network(self): - with tempfile.TemporaryDirectory() as temporary: - output = Path(temporary) - args = nvx.parse_args( - [ - "benchmark", - "--suite", - "performance", - "--backend", - "mshv", - "--output-dir", - str(output), - ] - ) - with ( - patch.object(benchmark, "benchmark_cold_start_workload"), - patch.object(benchmark, "benchmark_virtfs_workload"), - patch.object(benchmark, "benchmark_shell_snapshot_workload"), - patch.object( - benchmark, "benchmark_network_snapshot_workload" - ) as network, - ): - benchmark.run_workload_benchmarks( - args, - Path("openvmm"), - Path("vmlinux"), - Path("initramfs.cpio.gz"), - "mshv", - ) - - self.assertEqual( - {path.name for path in output.iterdir()}, - { - "benchmark-metadata.json", - "cold-start.log", - "virtfs.log", - "shell-snapshot.log", - "network.log", - }, - ) - network.assert_called_once() - - def test_mshv_runs_explicit_network_snapshot_suite(self): - args = nvx.parse_args( - ["benchmark", "--suite", "network-snapshot", "--backend", "mshv"] - ) - - with patch.object(benchmark, "benchmark_network_snapshot_workload") as network: - benchmark.run_workload_benchmarks( - args, - Path("openvmm"), - Path("vmlinux"), - Path("initramfs.cpio.gz"), - "mshv", - ) - - network.assert_called_once() - - def test_performance_suite_defaults_to_data_runs_platform(self): - with tempfile.TemporaryDirectory() as temporary: - repository = Path(temporary) - backend = "whp" if os.name == "nt" else "kvm" - platform = "windows-whp" if os.name == "nt" else "linux-kvm" - args = nvx.parse_args( - [ - "benchmark", - "--suite", - "performance", - "--backend", - backend, - "--nvx-dir", - str(repository), - ] - ) - with ( - patch.object(benchmark, "benchmark_cold_start_workload"), - patch.object(benchmark, "benchmark_virtfs_workload"), - patch.object(benchmark, "benchmark_shell_snapshot_workload"), - patch.object(benchmark, "benchmark_network_snapshot_workload"), - ): - benchmark.run_workload_benchmarks( - args, - Path("openvmm.exe"), - Path("vmlinux"), - Path("initramfs.cpio.gz"), - backend, - ) - - output = repository / "data" / "runs" / f"{platform}-microvm-v2-1vcpu" - self.assertEqual( - {path.name for path in output.iterdir()}, - { - "benchmark-metadata.json", - "cold-start.log", - "virtfs.log", - "shell-snapshot.log", - "network.log", - }, - ) - - def test_device_io_defaults_to_dedicated_abi_v2_directory(self): - with tempfile.TemporaryDirectory() as temporary: - repository = Path(temporary).resolve() - backend = "whp" if os.name == "nt" else "kvm" - platform = "windows-whp" if os.name == "nt" else "linux-kvm" - args = nvx.parse_args( - [ - "benchmark", - "--suite", - "device-io", - "--backend", - backend, - "--nvx-dir", - str(repository), - ] - ) - with ( - patch.object( - benchmark, - "write_benchmark_metadata", - return_value=Path("benchmark-metadata.json"), - ) as metadata, - patch.object( - benchmark, "benchmark_device_io_workload", return_value=0 - ) as device_io, - ): - benchmark.run_workload_benchmarks( - args, - Path("openvmm.exe"), - Path("vmlinux"), - Path("initramfs.cpio.gz"), - backend, - ) - - output = ( - repository - / "data" - / "runs" - / platform - / "microvm-v2" - / "1vcpu" - / "device-io" - ) - self.assertEqual( - device_io.call_args.kwargs["output_path"], - output / "device-io.log", - ) - self.assertEqual(metadata.call_args.args[1], output) - - def test_package_command_forwards_parsed_options(self): - args = nvx.parse_args( - [ - "package", - "--version", - "2.0.0", - "--destination", - "dist/custom", - "--binary-only", - "--force", - ] - ) - - with patch.object(nvx, "package_release") as package_release: - args.handler(args) - - package_release.assert_called_once_with( - version="2.0.0", - destination=Path("dist/custom"), - include_source=False, - force=True, - ) - - def test_archive_release_command_forwards_paths(self): - args = nvx.parse_args( - [ - "archive-release", - "--source", - "dist/package", - "--destination", - "dist/package.zip", - ] - ) - - with patch.object(nvx, "create_release_archive") as create_archive: - args.handler(args) - - create_archive.assert_called_once_with( - Path("dist/package"), - Path("dist/package.zip"), - ) - - def test_download_command_selects_host_release(self): - args = nvx.parse_args(["download", "--repository", "example/nvx"]) - expected_platform = "windows-whp" if os.name == "nt" else "linux-kvm" - - with patch.object(nvx, "download_checkout_release") as download_release: - args.handler(args) - - download_release.assert_called_once_with("example/nvx", expected_platform) - - def test_download_command_defaults_to_integration_repository(self): - args = nvx.parse_args(["download"]) - expected_platform = "windows-whp" if os.name == "nt" else "linux-kvm" - - with patch.object(nvx, "download_checkout_release") as download_release: - args.handler(args) - - download_release.assert_called_once_with("microsoft/nvx", expected_platform) - - -AUTHORIZATION_VALUE = "Bearer placeholder-value" - - -def _git_history(*commits: tuple[str, str | None]) -> Callable[..., str]: - """Fakes git_output for HEAD's first-parent history and its VERSION files.""" - - versions = dict(commits) - - def git_output(*arguments: str) -> str: - if arguments[:2] == ("rev-list", "--first-parent") and arguments[3:] == ( - "HEAD", - ): - limit = int(arguments[2].removeprefix("--max-count=")) - return "\n".join(list(versions)[:limit]) - if arguments[0] != "show" or not arguments[1].endswith(":VERSION"): - raise AssertionError(f"unexpected git command: {arguments}") - version = versions[arguments[1].removesuffix(":VERSION")] - if version is None: - raise subprocess.CalledProcessError(128, ["git", *arguments]) - return version - - return git_output - - -def _missing_release() -> urllib.error.HTTPError: - return urllib.error.HTTPError( - "https://api.github.invalid/releases/tags/missing", - 404, - "Not Found", - http.client.HTTPMessage(), - io.BytesIO(b'{"message": "Not Found"}'), - ) - - -class ReleaseTests(unittest.TestCase): - def test_alpine_source_validation_rejects_malformed_manifest(self): - with tempfile.TemporaryDirectory() as temporary: - source_dir = Path(temporary) - alpine_dir = source_dir / "alpine" - alpine_dir.mkdir() - (alpine_dir / "manifest.json").write_text("{", encoding="utf-8") - - with ( - patch.object(BuildConstants, "SOURCE_DIR", source_dir), - self.assertRaisesRegex( - common.ScriptError, - "invalid collected Alpine source manifest", - ), - ): - release._validate_alpine_sources([]) - - def test_alpine_source_validation_checks_recorded_executables(self): - with tempfile.TemporaryDirectory() as temporary: - source_dir = Path(temporary) - alpine_dir = source_dir / "alpine" - recipe = alpine_dir / "recipes" / "c0ffee" / "main" / "foo" - recipe.mkdir(parents=True) - (recipe / "APKBUILD").write_bytes(b"pkgname=foo\n") - (recipe / "foo.post-install").write_bytes(b"#!/bin/sh\n") - installer = "recipes/c0ffee/main/foo/foo.post-install" - current = AlpineBuildConstants.SOURCE_MANIFEST_FORMAT - cases: dict[str, tuple[dict[str, object], str | None]] = { - "recorded executable": ( - {"format": current, "executables": [installer]}, - None, - ), - "older format": ( - {"format": 1, "executables": [installer]}, - "unsupported manifest format", - ), - "missing executables": ({"format": current}, "invalid executables"), - "unlisted executable": ( - {"format": current, "executables": [f"{installer}.missing"]}, - "not a checksummed recipe file", - ), - "executable outside recipes": ( - {"format": current, "executables": ["manifest.json"]}, - "not a checksummed recipe file", - ), - } - for name, (document, error) in cases.items(): - (alpine_dir / "manifest.json").write_text( - json.dumps({"packages": [], **document}), encoding="utf-8" - ) - common.write_sha256_sums(alpine_dir) - with ( - self.subTest(case=name), - patch.object(BuildConstants, "SOURCE_DIR", source_dir), - ): - if error is None: - self.assertEqual( - release._validate_alpine_sources([]), {installer} - ) - continue - with self.assertRaisesRegex(common.ScriptError, error): - release._validate_alpine_sources([]) - - def test_alpine_source_archive_applies_recorded_modes(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - source_dir = root / "sources" - recipe = source_dir / "alpine" / "recipes" / "c0ffee" / "main" / "foo" - recipe.mkdir(parents=True) - for name in ("APKBUILD", "foo.post-install"): - (recipe / name).write_bytes(b"#!/bin/sh\n") - package_manifest = root / "initramfs.cpio.gz.packages.json" - package_manifest.write_text("{}", encoding="utf-8") - # Report every mode as Windows does, whatever the host stores. - for path in (*source_dir.rglob("*"), package_manifest): - path.chmod(0o777 if path.is_dir() else 0o666) - output = root / "alpine-source.tar.gz" - - with ( - patch.object(BuildConstants, "SOURCE_DIR", source_dir), - contextlib.redirect_stdout(io.StringIO()), - ): - release._alpine_source_archive( - output, - "1.0.0", - [package_manifest], - frozenset({"recipes/c0ffee/main/foo/foo.post-install"}), - ) - - with tarfile.open(output, "r:gz") as package: - modes = {member.name: member.mode for member in package.getmembers()} - prefix = "nvx-alpine-source-1.0.0" - self.assertEqual( - modes, - { - f"{prefix}/sources": 0o755, - f"{prefix}/sources/recipes": 0o755, - f"{prefix}/sources/recipes/c0ffee": 0o755, - f"{prefix}/sources/recipes/c0ffee/main": 0o755, - f"{prefix}/sources/recipes/c0ffee/main/foo": 0o755, - f"{prefix}/sources/recipes/c0ffee/main/foo/APKBUILD": 0o644, - f"{prefix}/sources/recipes/c0ffee/main/foo/foo.post-install": ( - 0o755 - ), - f"{prefix}/manifests/initramfs.cpio.gz.packages.json": 0o644, - }, - ) - - def test_downloads_release_built_from_head(self): - published = { - "draft": False, - "prerelease": True, - "tag_name": "v1.2.3-dev.aaaaaaaaaaaa", - "assets": [ - { - "name": "nvx-invalid-linux-kvm.tar.gz", - "url": "https://api.example.invalid/invalid", - "size": True, - }, - { - "name": "nvx-1.2.3-windows-whp.zip", - "url": "https://api.example.invalid/windows", - "size": 200, - }, - { - "name": "nvx-1.2.3-linux-kvm.tar.gz", - "url": "https://api.example.invalid/linux", - "size": 300, - }, - ], - } - - with ( - patch.object( - release, - "git_output", - side_effect=_git_history(("a" * 40, "1.2.3"), ("b" * 40, "1.2.3")), - ) as git_output, - patch( - "nvx_tools.release.urllib.request.OpenerDirector.open", - return_value=io.BytesIO(json.dumps(published).encode("utf-8")), - ) as opener_open, - contextlib.redirect_stdout(io.StringIO()) as stdout, - ): - asset = release._checkout_release_asset( - "example/nvx", - "linux-kvm", - "token", - ) - - self.assertEqual( - git_output.call_args_list[0], - call( - "rev-list", - "--first-parent", - f"--max-count={release._RELEASE_SEARCH_COMMITS}", - "HEAD", - ), - ) - self.assertEqual(opener_open.call_count, 1) - self.assertEqual( - opener_open.call_args.args[0].full_url, - "https://api.github.com/repos/example/nvx/releases/tags/" - "v1.2.3-dev.aaaaaaaaaaaa", - ) - self.assertEqual( - asset, - release._ReleaseAsset( - "v1.2.3-dev.aaaaaaaaaaaa", - "nvx-1.2.3-linux-kvm.tar.gz", - "https://api.example.invalid/linux", - 300, - ), - ) - self.assertEqual(stdout.getvalue(), "") - - def test_skips_draft_release_built_from_head(self): - def response(commit: str, draft: bool) -> io.BytesIO: - tag = f"v1.2.3-dev.{commit[:12]}" - document = { - "draft": draft, - "tag_name": tag, - "assets": [ - { - "name": "nvx-1.2.3-linux-kvm.tar.gz", - "url": f"https://api.example.invalid/{tag}/linux-kvm.tar.gz", - "size": 300, - } - ], - } - return io.BytesIO(json.dumps(document).encode("utf-8")) - - with ( - patch.object( - release, - "git_output", - side_effect=_git_history(("a" * 40, "1.2.3"), ("b" * 40, "1.2.3")), - ), - patch( - "nvx_tools.release.urllib.request.OpenerDirector.open", - side_effect=[ - response("a" * 40, True), - response("b" * 40, False), - ], - ), - contextlib.redirect_stdout(io.StringIO()) as stdout, - ): - asset = release._checkout_release_asset( - "example/nvx", - "linux-kvm", - "token", - ) - - self.assertEqual(asset.tag, "v1.2.3-dev.bbbbbbbbbbbb") - self.assertEqual( - stdout.getvalue(), - ">> no linux-kvm release for HEAD; using v1.2.3-dev.bbbbbbbbbbbb, " - "1 first-parent commit earlier\n", - ) - - def test_walks_first_parent_history_to_the_nearest_release(self): - def published(version: str, commit: str, *packages: str) -> io.BytesIO: - tag = f"v{version}-dev.{commit[:12]}" - document = { - "tag_name": tag, - "assets": [ - { - "name": f"nvx-{version}-{package}", - "url": f"https://api.example.invalid/{tag}/{package}", - "size": 100, - } - for package in packages - ], - } - return io.BytesIO(json.dumps(document).encode("utf-8")) - - # HEAD has no release, b has no VERSION file, c's release has no KVM - # package, and d's release predates a version bump. - history = _git_history( - ("a" * 40, "1.2.3"), - ("b" * 40, None), - ("c" * 40, "1.2.3"), - ("d" * 40, "1.2.2"), - ("e" * 40, "1.2.2"), - ) - responses = [ - _missing_release(), - published("1.2.3", "c" * 40, "windows-whp.zip"), - published("1.2.2", "d" * 40, "windows-whp.zip", "linux-kvm.tar.gz"), - ] - - with ( - patch.object(release, "git_output", side_effect=history), - patch( - "nvx_tools.release.urllib.request.OpenerDirector.open", - side_effect=responses, - ) as opener_open, - contextlib.redirect_stdout(io.StringIO()) as stdout, - ): - asset = release._checkout_release_asset("example/nvx", "linux-kvm", None) - - tags = "https://api.github.com/repos/example/nvx/releases/tags" - self.assertEqual( - [request.args[0].full_url for request in opener_open.call_args_list], - [ - f"{tags}/v1.2.3-dev.aaaaaaaaaaaa", - f"{tags}/v1.2.3-dev.cccccccccccc", - f"{tags}/v1.2.2-dev.dddddddddddd", - ], - ) - self.assertEqual(asset.tag, "v1.2.2-dev.dddddddddddd") - self.assertEqual( - asset.url, - "https://api.example.invalid/v1.2.2-dev.dddddddddddd/linux-kvm.tar.gz", - ) - self.assertEqual( - stdout.getvalue(), - ">> no linux-kvm release for HEAD; using v1.2.2-dev.dddddddddddd, " - "3 first-parent commits earlier\n", - ) - - def test_reports_when_no_searched_commit_has_a_release(self): - with ( - patch.object( - release, - "git_output", - side_effect=_git_history(("a" * 40, "1.2.3"), ("b" * 40, "1.2.3")), - ), - patch( - "nvx_tools.release.urllib.request.OpenerDirector.open", - side_effect=[_missing_release(), _missing_release(), io.BytesIO(b"{}")], - ) as opener_open, - self.assertRaisesRegex( - release.ScriptError, - r"no example/nvx release with an NVX package for linux-kvm was built " - r"from HEAD or its first-parent ancestors \(2 commits searched\)", - ), - ): - release._checkout_release_asset("example/nvx", "linux-kvm", None) - - self.assertEqual( - opener_open.call_args.args[0].full_url, - "https://api.github.com/repos/example/nvx", - ) - - def test_download_requires_a_git_checkout(self): - failure = subprocess.CalledProcessError( - 128, - ["git", "rev-list"], - stderr="fatal: not a git repository (or any of the parent directories)\n", - ) - - with ( - patch.object(release, "git_output", side_effect=failure), - patch( - "nvx_tools.release.urllib.request.OpenerDirector.open" - ) as opener_open, - self.assertRaisesRegex( - release.ScriptError, - "download needs a Git checkout to find its release: " - "fatal: not a git repository", - ), - ): - release._checkout_release_asset("example/nvx", "linux-kvm", None) - - opener_open.assert_not_called() - - def test_version_lookup_reports_git_execution_failures(self): - commit = "a" * 40 - failures = ( - OSError("git unavailable"), - subprocess.TimeoutExpired(["git", "show"], 30), - ) - for failure in failures: - with ( - self.subTest(failure=type(failure).__name__), - patch.object(release, "git_output", side_effect=failure), - self.assertRaisesRegex( - release.ScriptError, - f"download failed to read VERSION at {commit}", - ) as raised, - ): - release._version_at(commit) - - self.assertIs(raised.exception.__cause__, failure) - - def test_first_parent_history_and_versions_come_from_git(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - environment = { - key: value - for key, value in os.environ.items() - if not key.startswith("GIT_CONFIG_") - } - environment.update( - GIT_AUTHOR_NAME="NVX Tests", - GIT_AUTHOR_EMAIL="nvx-tests@example.com", - GIT_COMMITTER_NAME="NVX Tests", - GIT_COMMITTER_EMAIL="nvx-tests@example.com", - ) - - def git(*args: str, data: bytes = b"") -> str: - result = subprocess.run( - ["git", "-C", str(root), *args], - input=data, - env=environment, - capture_output=True, - check=True, - ) - return result.stdout.decode("ascii").strip() - - def commit(version: str | None, *parents: str) -> str: - entries = b"" - if version is not None: - blob = git( - "hash-object", "-w", "--stdin", data=f"{version}\n".encode() - ) - entries = f"100644 blob {blob}\tVERSION\n".encode() - tree = git("mktree", data=entries) - parent_arguments = [ - argument for parent in parents for argument in ("-p", parent) - ] - return git( - "commit-tree", "--no-gpg-sign", tree, *parent_arguments, "-m", "x" - ) - - git("init", "-q") - first = commit("1.2.2") - merged = commit("9.9.9", first) - second = commit("1.2.3", first) - merge = commit("1.2.3", second, merged) - head = commit(None, merge) - git("update-ref", "HEAD", head) - - with patch.object(BuildConstants, "REPO_ROOT", root): - commits = release._first_parent_commits(10) - limited = release._first_parent_commits(2) - versions = [release._version_at(commit_id) for commit_id in commits] - - self.assertEqual(commits, [head, merge, second, first]) - self.assertEqual(limited, [head, merge]) - self.assertEqual(versions, [None, "1.2.3", "1.2.3", "1.2.2"]) - - def test_forbidden_release_query_reports_github_message_and_hint(self): - error = urllib.error.HTTPError( - "https://api.github.invalid/releases", - 403, - "Forbidden", - http.client.HTTPMessage(), - io.BytesIO( - json.dumps( - {"message": "Resource protected by organization SAML enforcement."} - ).encode("utf-8") - ), - ) - - with ( - patch.object( - release, - "git_output", - side_effect=_git_history(("a" * 40, "1.2.3")), - ), - patch( - "nvx_tools.release.urllib.request.OpenerDirector.open", - side_effect=error, - ), - self.assertRaises(release.ScriptError) as context, - ): - release._checkout_release_asset("example/nvx", "linux-kvm", "token") - - message = str(context.exception) - self.assertIn("HTTP 403", message) - self.assertIn("organization SAML enforcement", message) - self.assertIn("read access to the repository contents", message) - - def test_unauthenticated_release_query_reports_token_hint(self): - error = urllib.error.HTTPError( - "https://api.github.invalid/releases", - 404, - "Not Found", - http.client.HTTPMessage(), - io.BytesIO(b"not json"), - ) - - # The release lookup's 404 is a miss; the repository's 404 is the error. - with ( - patch.object( - release, - "git_output", - side_effect=_git_history(("a" * 40, "1.2.3")), - ), - patch( - "nvx_tools.release.urllib.request.OpenerDirector.open", - side_effect=error, - ) as opener_open, - self.assertRaisesRegex(release.ScriptError, "HTTP 404.*set GH_TOKEN"), - ): - release._checkout_release_asset("example/nvx", "linux-kvm", None) - - self.assertEqual(opener_open.call_count, 2) - - def test_exhausted_rate_limit_reports_retry_hint(self): - headers = http.client.HTTPMessage() - headers["x-ratelimit-remaining"] = "0" - error = urllib.error.HTTPError( - "https://api.github.invalid/releases", - 403, - "Forbidden", - headers, - io.BytesIO(b""), - ) - - with ( - patch.object( - release, - "git_output", - side_effect=_git_history(("a" * 40, "1.2.3")), - ), - patch( - "nvx_tools.release.urllib.request.OpenerDirector.open", - side_effect=error, - ), - self.assertRaisesRegex(release.ScriptError, "rate limit is exhausted"), - ): - release._checkout_release_asset("example/nvx", "linux-kvm", "token") - - def test_forbidden_query_falls_back_to_public_release(self): - error = urllib.error.HTTPError( - "https://api.github.invalid/releases", - 403, - "Forbidden", - http.client.HTTPMessage(), - io.BytesIO( - json.dumps( - {"message": "Resource protected by organization SAML enforcement."} - ).encode("utf-8") - ), - ) - public_response = io.BytesIO( - json.dumps( - { - "draft": False, - "tag_name": "v1.2.3-dev.aaaaaaaaaaaa", - "assets": [ - { - "name": "nvx-1.2.3-linux-kvm.tar.gz", - "url": "https://api.example.invalid/linux", - "size": 300, - } - ], - } - ).encode("utf-8") - ) - - with ( - patch.object( - release, - "git_output", - side_effect=_git_history(("a" * 40, "1.2.3")), - ), - patch( - "nvx_tools.release.urllib.request.OpenerDirector.open", - side_effect=[error, public_response], - ) as opener_open, - patch("sys.stderr", io.StringIO()) as stderr, - ): - asset, download_token = release._checkout_release_asset_with_fallback( - "example/nvx", - "linux-kvm", - "token", - ) - - self.assertEqual(opener_open.call_count, 2) - authenticated_request = opener_open.call_args_list[0].args[0] - public_request = opener_open.call_args_list[1].args[0] - self.assertIsNotNone(authenticated_request.get_header("Authorization")) - self.assertIsNone(public_request.get_header("Authorization")) - self.assertEqual(asset.tag, "v1.2.3-dev.aaaaaaaaaaaa") - self.assertIsNone(download_token) - self.assertIn("retrying without credentials", stderr.getvalue()) - - def test_asset_download_drops_credentials_across_origins(self): - handler = common._CrossOriginRedirectHandler() - request = urllib.request.Request( - "https://api.github.invalid/assets/1", - headers={"Authorization": AUTHORIZATION_VALUE, "Accept": "*/*"}, - ) - - redirected = handler.redirect_request( - request, - io.BytesIO(b""), - 302, - "Found", - http.client.HTTPMessage(), - "https://objects.github.invalid/assets/1?signature=abc", - ) - same_host = handler.redirect_request( - request, - io.BytesIO(b""), - 302, - "Found", - http.client.HTTPMessage(), - "https://api.github.invalid/assets/2", - ) - downgraded = handler.redirect_request( - request, - io.BytesIO(b""), - 302, - "Found", - http.client.HTTPMessage(), - "http://api.github.invalid/assets/3", - ) - - self.assertIsNotNone(redirected) - self.assertIsNotNone(same_host) - self.assertIsNotNone(downgraded) - assert ( - redirected is not None and same_host is not None and downgraded is not None - ) - self.assertIsNone(redirected.get_header("Authorization")) - self.assertEqual(redirected.get_header("Accept"), "*/*") - self.assertEqual(same_host.get_header("Authorization"), AUTHORIZATION_VALUE) - self.assertIsNone(downgraded.get_header("Authorization")) - - def test_rejected_token_falls_back_to_public_release(self): - asset = release._ReleaseAsset( - "v1.2.3", - "nvx-1.2.3-linux-kvm.tar.gz", - "https://api.github.invalid/assets/1", - 4, - ) - authenticated_error = release._GitHubReleaseQueryError( - 403, - "authenticated query failed", - ) - stderr = io.StringIO() - - def write_archive( - _url: str, - destination: Path, - **kwargs: object, - ) -> None: - self.assertIsInstance( - kwargs["opener"], - urllib.request.OpenerDirector, - ) - self.assertNotIn("Authorization", cast(dict[str, str], kwargs["headers"])) - destination.write_bytes(b"data") - - with ( - patch.dict(os.environ, {"GH_TOKEN": "token"}), - patch.object( - release, - "_checkout_release_asset", - side_effect=[authenticated_error, asset], - ) as latest_release_asset, - patch.object(release, "download", side_effect=write_archive) as download, - patch.object(release, "_install_release_archive") as install, - patch("sys.stderr", stderr), - ): - release.download_checkout_release("example/nvx", "linux-kvm") - - self.assertEqual( - latest_release_asset.call_args_list, - [ - call("example/nvx", "linux-kvm", "token"), - call("example/nvx", "linux-kvm", None), - ], - ) - self.assertEqual(download.call_count, 1) - install.assert_called_once() - self.assertIn("retrying without credentials", stderr.getvalue()) - - def test_failed_public_fallback_preserves_authenticated_error(self): - authenticated_error = release._GitHubReleaseQueryError( - 403, - "authenticated query failed", - ) - public_error = release._GitHubReleaseQueryError( - 404, - "public query failed", - ) - - with ( - patch.dict(os.environ, {"GH_TOKEN": "token"}), - patch.object( - release, - "_checkout_release_asset", - side_effect=[authenticated_error, public_error], - ), - patch("sys.stderr", io.StringIO()), - self.assertRaisesRegex( - release.ScriptError, - "authenticated query failed", - ), - ): - release.download_checkout_release("example/nvx", "linux-kvm") - - def test_authenticated_asset_download_uses_credential_safe_opener(self): - asset = release._ReleaseAsset( - "v1.2.3", - "nvx-1.2.3-linux-kvm.tar.gz", - "https://api.github.invalid/assets/1", - 4, - ) - - def write_archive( - _url: str, - destination: Path, - **kwargs: object, - ) -> None: - self.assertIsInstance( - kwargs["opener"], - urllib.request.OpenerDirector, - ) - self.assertEqual( - kwargs["headers"], - release._github_headers("token", "application/octet-stream"), - ) - destination.write_bytes(b"data") - - with ( - patch.dict(os.environ, {"GH_TOKEN": "token"}), - patch.object(release, "_checkout_release_asset", return_value=asset), - patch.object(release, "download", side_effect=write_archive) as download, - patch.object(release, "_install_release_archive") as install, - ): - release.download_checkout_release("example/nvx", "linux-kvm") - - self.assertEqual(download.call_count, 1) - install.assert_called_once() - - def test_binary_package_stages_files_and_checksums(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - paths, kernel_inputs, revision = _write_release_fixture(root) - build_dir = paths["build"] - source_dir = paths["source"] - openvmm_dir = paths["openvmm"] - binary = paths["binary"] - destination = root / "staged" - stderr = io.StringIO() - - def artifact_path(name: str) -> Path: - return build_dir / name - - with ( - patch.object(BuildConstants, "REPO_ROOT", root), - patch.object(BuildConstants, "SOURCE_DIR", source_dir), - patch.object( - UbuntuBuildConstants, - "PACKAGE_LOCK", - root / "ubuntu" / "packages.lock.json", - ), - patch.object(OpenVMMBuildConstants, "DIRECTORY", openvmm_dir), - patch.object( - release, - "artifact_path", - side_effect=artifact_path, - ), - patch.object(release, "openvmm_binary_path", return_value=binary), - patch.object( - release, - "kernel_provenance_inputs", - return_value=kernel_inputs, - ), - patch.object( - release, - "openvmm_git_state", - return_value=(revision, True), - ), - patch("sys.stderr", stderr), - ): - release.package_release( - version="1.0.0", - destination=destination, - include_source=False, - force=False, - ) - - self.assertTrue((destination / "bin" / binary.name).is_file()) - for name in ReleaseBuildConstants.GUEST_ARTIFACT_NAMES: - self.assertTrue((destination / "guest" / name).is_file()) - self.assertTrue( - ( - destination / "provenance" / OpenVMMBuildConstants.PROVENANCE_NAME - ).is_file() - ) - self.assertTrue( - ( - destination / "provenance" / KernelBuildConstants.PROVENANCE_NAME - ).is_file() - ) - self.assertTrue( - ( - destination / "provenance" / InitramfsBuildConstants.PROVENANCE_NAME - ).is_file() - ) - manifest = json.loads( - (destination / "SOURCE-MANIFEST.json").read_text(encoding="utf-8") - ) - self.assertEqual(manifest["distribution"]["version"], "1.0.0") - self.assertNotIn("guest_agent", manifest) - self.assertEqual(manifest["openvmm"]["source_revision"], revision) - self.assertEqual( - manifest["openvmm"]["executable_sha256"], - common.sha256_file(destination / "bin" / binary.name), - ) - self.assertEqual( - manifest["openvmm"]["microvm_abi_version"], - 2, - ) - self.assertEqual( - manifest["openvmm"]["control_session_protocol_version"], - 1, - ) - self.assertEqual( - manifest["openvmm"]["control_contract_revision"], - "nvx-microvm-v2-control-v2", - ) - self.assertEqual( - manifest["linux"]["kernel_sha256"], - common.sha256_file(destination / "guest" / "vmlinux"), - ) - self.assertEqual( - manifest["linux"]["config_sha256"], - common.sha256_file(destination / "guest" / "vmlinux.config"), - ) - self.assertEqual( - manifest["azurelinux"]["initramfs_sha256"], - common.sha256_file( - destination / "guest" / AzureLinuxBuildConstants.INITRAMFS_NAME - ), - ) - self.assertEqual( - manifest["azurelinux"]["initramfs_package_manifest_sha256"], - common.sha256_file( - destination - / "guest" - / AzureLinuxBuildConstants.PACKAGE_MANIFEST_NAME - ), - ) - common.verify_sha256_sums(destination) - self.assertIn("binary-only package", stderr.getvalue()) - - def test_source_package_omits_azure_linux_artifacts(self): - for azurelinux_built in (True, False): - with ( - self.subTest(azurelinux_built=azurelinux_built), - tempfile.TemporaryDirectory() as temporary, - ): - root = Path(temporary) - paths, kernel_inputs, revision = _write_release_fixture(root) - build_dir = paths["build"] - if not azurelinux_built: - for name in ReleaseBuildConstants.AZURELINUX_ARTIFACT_NAMES: - (build_dir / name).unlink() - self._package_source_release_without_azure_linux( - root, paths, kernel_inputs, revision - ) - - def _package_source_release_without_azure_linux( - self, - root: Path, - paths: dict[str, Path], - kernel_inputs: dict[str, object], - revision: str, - ) -> None: - build_dir = paths["build"] - source_dir = paths["source"] - openvmm_dir = paths["openvmm"] - binary = paths["binary"] - destination = root / "staged" - stderr = io.StringIO() - linux_source_archive = ( - source_dir - / KernelBuildConstants.SOURCE_DIRECTORY_NAME - / KernelBuildConstants.SOURCE_ARCHIVE_NAME - ) - linux_source_archive.parent.mkdir(parents=True, exist_ok=True) - linux_source_archive.write_bytes(b"linux-source") - - def artifact_path(name: str) -> Path: - return build_dir / name - - def write_source_archive(output: Path, *_args: object) -> None: - output.parent.mkdir(parents=True, exist_ok=True) - output.write_bytes(output.name.encode("ascii")) - - with ( - patch.object(BuildConstants, "REPO_ROOT", root), - patch.object(BuildConstants, "SOURCE_DIR", source_dir), - patch.object( - UbuntuBuildConstants, - "PACKAGE_LOCK", - root / "ubuntu" / "packages.lock.json", - ), - patch.object(OpenVMMBuildConstants, "DIRECTORY", openvmm_dir), - patch.object( - release, - "artifact_path", - side_effect=artifact_path, - ), - patch.object(release, "openvmm_binary_path", return_value=binary), - patch.object( - release, - "kernel_provenance_inputs", - return_value=kernel_inputs, - ), - patch.object( - release, - "openvmm_git_state", - return_value=(revision, True), - ), - patch.object(release, "_validate_alpine_sources"), - patch.object(release, "_validate_ubuntu_sources"), - patch.object(release, "_validate_linux_source_archive"), - patch.object( - release, - "_project_source_archive", - side_effect=write_source_archive, - ) as project_source_archive, - patch.object( - release, - "_alpine_source_archive", - side_effect=write_source_archive, - ), - patch.object( - release, - "_ubuntu_source_archive", - side_effect=write_source_archive, - ), - patch("sys.stderr", stderr), - ): - release.package_release( - version="1.0.0", - destination=destination, - include_source=True, - force=False, - ) - - for name in ReleaseBuildConstants.AZURELINUX_ARTIFACT_NAMES: - self.assertFalse((destination / "guest" / name).exists()) - for name in ReleaseBuildConstants.GUEST_ARTIFACT_NAMES: - if name in ReleaseBuildConstants.AZURELINUX_ARTIFACT_NAMES: - continue - self.assertTrue((destination / "guest" / name).is_file()) - self.assertNotIn( - build_dir / AzureLinuxBuildConstants.PACKAGE_MANIFEST_NAME, - project_source_archive.call_args.args[2], - ) - manifest = json.loads( - (destination / "SOURCE-MANIFEST.json").read_text(encoding="utf-8") - ) - self.assertNotIn("azurelinux", manifest) - common.verify_sha256_sums(destination) - self.assertIn("omits the Azure Linux guest", stderr.getvalue()) - - def test_runtime_provenance_validation_uses_required_paths(self): - build_dir = Path("build") - binary = Path("openvmm") - - def artifact_path(name: str) -> Path: - return build_dir / name - - def require_file(path: Path, _description: str) -> Path: - return path - - with ( - patch.object(release, "artifact_path", side_effect=artifact_path), - patch.object(release, "openvmm_binary_path", return_value=binary), - patch.object( - release, - "require_file", - side_effect=require_file, - ) as require, - patch.object(release, "_validate_openvmm_provenance") as openvmm, - patch.object(release, "_validate_kernel_provenance"), - patch.object(release, "_validate_initramfs_provenance"), - ): - release.validate_runtime_artifact_provenance() - - self.assertEqual( - require.call_args_list[-3:], - [ - call( - build_dir / OpenVMMBuildConstants.PROVENANCE_NAME, - "OpenVMM build provenance", - ), - call( - build_dir / KernelBuildConstants.PROVENANCE_NAME, - "kernel build provenance", - ), - call( - build_dir / InitramfsBuildConstants.PROVENANCE_NAME, - "initramfs build provenance", - ), - ], - ) - openvmm.assert_called_once_with( - binary, build_dir / OpenVMMBuildConstants.PROVENANCE_NAME - ) - - def test_package_rejects_dirty_openvmm_provenance(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - paths, _kernel_inputs, revision = _write_release_fixture(root) - - with ( - patch.object( - release, - "openvmm_git_state", - return_value=(revision, False), - ), - self.assertRaisesRegex( - common.ScriptError, - "current clean pinned source", - ), - ): - release._validate_openvmm_provenance( - paths["binary"], - paths["build"] / OpenVMMBuildConstants.PROVENANCE_NAME, - ) - - def test_package_rejects_stale_kernel_provenance(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - paths, kernel_inputs, _revision = _write_release_fixture(root) - kernel = paths["build"] / "vmlinux" - kernel.write_bytes(b"new kernel bytes") - - with ( - patch.object( - release, - "kernel_provenance_inputs", - return_value=kernel_inputs, - ), - self.assertRaisesRegex( - common.ScriptError, - "kernel build provenance", - ), - ): - release._validate_kernel_provenance( - kernel, - paths["build"] / "vmlinux.config", - paths["build"] / KernelBuildConstants.PROVENANCE_NAME, - ) - - def test_package_rejects_stale_initramfs_provenance(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - paths, _kernel_inputs, _revision = _write_release_fixture(root) - initramfs = paths["build"] / "initramfs.cpio.gz" - initramfs.write_bytes(b"new initramfs bytes") - - with self.assertRaisesRegex( - common.ScriptError, - "initramfs build provenance", - ): - release._validate_initramfs_provenance( - initramfs, - paths["build"] / "initramfs.cpio.gz.packages.json", - paths["build"] / InitramfsBuildConstants.PROVENANCE_NAME, - ) - - def test_package_rejects_stale_initramfs_package_manifest_provenance(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - paths, _kernel_inputs, _revision = _write_release_fixture(root) - package_manifest = paths["build"] / "initramfs.cpio.gz.packages.json" - package_manifest.write_bytes(b"new package manifest bytes") - - with self.assertRaisesRegex( - common.ScriptError, - "initramfs build provenance", - ): - release._validate_initramfs_provenance( - paths["build"] / "initramfs.cpio.gz", - package_manifest, - paths["build"] / InitramfsBuildConstants.PROVENANCE_NAME, - ) - - def test_guest_release_inputs_reject_stale_ubuntu_artifacts(self): - for artifact_name in ( - "initramfs-ubuntu.cpio.gz", - "ubuntu-distro.erofs", - ): - with ( - self.subTest(artifact=artifact_name), - tempfile.TemporaryDirectory() as temporary, - ): - root = Path(temporary) - paths, _kernel_inputs, _revision = _write_release_fixture(root) - build_dir = paths["build"] - (build_dir / artifact_name).write_bytes(b"stale artifact") - with ( - patch.object( - release, - "artifact_path", - side_effect=build_dir.joinpath, - ), - self.assertRaisesRegex( - common.ScriptError, - "Ubuntu artifact manifest does not match", - ), - ): - release._guest_release_inputs(include_azurelinux=True) - - def test_guest_release_inputs_reject_stale_azurelinux_initramfs(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - paths, _kernel_inputs, _revision = _write_release_fixture(root) - build_dir = paths["build"] - (build_dir / AzureLinuxBuildConstants.INITRAMFS_NAME).write_bytes( - b"stale artifact" - ) - with ( - patch.object( - release, - "artifact_path", - side_effect=build_dir.joinpath, - ), - self.assertRaisesRegex( - common.ScriptError, - "Azure Linux initramfs manifest is invalid", - ), - ): - release._guest_release_inputs(include_azurelinux=True) - - def test_guest_release_inputs_reject_unpinned_azurelinux_image(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - paths, _kernel_inputs, _revision = _write_release_fixture(root) - build_dir = paths["build"] - manifest_path = build_dir / AzureLinuxBuildConstants.PACKAGE_MANIFEST_NAME - manifest = json.loads(manifest_path.read_text(encoding="utf-8")) - manifest["image"] = "mcr.microsoft.com/azurelinux/base/core@sha256:" + ( - "0" * 64 - ) - manifest_path.write_text(json.dumps(manifest), encoding="utf-8") - with ( - patch.object( - release, - "artifact_path", - side_effect=build_dir.joinpath, - ), - self.assertRaisesRegex( - common.ScriptError, - "Azure Linux initramfs manifest is invalid", - ), - ): - release._guest_release_inputs(include_azurelinux=True) - - def test_guest_release_inputs_reject_stale_azurelinux_build_inputs(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - paths, _kernel_inputs, _revision = _write_release_fixture(root) - build_dir = paths["build"] - manifest_path = build_dir / AzureLinuxBuildConstants.PACKAGE_MANIFEST_NAME - current_manifest = manifest_path.read_text(encoding="utf-8") - with ( - patch.object(BuildConstants, "REPO_ROOT", root), - patch.object( - UbuntuBuildConstants, - "PACKAGE_LOCK", - root / "ubuntu" / "packages.lock.json", - ), - patch.object( - release, - "artifact_path", - side_effect=build_dir.joinpath, - ), - ): - release._guest_release_inputs(include_azurelinux=True) - with self.subTest(case="missing digest"): - document = json.loads(current_manifest) - del document["input_sha256"] - manifest_path.write_text(json.dumps(document), encoding="utf-8") - with self.assertRaisesRegex( - common.ScriptError, - "Azure Linux initramfs manifest does not match the " - "current build inputs", - ): - release._guest_release_inputs(include_azurelinux=True) - manifest_path.write_text(current_manifest, encoding="utf-8") - with self.subTest(case="edited Dockerfile"): - dockerfile = root / "docker" / "Dockerfile" - dockerfile.write_text( - dockerfile.read_text(encoding="utf-8") + "# changed\n", - encoding="utf-8", - ) - with self.assertRaisesRegex( - common.ScriptError, - "Azure Linux initramfs manifest does not match the " - "current build inputs", - ): - release._guest_release_inputs(include_azurelinux=True) - - def test_guest_release_inputs_require_azurelinux_only_when_included(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - paths, _kernel_inputs, _revision = _write_release_fixture(root) - build_dir = paths["build"] - for name in ReleaseBuildConstants.AZURELINUX_ARTIFACT_NAMES: - (build_dir / name).unlink() - with patch.object( - release, - "artifact_path", - side_effect=build_dir.joinpath, - ): - guest_names, _alpine, _ubuntu, azurelinux_manifests = ( - release._guest_release_inputs(include_azurelinux=False) - ) - with self.assertRaisesRegex( - common.ScriptError, - "required guest artifact initramfs-azurelinux.cpio.gz not found", - ): - release._guest_release_inputs(include_azurelinux=True) - - self.assertEqual( - guest_names, - [ - name - for name in ReleaseBuildConstants.GUEST_ARTIFACT_NAMES - if name not in ReleaseBuildConstants.AZURELINUX_ARTIFACT_NAMES - ], - ) - self.assertEqual(azurelinux_manifests, []) - - def test_guest_release_inputs_reject_stale_ubuntu_source_inputs(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - paths, _kernel_inputs, _revision = _write_release_fixture(root) - build_dir = paths["build"] - manifest_path = build_dir / "initramfs-ubuntu.cpio.gz.packages.json" - manifest = json.loads(manifest_path.read_text(encoding="utf-8")) - manifest["input_sha256"] = "0" * 64 - manifest_path.write_text(json.dumps(manifest), encoding="utf-8") - with ( - patch.object( - release, - "artifact_path", - side_effect=build_dir.joinpath, - ), - self.assertRaisesRegex( - common.ScriptError, - "Ubuntu artifact manifest does not match", - ), - ): - release._guest_release_inputs(include_azurelinux=True) - - def test_package_rejects_tampered_source_metadata_without_replacing_output(self): - cases: tuple[tuple[str, str, object, str], ...] = ( - ("root", "format", 2, "format must be 1"), - ("linux", "version", "6.18.37", "Linux version"), - ( - "linux", - "upstream_archive_sha256", - "0" * 64, - "Linux upstream_archive_sha256", - ), - ("linux", "patches", [], "Linux patches"), - ( - "alpine", - "minirootfs_sha256", - "0" * 64, - "Alpine minirootfs_sha256", - ), - ) - for section, field, invalid_value, error in cases: - with ( - self.subTest(section=section, field=field), - tempfile.TemporaryDirectory() as temporary, - ): - root = Path(temporary) - paths, kernel_inputs, revision = _write_release_fixture(root) - fixture_build_dir = paths["build"] - manifest_path = root / "SOURCE-MANIFEST.json" - manifest = json.loads(manifest_path.read_text(encoding="utf-8")) - if section == "root": - manifest[field] = invalid_value - else: - manifest_section = cast(dict[str, object], manifest[section]) - manifest_section[field] = invalid_value - manifest_path.write_text(json.dumps(manifest), encoding="utf-8") - destination = root / "dist" / "1.0.0" - destination.mkdir(parents=True) - marker = destination / "prior.txt" - marker.write_text("prior release", encoding="utf-8") - - def artifact_path( - name: str, - build_dir: Path = fixture_build_dir, - ) -> Path: - return build_dir / name - - with ( - patch.object(BuildConstants, "REPO_ROOT", root), - patch.object(BuildConstants, "SOURCE_DIR", paths["source"]), - patch.object( - UbuntuBuildConstants, - "PACKAGE_LOCK", - root / "ubuntu" / "packages.lock.json", - ), - patch.object(OpenVMMBuildConstants, "DIRECTORY", paths["openvmm"]), - patch.object( - release, - "artifact_path", - side_effect=artifact_path, - ), - patch.object( - release, - "openvmm_binary_path", - return_value=paths["binary"], - ), - patch.object( - release, - "kernel_provenance_inputs", - return_value=kernel_inputs, - ), - patch.object( - release, - "openvmm_git_state", - return_value=(revision, True), - ), - patch("sys.stderr", io.StringIO()), - self.assertRaisesRegex( - common.ScriptError, - error, - ), - ): - release.package_release( - version="1.0.0", - destination=destination, - include_source=False, - force=True, - ) - - self.assertEqual( - marker.read_text(encoding="utf-8"), - "prior release", - ) - - def test_failed_staged_verification_preserves_existing_release(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - paths, kernel_inputs, revision = _write_release_fixture(root) - build_dir = paths["build"] - destination = root / "dist" / "1.0.0" - destination.mkdir(parents=True) - marker = destination / "prior.txt" - marker.write_text("prior release", encoding="utf-8") - - def artifact_path(name: str) -> Path: - return build_dir / name - - original_copy = release._copy_release_file - - def corrupt_packaged_binary(source: Path, target: Path) -> None: - original_copy(source, target) - if target.name == paths["binary"].name: - target.write_bytes(b"corrupt") - - with ( - patch.object(BuildConstants, "REPO_ROOT", root), - patch.object(BuildConstants, "SOURCE_DIR", paths["source"]), - patch.object( - UbuntuBuildConstants, - "PACKAGE_LOCK", - root / "ubuntu" / "packages.lock.json", - ), - patch.object(OpenVMMBuildConstants, "DIRECTORY", paths["openvmm"]), - patch.object(release, "artifact_path", side_effect=artifact_path), - patch.object( - release, - "openvmm_binary_path", - return_value=paths["binary"], - ), - patch.object( - release, - "kernel_provenance_inputs", - return_value=kernel_inputs, - ), - patch.object( - release, - "openvmm_git_state", - return_value=(revision, True), - ), - patch.object( - release, - "_copy_release_file", - side_effect=corrupt_packaged_binary, - ), - patch("sys.stderr", io.StringIO()), - self.assertRaisesRegex( - common.ScriptError, - "packaged OpenVMM executable", - ), - ): - release.package_release( - version="1.0.0", - destination=destination, - include_source=False, - force=True, - ) - - self.assertEqual(marker.read_text(encoding="utf-8"), "prior release") - self.assertEqual( - [path.name for path in destination.iterdir()], ["prior.txt"] - ) - - def test_publication_failure_restores_existing_release(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary).resolve() - destination = root / "dist" / "release" - destination.mkdir(parents=True) - (destination / "prior.txt").write_text("prior", encoding="utf-8") - staging = root / "staging" - staging.mkdir() - (staging / "new.txt").write_text("new", encoding="utf-8") - original_replace = Path.replace - - def fail_staging_replace(source: Path, target: Path) -> Path: - if source == staging: - raise OSError("injected publication failure") - return original_replace(source, target) - - with ( - patch.object(BuildConstants, "REPO_ROOT", root), - patch.object(Path, "replace", new=fail_staging_replace), - self.assertRaisesRegex( - common.ScriptError, - "failed to publish staged release", - ), - ): - release._publish_release_directory( - staging, - destination, - force=True, - ) - - self.assertEqual( - (destination / "prior.txt").read_text(encoding="utf-8"), - "prior", - ) - self.assertFalse(staging.exists()) - - def test_failed_rollback_preserves_prior_backup_and_staging(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary).resolve() - destination = root / "dist" / "release" - destination.mkdir(parents=True) - (destination / "prior.txt").write_text("prior", encoding="utf-8") - staging = root / "staging" - staging.mkdir() - (staging / "new.txt").write_text("new", encoding="utf-8") - original_replace = Path.replace - - def fail_publication_and_restore(source: Path, target: Path) -> Path: - if source == staging or source.name.startswith(".release.backup-"): - raise OSError("injected rename failure") - return original_replace(source, target) - - with ( - patch.object(BuildConstants, "REPO_ROOT", root), - patch.object(Path, "replace", new=fail_publication_and_restore), - self.assertRaisesRegex( - common.ScriptError, - "prior release remains", - ), - ): - release._publish_release_directory( - staging, - destination, - force=True, - ) - - backups = list(destination.parent.glob(".release.backup-*")) - self.assertEqual(len(backups), 1) - self.assertEqual( - (backups[0] / "prior.txt").read_text(encoding="utf-8"), - "prior", - ) - self.assertEqual( - (staging / "new.txt").read_text(encoding="utf-8"), - "new", - ) - - def test_publication_rechecks_no_force_after_staging(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - destination = root / "release" - destination.mkdir() - (destination / "prior.txt").write_text("prior", encoding="utf-8") - staging = root / "staging" - staging.mkdir() - (staging / "new.txt").write_text("new", encoding="utf-8") - - with self.assertRaisesRegex( - common.ScriptError, - "pass --force", - ): - release._publish_release_directory( - staging, - destination, - force=False, - ) - - self.assertEqual( - (destination / "prior.txt").read_text(encoding="utf-8"), - "prior", - ) - self.assertFalse(staging.exists()) - - def test_publication_rechecks_force_destination_scope(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - destination = root / "outside-dist" - destination.mkdir() - (destination / "prior.txt").write_text("prior", encoding="utf-8") - staging = root / "staging" - staging.mkdir() - (staging / "new.txt").write_text("new", encoding="utf-8") - - with ( - patch.object(BuildConstants, "REPO_ROOT", root), - self.assertRaisesRegex( - common.ScriptError, - "below dist", - ), - ): - release._publish_release_directory( - staging, - destination, - force=True, - ) - - self.assertEqual( - (destination / "prior.txt").read_text(encoding="utf-8"), - "prior", - ) - self.assertFalse(staging.exists()) - - def test_release_archive_installs_runtime_layout(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - package_root = root / "package" / "nvx-1.2.3-test" - binary_name = "openvmm.exe" if os.name == "nt" else "openvmm" - binary = package_root / "bin" / binary_name - binary.parent.mkdir(parents=True) - binary.write_bytes(b"openvmm") - for name in ReleaseBuildConstants.GUEST_ARTIFACT_NAMES: - guest = package_root / "guest" / name - guest.parent.mkdir(parents=True, exist_ok=True) - guest.write_bytes(name.encode("ascii")) - for name in ( - OpenVMMBuildConstants.PROVENANCE_NAME, - KernelBuildConstants.PROVENANCE_NAME, - ): - provenance = package_root / "provenance" / name - provenance.parent.mkdir(parents=True, exist_ok=True) - provenance.write_bytes(name.encode("ascii")) - common.write_sha256_sums(package_root) - - if os.name == "nt": - archive_path = root / "nvx-1.2.3-windows-whp.zip" - with zipfile.ZipFile(archive_path, "w") as package: - for path in package_root.rglob("*"): - package.write(path, path.relative_to(package_root.parent)) - else: - archive_path = root / "nvx-1.2.3-linux-kvm.tar.gz" - with tarfile.open(archive_path, "w:gz") as package: - package.add(package_root, arcname=package_root.name) - - build_dir = root / "runtime" / "build" - binary_destination = ( - root / "runtime" / "openvmm" / "target" / "release" / binary_name - ) - - def artifact_path(name: str) -> Path: - return build_dir / name - - with ( - patch.object( - release, - "artifact_path", - side_effect=artifact_path, - ), - patch.object( - release, - "openvmm_binary_path", - return_value=binary_destination, - ), - ): - release._install_release_archive(archive_path) - - self.assertEqual(binary_destination.read_bytes(), b"openvmm") - for name in ReleaseBuildConstants.GUEST_ARTIFACT_NAMES: - self.assertEqual( - (build_dir / name).read_bytes(), - name.encode("ascii"), - ) - for name in ( - OpenVMMBuildConstants.PROVENANCE_NAME, - KernelBuildConstants.PROVENANCE_NAME, - ): - self.assertEqual( - (build_dir / name).read_bytes(), - name.encode("ascii"), - ) - - def test_source_release_archive_removes_stale_azure_artifacts(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - package_root = root / "package" / "nvx-1.2.3-test" - binary_name = "openvmm.exe" if os.name == "nt" else "openvmm" - binary = package_root / "bin" / binary_name - binary.parent.mkdir(parents=True) - binary.write_bytes(b"openvmm") - for name in ReleaseBuildConstants.GUEST_ARTIFACT_NAMES: - if name in ReleaseBuildConstants.AZURELINUX_ARTIFACT_NAMES: - continue - guest = package_root / "guest" / name - guest.parent.mkdir(parents=True, exist_ok=True) - guest.write_bytes(name.encode("ascii")) - for name in ( - OpenVMMBuildConstants.PROVENANCE_NAME, - KernelBuildConstants.PROVENANCE_NAME, - ): - provenance = package_root / "provenance" / name - provenance.parent.mkdir(parents=True, exist_ok=True) - provenance.write_bytes(name.encode("ascii")) - (package_root / "SOURCE-MANIFEST.json").write_text( - json.dumps({"linux": {}, "alpine": {}, "ubuntu": {}}), - encoding="utf-8", - ) - common.write_sha256_sums(package_root) - - if os.name == "nt": - archive_path = root / "nvx-1.2.3-windows-whp.zip" - with zipfile.ZipFile(archive_path, "w") as package: - for path in package_root.rglob("*"): - package.write(path, path.relative_to(package_root.parent)) - else: - archive_path = root / "nvx-1.2.3-linux-kvm.tar.gz" - with tarfile.open(archive_path, "w:gz") as package: - package.add(package_root, arcname=package_root.name) - - build_dir = root / "runtime" / "build" - binary_destination = ( - root / "runtime" / "openvmm" / "target" / "release" / binary_name - ) - build_dir.mkdir(parents=True) - for name in ReleaseBuildConstants.GUEST_ARTIFACT_NAMES: - (build_dir / name).write_bytes(b"stale " + name.encode("ascii")) - - def artifact_path(name: str) -> Path: - return build_dir / name - - with ( - patch.object( - release, - "artifact_path", - side_effect=artifact_path, - ), - patch.object( - release, - "openvmm_binary_path", - return_value=binary_destination, - ), - ): - release._install_release_archive(archive_path) - - self.assertEqual(binary_destination.read_bytes(), b"openvmm") - for name in ReleaseBuildConstants.GUEST_ARTIFACT_NAMES: - destination = build_dir / name - if name in ReleaseBuildConstants.AZURELINUX_ARTIFACT_NAMES: - self.assertFalse(destination.exists()) - else: - self.assertEqual(destination.read_bytes(), name.encode("ascii")) - - -class PositiveIntTests(unittest.TestCase): - def test_accepts_positive_integer(self): - self.assertEqual(common.positive_int("1"), 1) - - def test_rejects_zero_and_negative_values(self): - for value in ("0", "-1"): - with self.subTest(value=value): - with self.assertRaisesRegex( - argparse.ArgumentTypeError, "^must be greater than zero$" - ): - common.positive_int(value) - - def test_rejects_non_integer_input(self): - with self.assertRaises(ValueError): - common.positive_int("not-an-int") - - def test_accepts_custom_error_message(self): - with self.assertRaisesRegex(argparse.ArgumentTypeError, "^must be at least 1$"): - common.positive_int("0", message="must be at least 1") - - -class PositiveFloatTests(unittest.TestCase): - def test_accepts_positive_finite_float(self): - self.assertEqual(benchmark.positive_float("1.5"), 1.5) - - def test_rejects_nonpositive_and_nonfinite_values(self): - for value in ("0", "-1", "nan", "inf"): - with self.subTest(value=value): - with self.assertRaisesRegex( - argparse.ArgumentTypeError, - "^must be finite and greater than 0$", - ): - benchmark.positive_float(value) - - -class GitOutputTests(unittest.TestCase): - def test_runs_git_from_repository_root_and_strips_output(self): - completed = MagicMock(stdout=" output \n") - with patch.object(common.subprocess, "run", return_value=completed) as run: - self.assertEqual(common.git_output("status", "--short"), "output") - run.assert_called_once_with( - ["git", "-C", str(BuildConstants.REPO_ROOT), "status", "--short"], - check=True, - capture_output=True, - text=True, - encoding="utf-8", - errors="strict", - timeout=30.0, - ) - - -class SharedCommandTests(unittest.TestCase): - def test_commands_that_cannot_start_raise_script_errors(self): - with tempfile.TemporaryDirectory() as temporary: - missing = Path(temporary) / "missing-command" - for run in (common.run_checked, common.run_capture): - with self.subTest(helper=run.__name__): - with self.assertRaisesRegex( - common.ScriptError, - f"^failed to run command {re.escape(str(missing))} --version: ", - ) as raised: - run([missing, "--version"]) - self.assertIsInstance(raised.exception.__cause__, FileNotFoundError) - - -class SharedFileTests(unittest.TestCase): - def test_checksum_manifest_detects_modified_file(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - payload = root / "nested" / "payload.txt" - payload.parent.mkdir() - payload.write_text("original", encoding="ascii") - - common.write_sha256_sums(root) - common.verify_sha256_sums(root) - payload.write_text("modified", encoding="ascii") - - with self.assertRaisesRegex(common.ScriptError, "checksum mismatch"): - common.verify_sha256_sums(root) - - def test_checksum_manifest_rejects_unlisted_tampered_provenance(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - payload = root / "bin" / "openvmm" - provenance = root / "provenance" / "openvmm.provenance.json" - payload.parent.mkdir() - provenance.parent.mkdir() - payload.write_bytes(b"openvmm") - provenance.write_bytes(b"original provenance") - common.write_sha256_sums(root) - checksum_file = root / "SHA256SUMS" - checksum_file.write_text( - "\n".join( - line - for line in checksum_file.read_text(encoding="ascii").splitlines() - if not line.endswith("provenance/openvmm.provenance.json") - ) - + "\n", - encoding="ascii", - ) - provenance.write_bytes(b"tampered provenance") - - with self.assertRaisesRegex( - common.ScriptError, - "unlisted file.*openvmm.provenance.json", - ): - common.verify_sha256_sums(root) - - def test_checksum_manifest_rejects_missing_and_duplicate_paths(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - payload = root / "payload" - payload.write_bytes(b"payload") - common.write_sha256_sums(root) - checksum_file = root / "SHA256SUMS" - line = checksum_file.read_text(encoding="ascii") - - payload.unlink() - with self.assertRaisesRegex(common.ScriptError, "invalid checksum path"): - common.verify_sha256_sums(root) - - payload.write_bytes(b"payload") - checksum_file.write_text(line + line, encoding="ascii") - with self.assertRaisesRegex(common.ScriptError, "duplicate checksum path"): - common.verify_sha256_sums(root) - - def test_checksum_manifest_rejects_non_ascii_text(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - (root / "SHA256SUMS").write_bytes(b"\xff") - - with self.assertRaisesRegex(common.ScriptError, "only ASCII text"): - common.verify_sha256_sums(root) - - def test_checksum_manifest_rejects_unsafe_paths_and_symlinks(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - payload = root / "payload" - payload.write_bytes(b"payload") - common.write_sha256_sums(root) - checksum_file = root / "SHA256SUMS" - checksum = hashlib.sha256(b"payload").hexdigest() - checksum_file.write_text( - f"{checksum} ../payload\n", - encoding="ascii", - ) - with self.assertRaisesRegex(common.ScriptError, "malformed checksum line"): - common.verify_sha256_sums(root) - - common.write_sha256_sums(root) - link = root / "payload-link" - try: - link.symlink_to(payload) - except OSError as error: - self.skipTest(f"symlinks are unavailable: {error}") - with self.assertRaisesRegex(common.ScriptError, "symlink is not allowed"): - common.verify_sha256_sums(root) - - def test_checksum_writer_rejects_trees_the_verifier_rejects(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - payload = root / "payload" - payload.write_bytes(b"payload") - nested = root / "nested" / "SHA256SUMS" - nested.parent.mkdir() - nested.write_bytes(b"nested") - common.write_sha256_sums(root) - self.assertIn( - "nested/SHA256SUMS", - dict(common.verify_sha256_sums(root).files), - ) - checksums = (root / "SHA256SUMS").read_bytes() - - link = root / "payload-link" - try: - link.symlink_to(payload) - except OSError as error: - self.skipTest(f"symlinks are unavailable: {error}") - with self.assertRaisesRegex( - common.ScriptError, - "symlink is not allowed in checksummed tree: payload-link", - ): - common.write_sha256_sums(root) - self.assertEqual((root / "SHA256SUMS").read_bytes(), checksums) - - link.unlink() - (root / "SHA256SUMS").unlink() - (root / "SHA256SUMS").symlink_to(payload) - with self.assertRaisesRegex( - common.ScriptError, - "symlink is not allowed in checksummed tree: SHA256SUMS", - ): - common.write_sha256_sums(root) - self.assertEqual(payload.read_bytes(), b"payload") - - def test_checksum_writer_rejects_special_files(self): - if sys.platform == "win32": - self.skipTest("FIFOs are unavailable on Windows") - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - os.mkfifo(root / "pipe") - - with self.assertRaisesRegex( - common.ScriptError, - "special file is not allowed in checksummed tree: pipe", - ): - common.write_sha256_sums(root) - - self.assertFalse((root / "SHA256SUMS").exists()) - - def test_release_archives_are_reproducible_with_fixed_layout_and_modes(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - source = root / "nvx-1.0.0-test" - executable = source / "bin" / "openvmm" - data = source / "provenance" / "openvmm.provenance.json" - executable.parent.mkdir(parents=True) - data.parent.mkdir() - executable.write_bytes(b"openvmm") - data.write_bytes(b"provenance") - executable.chmod(0o755) - data.chmod(0o644) - common.write_sha256_sums(source) - - for suffix in (".tar.gz", ".zip"): - with self.subTest(suffix=suffix): - first = root / f"first{suffix}" - second = root / f"second{suffix}" - release.create_release_archive(source, first) - os.utime(executable, (1000, 1000)) - release.create_release_archive(source, second) - self.assertEqual(first.read_bytes(), second.read_bytes()) - - if suffix == ".tar.gz": - with tarfile.open(first, "r:gz") as package: - member_list = package.getmembers() - members = {member.name: member for member in member_list} - self.assertEqual( - [member.name for member in member_list], - sorted(member.name for member in member_list), - ) - self.assertEqual( - {member.name for member in member_list if member.isfile()}, - { - f"{source.name}/SHA256SUMS", - f"{source.name}/bin/openvmm", - (f"{source.name}/provenance/openvmm.provenance.json"), - }, - ) - self.assertEqual( - members[f"{source.name}/bin/openvmm"].mode, - 0o755, - ) - self.assertEqual( - members[ - f"{source.name}/provenance/openvmm.provenance.json" - ].mode, - 0o644, - ) - self.assertTrue( - all(member.mtime == 0 for member in members.values()) - ) - else: - with zipfile.ZipFile(first) as package: - member_list = package.infolist() - members = { - member.filename: member for member in member_list - } - self.assertEqual( - [member.filename for member in member_list], - sorted(member.filename for member in member_list), - ) - self.assertEqual( - { - member.filename - for member in member_list - if not member.is_dir() - }, - { - f"{source.name}/SHA256SUMS", - f"{source.name}/bin/openvmm", - (f"{source.name}/provenance/openvmm.provenance.json"), - }, - ) - executable_mode = ( - members[f"{source.name}/bin/openvmm"].external_attr >> 16 - ) & 0o777 - data_mode = ( - members[ - f"{source.name}/provenance/openvmm.provenance.json" - ].external_attr - >> 16 - ) & 0o777 - self.assertEqual(executable_mode, 0o755) - self.assertEqual(data_mode, 0o644) - self.assertTrue( - all( - member.date_time == (1980, 1, 1, 0, 0, 0) - for member in members.values() - ) - ) - - def test_release_archive_rejects_internally_inconsistent_staging(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - source = root / "nvx-1.0.0-test" - payload = source / "bin" / "openvmm" - payload.parent.mkdir(parents=True) - payload.write_bytes(b"original") - common.write_sha256_sums(source) - destination = root / "release.tar.gz" - destination.write_bytes(b"prior archive") - create_archive = archive.create_reproducible_release_archive - - def archive_mutated_source(source: Path, output: Path) -> None: - snapshot_payload = source / "bin" / "openvmm" - snapshot_payload.write_bytes(b"replacement") - create_archive(source, output) - snapshot_payload.write_bytes(b"original") - - with ( - patch.object( - release, - "create_reproducible_release_archive", - side_effect=archive_mutated_source, - ), - self.assertRaisesRegex(common.ScriptError, "checksum mismatch"), - ): - release.create_release_archive(source, destination) - - self.assertEqual(destination.read_bytes(), b"prior archive") - self.assertEqual(payload.read_bytes(), b"original") - self.assertEqual( - list(root.glob(".staging-*-release.tar.gz")), - [], - ) - - def test_release_snapshot_rejects_coherent_source_replacement(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - source = root / "nvx-1.0.0-test" - payload = source / "bin" / "openvmm" - payload.parent.mkdir(parents=True) - payload.write_bytes(b"original") - common.write_sha256_sums(source) - checksum_file = source / "SHA256SUMS" - original_checksum = checksum_file.read_bytes() - replacement = b"coherent replacement" - replacement_checksum = ( - f"{hashlib.sha256(replacement).hexdigest()} bin/openvmm\n" - ).encode("ascii") - destination = root / "release.tar.gz" - destination.write_bytes(b"prior archive") - copy_pinned_file = release._copy_pinned_regular_file - - def replace_between_copies( - source_root: Path, - snapshot_root: Path, - relative_name: str, - expected_sha256: str, - ) -> None: - copy_pinned_file( - source_root, - snapshot_root, - relative_name, - expected_sha256, - ) - if relative_name == "SHA256SUMS": - payload.write_bytes(replacement) - checksum_file.write_bytes(replacement_checksum) - - try: - with ( - patch.object( - release, - "_copy_pinned_regular_file", - side_effect=replace_between_copies, - ), - self.assertRaisesRegex( - common.ScriptError, - "release snapshot source changed", - ), - ): - release.create_release_archive(source, destination) - finally: - if payload.exists(): - payload.write_bytes(b"original") - if checksum_file.exists(): - checksum_file.write_bytes(original_checksum) - - self.assertEqual(destination.read_bytes(), b"prior archive") - self.assertEqual(payload.read_bytes(), b"original") - self.assertEqual(checksum_file.read_bytes(), original_checksum) - self.assertEqual(list(root.glob(".snapshot-*")), []) - self.assertEqual(list(root.glob(".staging-*-release.tar.gz")), []) - - def test_source_archives_are_reproducible(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - source = root / "source" - source.mkdir() - (source / "payload.txt").write_text("payload", encoding="ascii") - cache = source / "__pycache__" - cache.mkdir() - (cache / "ignored.pyc").write_bytes(b"cache") - first = root / "first.tar.gz" - second = root / "second.tar.gz" - - archive.create_reproducible_tar_gz(first, [(source, "bundle")]) - os.utime(source / "payload.txt", (1000, 1000)) - archive.create_reproducible_tar_gz(second, [(source, "bundle")]) - - self.assertEqual(first.read_bytes(), second.read_bytes()) - with tarfile.open(first, "r:gz") as package: - members = package.getmembers() - self.assertNotIn("bundle/__pycache__", {member.name for member in members}) - self.assertTrue(all(member.mtime == 0 for member in members)) - self.assertTrue(all(member.uid == member.gid == 0 for member in members)) - - -class DownloadTests(unittest.TestCase): - def test_rejects_nonpositive_attempts(self): - with tempfile.TemporaryDirectory() as temporary: - destination = Path(temporary) / "archive.tar.xz" - with ( - patch("nvx_tools.common.urllib.request.urlopen") as urlopen, - self.assertRaisesRegex( - common.ScriptError, "download attempts must be positive" - ), - ): - common.download( - "https://example.invalid/archive.tar.xz", - destination, - attempts=0, - ) - - urlopen.assert_not_called() - self.assertFalse(destination.exists()) - - def test_retries_checksum_mismatch(self): - payload = b"verified archive" - expected_sha256 = hashlib.sha256(payload).hexdigest() - responses = [io.BytesIO(b"corrupt archive"), io.BytesIO(payload)] - - with tempfile.TemporaryDirectory() as temporary: - destination = Path(temporary) / "archive.tar.xz" - with patch( - "nvx_tools.common.urllib.request.urlopen", side_effect=responses - ) as urlopen: - common.download( - "https://example.invalid/archive.tar.xz", - destination, - expected_sha256=expected_sha256, - ) - - self.assertEqual(destination.read_bytes(), payload) - self.assertEqual(urlopen.call_count, 2) - self.assertFalse(destination.with_name("archive.tar.xz.part").exists()) - - def test_does_not_publish_checksum_mismatch(self): - payload = b"verified archive" - expected_sha256 = hashlib.sha256(payload).hexdigest() - responses = [io.BytesIO(b"corrupt archive") for _ in range(3)] - - with tempfile.TemporaryDirectory() as temporary: - destination = Path(temporary) / "archive.tar.xz" - with ( - patch( - "nvx_tools.common.urllib.request.urlopen", - side_effect=responses, - ), - self.assertRaisesRegex(common.ScriptError, "SHA-256 is"), - ): - common.download( - "https://example.invalid/archive.tar.xz", - destination, - expected_sha256=expected_sha256, - ) - - self.assertFalse(destination.exists()) - self.assertFalse(destination.with_name("archive.tar.xz.part").exists()) - - def test_replaces_mismatched_cached_archive(self): - payload = b"verified archive" - expected_sha256 = hashlib.sha256(payload).hexdigest() - - with tempfile.TemporaryDirectory() as temporary: - destination = Path(temporary) / "archive.tar.xz" - destination.write_bytes(b"corrupt archive") - - def write_verified( - _url: str, - output: Path, - *, - expected_sha256: str, - ) -> None: - self.assertFalse(output.exists()) - self.assertEqual( - expected_sha256, - hashlib.sha256(payload).hexdigest(), - ) - output.write_bytes(payload) - - with patch.object(common, "download", side_effect=write_verified): - common.download_verified( - "https://example.invalid/archive.tar.xz", - destination, - expected_sha256, - ) - - self.assertEqual(destination.read_bytes(), payload) - - -if __name__ == "__main__": - unittest.main() From 8929a27d4730ac096db9506123e0acf746bb7536 Mon Sep 17 00:00:00 2001 From: Akhil Gupta Date: Sun, 11 Oct 2026 16:11:41 -0700 Subject: [PATCH 4/5] Rename nvx497-release.py to release.py --- scripts/{nvx497-release.py => nvx_tools/release.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename scripts/{nvx497-release.py => nvx_tools/release.py} (100%) diff --git a/scripts/nvx497-release.py b/scripts/nvx_tools/release.py similarity index 100% rename from scripts/nvx497-release.py rename to scripts/nvx_tools/release.py From 73b0e978953a5e95eea4c290356d1ed2b6b27540 Mon Sep 17 00:00:00 2001 From: Akhil Gupta Date: Sun, 11 Oct 2026 16:12:13 -0700 Subject: [PATCH 5/5] Rename nvx497-test.py to test_nvx_tools.py --- scripts/{nvx497-test.py => test_nvx_tools.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename scripts/{nvx497-test.py => test_nvx_tools.py} (100%) diff --git a/scripts/nvx497-test.py b/scripts/test_nvx_tools.py similarity index 100% rename from scripts/nvx497-test.py rename to scripts/test_nvx_tools.py