diff --git a/kt/commands/content_release/impl.py b/kt/commands/content_release/impl.py index 2e2cddf..4ec8e02 100644 --- a/kt/commands/content_release/impl.py +++ b/kt/commands/content_release/impl.py @@ -6,6 +6,7 @@ from kt.ktlib.config import Config from kt.ktlib.kernel_workspace import KernelWorkspace +from kt.ktlib.kernels import KernelsInfo from kt.ktlib.local import LocalCommand from kt.ktlib.mock import Mock from kt.ktlib.ssh import SshCommand @@ -302,12 +303,19 @@ def test(cls, kernel_workspace: str): # Load kernel workspace kernel_workspace_obj = KernelWorkspace.load_from_name(kernel_workspace) + # Determine package manager based on OS variant + config = Config.load() + kernels_info = KernelsInfo.from_yaml(config) + kernel_name = Vm._extract_kernel_name(kernel_workspace) + kernel_info = kernels_info.kernels.get(kernel_name) + pkg_mgr = "yum" if (kernel_info and (kernel_info.os_variant or "").startswith("centos7")) else "dnf" + # Setup and spin up the VM (reuses common code from vm command) vm_instance = Vm.setup_and_spinup(kernel_workspace_name=kernel_workspace) - # Wait for dependencies to be installed if VM was just created + # Wait for cloud-init to finish (may reboot the VM, e.g. CentOS 7) logging.info("Waiting for VM dependencies to be installed...") - SshCommand.run(domain=vm_instance.domain, command=["sudo cloud-init status --wait || true"]) + vm_instance.wait_for_cloud_init() # Install the built RPMs build_files_dir = kernel_workspace_obj.folder / "build_files" @@ -331,17 +339,22 @@ def test(cls, kernel_workspace: str): command=[ 'sudo dnf install -y "https://depot.ciq.com/public/files/depot-client/depot/depot.x86_64.rpm"' ], + ssh_key=vm_instance.ssh_key, ) logging.info("Depot client installed") # Register depot with credentials SshCommand.run( - domain=vm_instance.domain, command=[f"sudo depot register -u {depot_user} -t {depot_token}"] + domain=vm_instance.domain, + command=[f"sudo depot register -u {depot_user} -t {depot_token}"], + ssh_key=vm_instance.ssh_key, ) logging.info("Depot registered") # Enable fips-legacy-8 - SshCommand.run(domain=vm_instance.domain, command=["sudo depot enable fips-legacy-8"]) + SshCommand.run( + domain=vm_instance.domain, command=["sudo depot enable fips-legacy-8"], ssh_key=vm_instance.ssh_key + ) logging.info("fips-legacy-8 enabled via depot") except RuntimeError as e: logging.error(f"Failed to enable depot for fipslegacy-8.6: {e}") @@ -361,18 +374,22 @@ def test(cls, kernel_workspace: str): raise RuntimeError(f"No installable RPMs found in {build_files_dir}") rpm_paths = " ".join(str(rpm.absolute()) for rpm in install_rpms) - # Remove libtraceevent first to avoid file conflicts with perf package - install_cmd = ( - f"sudo dnf remove -y libtraceevent || true && sudo dnf install --skip-broken --allowerasing {rpm_paths} -y" - ) - # install_cmd = f"sudo dnf clean all && sudo dnf install --skip-broken --allowerasing {rpm_paths} -y" + if pkg_mgr == "yum": + install_cmd = f"sudo yum install -y {rpm_paths}" + else: + install_cmd = ( + f"sudo dnf remove -y libtraceevent || true && " + f"sudo dnf install --skip-broken --allowerasing {rpm_paths} -y" + ) install_log = kernel_workspace_obj.folder.absolute() / "install.log" logging.info(f"Installing {len(install_rpms)} RPM(s)") logging.info(f"RPM install output will be written to {install_log}") try: - SshCommand.run_with_output(output_file=install_log, domain=vm_instance.domain, command=[install_cmd]) + SshCommand.run_with_output( + output_file=install_log, domain=vm_instance.domain, command=[install_cmd], ssh_key=vm_instance.ssh_key + ) logging.info("RPMs installed successfully") except RuntimeError as e: logging.error(f"RPM installation failed: {e}") @@ -407,7 +424,7 @@ def test(cls, kernel_workspace: str): "fi'" ) try: - SshCommand.run(domain=vm_instance.domain, command=[grubenv_fix_cmd]) + SshCommand.run(domain=vm_instance.domain, command=[grubenv_fix_cmd], ssh_key=vm_instance.ssh_key) logging.info("grubenv symlink fixed if needed") except RuntimeError as e: logging.error(f"Failed to fix grubenv symlink: {e}") @@ -419,7 +436,7 @@ def test(cls, kernel_workspace: str): kernel_path = f"/boot/vmlinuz-{expected_version}" set_default_cmd = f"sudo grubby --set-default={kernel_path}" try: - SshCommand.run(domain=vm_instance.domain, command=[set_default_cmd]) + SshCommand.run(domain=vm_instance.domain, command=[set_default_cmd], ssh_key=vm_instance.ssh_key) logging.info("Default boot kernel set successfully") except RuntimeError as e: logging.error(f"Failed to set default boot kernel: {e}") @@ -438,11 +455,12 @@ def test(cls, kernel_workspace: str): raise RuntimeError(f"Kernel version mismatch! Expected: {expected_version}, Running: {kernel_version}") logging.info("Verified VM is running the newly installed kernel") - # Run kselftests using the installed kselftests - kselftest_log = kernel_workspace_obj.folder.absolute() / f"selftest-{kernel_version}.log" - vm_instance.kselftests_internal(kselftest_log) - - # Count passed tests - vm_instance.count_kselftest_passed(kselftest_log) + # Run kselftests using the installed kselftests (not available on CentOS 7) + if pkg_mgr == "yum": + logging.info("Skipping kselftests (not available on CentOS 7)") + else: + kselftest_log = kernel_workspace_obj.folder.absolute() / f"selftest-{kernel_version}.log" + vm_instance.kselftests_internal(kselftest_log) + vm_instance.count_kselftest_passed(kselftest_log) logging.info("Test step completed successfully") diff --git a/kt/commands/vm/impl.py b/kt/commands/vm/impl.py index cd92475..edd3e05 100644 --- a/kt/commands/vm/impl.py +++ b/kt/commands/vm/impl.py @@ -1,7 +1,6 @@ import logging from kt.ktlib.config import Config -from kt.ktlib.ssh import SshCommand from kt.ktlib.virt import VmCommand from kt.ktlib.vm import Vm @@ -43,7 +42,7 @@ def main( if test: logging.info("Waiting for cloud-init to finish...") - SshCommand.run(domain=vm_instance.domain, command=["sudo cloud-init status --wait || true"]) + vm_instance.wait_for_cloud_init() vm_instance.test(config=config) if console: diff --git a/kt/data/cloud_init_centos7.yaml b/kt/data/cloud_init_centos7.yaml index 1fbb12e..5a00a5d 100644 --- a/kt/data/cloud_init_centos7.yaml +++ b/kt/data/cloud_init_centos7.yaml @@ -15,6 +15,10 @@ ssh_pwauth: true # Ensure the system does not update on boot. package_upgrade: False +# Create mount point before mounts module runs +bootcmd: + - mkdir -p SHARED_DIR_PLACEHOLDER + mounts: - - NFS_SOURCE_PLACEHOLDER - SHARED_DIR_PLACEHOLDER diff --git a/kt/ktlib/ssh.py b/kt/ktlib/ssh.py index 300020c..04da7b5 100644 --- a/kt/ktlib/ssh.py +++ b/kt/ktlib/ssh.py @@ -3,12 +3,14 @@ class SshCommand(CommandRunner): COMMAND = "ssh" - EXTRA = "-o StrictHostKeyChecking=no" @classmethod - def _build_command(cls, domain: str, command: list[str]) -> list[str]: - return [cls.COMMAND, cls.EXTRA, domain] + command + def _build_command(cls, domain: str, command: list[str], ssh_key: str | None = None) -> list[str]: + cmd = [cls.COMMAND, "-o", "StrictHostKeyChecking=no", "-o", "BatchMode=yes"] + if ssh_key: + cmd += ["-i", ssh_key] + return cmd + [domain] + command @classmethod - def running_kernel_version(cls, domain): - return cls.run(domain=domain, command=["uname", "-r"]) + def running_kernel_version(cls, domain, ssh_key: str | None = None): + return cls.run(domain=domain, command=["uname", "-r"], ssh_key=ssh_key) diff --git a/kt/ktlib/virt.py b/kt/ktlib/virt.py index 7b83f6f..cbb8f16 100644 --- a/kt/ktlib/virt.py +++ b/kt/ktlib/virt.py @@ -1,3 +1,6 @@ +import logging +import re +import time from enum import Enum from pathlib3x import Path @@ -107,11 +110,23 @@ def domifaddr(cls, vm_name: str) -> list[str]: class VirtHelper: + IP_PATTERN = re.compile(r"\d+\.\d+\.\d+\.\d+") + IP_POLL_INTERVAL = 5 + IP_POLL_MAX_ATTEMPTS = 24 + @classmethod def ip_addr(cls, vm_name: str) -> str: - rc = VmCommand.domifaddr(vm_name=vm_name) - - return rc[-1].split("/")[0] + for attempt in range(cls.IP_POLL_MAX_ATTEMPTS): + output = VmCommand.domifaddr(vm_name=vm_name) + for token in output: + match = cls.IP_PATTERN.search(token) + if match: + return match.group() + logging.info(f"Waiting for IP address for {vm_name} (attempt {attempt + 1}/{cls.IP_POLL_MAX_ATTEMPTS})...") + time.sleep(cls.IP_POLL_INTERVAL) + raise RuntimeError( + f"VM {vm_name} did not get an IP address after {cls.IP_POLL_MAX_ATTEMPTS * cls.IP_POLL_INTERVAL}s" + ) @classmethod def is_running(cls, vm_name: str) -> bool: diff --git a/kt/ktlib/vm.py b/kt/ktlib/vm.py index ad73023..ceedc87 100644 --- a/kt/ktlib/vm.py +++ b/kt/ktlib/vm.py @@ -232,6 +232,7 @@ def _setup_cloud_init(self, config: Config, no_depot: bool = False): base_path_str = str(config.base_path.absolute()) if self._is_centos7(): nfs_source = f"{Constants.LIBVIRT_HOST_IP}:{base_path_str}" + data["bootcmd"][0] = f"mkdir -p {base_path_str}" data["mounts"][0][0] = nfs_source data["mounts"][0][1] = base_path_str data["mounts"][1][0] = base_path_str @@ -351,19 +352,19 @@ def spin_up(self, config: Config, vcpus: int = 12, memory: int = 32768, no_depot self._create_image(config=config, vcpus=vcpus, memory=memory, no_depot=no_depot) self._wait_for_running() - return VmInstance(name=self.name, kernel_workspace=self.kernel_workspace) + return VmInstance(name=self.name, kernel_workspace=self.kernel_workspace, config=config) logging.info(f"Vm {self.name} already exists") if VirtHelper.is_running(vm_name=self.name): logging.info(f"Vm {self.name} is running, nothing to do") - return VmInstance(name=self.name, kernel_workspace=self.kernel_workspace) + return VmInstance(name=self.name, kernel_workspace=self.kernel_workspace, config=config) logging.info(f"Vm {self.name} is not running, starting it") VmCommand.start(vm_name=self.name) self._wait_for_running() - return VmInstance(name=self.name, kernel_workspace=self.kernel_workspace) + return VmInstance(name=self.name, kernel_workspace=self.kernel_workspace, config=config) def destroy(self): if VirtHelper.is_running(vm_name=self.name): @@ -381,19 +382,52 @@ class VmInstance: ssh_domain: str kernel_workspace: KernelWorkspace - def __init__(self, name: str, kernel_workspace: KernelWorkspace): + def __init__(self, name: str, kernel_workspace: KernelWorkspace, config: Config): self.name = name ip_addr = VirtHelper.ip_addr(vm_name=self.name) - username = os.environ["USER"] - self.domain = f"{username}@{ip_addr}" + self.domain = f"{config.user}@{ip_addr}" self.kernel_workspace = kernel_workspace + ssh_pub = str(config.ssh_key) + self.ssh_key = ssh_pub.removesuffix(".pub") if ssh_pub.endswith(".pub") else ssh_pub + self._wait_for_ssh() + + def _wait_for_ssh(self): + for attempt in range(Constants.VM_POLL_MAX_ATTEMPTS): + try: + SshCommand.run(domain=self.domain, command=["true"], ssh_key=self.ssh_key) + logging.info(f"SSH connection to {self.domain} established") + return + except RuntimeError: + logging.info( + f"Waiting for SSH on {self.domain} (attempt {attempt + 1}/{Constants.VM_POLL_MAX_ATTEMPTS})..." + ) + time.sleep(Constants.VM_POLL_INTERVAL_SECONDS) + raise RuntimeError( + f"SSH to {self.domain} not available after " + f"{Constants.VM_POLL_MAX_ATTEMPTS * Constants.VM_POLL_INTERVAL_SECONDS}s" + ) + + def wait_for_cloud_init(self): + """ "Wait for cloud-init to finish on the VM. This method will block until cloud-init has completed its tasks.""" + try: + SshCommand.run( + domain=self.domain, + command=["sudo cloud-init status --wait || true"], + ssh_key=self.ssh_key, + ) + except RuntimeError as e: + if "closed by remote host" in str(e): + logging.info("VM rebooted during cloud-init, waiting for it to come back...") + self._wait_for_ssh() + else: + raise def reboot(self): logging.debug("Rebooting vm") command = ["sudo", "reboot"] try: - SshCommand.run(domain=self.domain, command=command) + SshCommand.run(domain=self.domain, command=command, ssh_key=self.ssh_key) except RuntimeError as e: if "closed by remote host" in str(e): pass @@ -416,7 +450,7 @@ def kselftests(self, config): output_file = self.kernel_workspace.folder.absolute() / Path(f"kselftest-{self.current_head_sha_short()}.log") ssh_cmd = f"cd {self.kernel_workspace.src_worktree.folder.absolute()} && sudo {script}" - SshCommand.run_with_output(output_file=output_file, domain=self.domain, command=[ssh_cmd]) + SshCommand.run_with_output(output_file=output_file, domain=self.domain, command=[ssh_cmd], ssh_key=self.ssh_key) def kselftests_internal(self, output_file: Path): """ @@ -434,7 +468,9 @@ def kselftests_internal(self, output_file: Path): kselftest_cmd = "sudo /usr/libexec/kselftests/run_kselftest.sh" try: - SshCommand.run_with_output(output_file=output_file, domain=self.domain, command=[kselftest_cmd]) + SshCommand.run_with_output( + output_file=output_file, domain=self.domain, command=[kselftest_cmd], ssh_key=self.ssh_key + ) logging.info("Kselftests completed successfully") except RuntimeError as e: logging.error(f"Kselftests failed: {e}") @@ -468,7 +504,7 @@ def running_kernel_version(self): Returns: str: The kernel version string (e.g., "5.14.0-284.30.1+23.1.el9_2_ciq.x86_64") """ - return SshCommand.running_kernel_version(domain=self.domain) + return SshCommand.running_kernel_version(domain=self.domain, ssh_key=self.ssh_key) def expected_kernel_version(self): """ @@ -499,7 +535,7 @@ def build_kernel(self, config): ) ssh_cmd = f"cd {self.kernel_workspace.src_worktree.folder.absolute()} && {build_script} -n" - SshCommand.run_with_output(output_file=output_file, domain=self.domain, command=[ssh_cmd]) + SshCommand.run_with_output(output_file=output_file, domain=self.domain, command=[ssh_cmd], ssh_key=self.ssh_key) def test(self, config): if self.expected_kernel_version(): diff --git a/tests/kt/ktlib/test_vm.py b/tests/kt/ktlib/test_vm.py index 990bb0e..c23bc0f 100644 --- a/tests/kt/ktlib/test_vm.py +++ b/tests/kt/ktlib/test_vm.py @@ -1,12 +1,13 @@ import os from io import StringIO -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock, call, patch +import pytest from ruamel.yaml import YAML from pathlib3x import Path from kt.ktlib.util import Constants -from kt.ktlib.vm import Vm +from kt.ktlib.vm import Vm, VmInstance _real_open = open @@ -225,3 +226,61 @@ def test_cloud_init_no_depot_skips_depot_install(): runcmd_strs = [str(c) for c in runcmd] assert not any("depot.x86_64.rpm" in s for s in runcmd_strs) assert any("kernel_install_dep.sh" in s for s in runcmd_strs) + + +def _make_vm_instance(**overrides): + inst = VmInstance.__new__(VmInstance) + inst.name = "lts-9.2" + inst.domain = "testuser@192.168.122.10" + inst.kernel_workspace = MagicMock() + inst.ssh_key = "/tmp/fake_key" + for k, v in overrides.items(): + setattr(inst, k, v) + return inst + + +@patch("kt.ktlib.vm.SshCommand.run") +def test_wait_for_cloud_init_success(mock_ssh_run): + """cloud-init finishes normally, no reboot.""" + inst = _make_vm_instance() + inst.wait_for_cloud_init() + + mock_ssh_run.assert_called_once_with( + domain="testuser@192.168.122.10", + command=["sudo cloud-init status --wait || true"], + ssh_key="/tmp/fake_key", + ) + + +@patch("kt.ktlib.vm.time.sleep") +@patch("kt.ktlib.vm.SshCommand.run") +def test_wait_for_cloud_init_reboot_recovery(mock_ssh_run, mock_sleep): + """cloud-init reboots the VM, then SSH comes back.""" + mock_ssh_run.side_effect = [ + RuntimeError("Connection to 192.168.122.10 closed by remote host."), + None, # _wait_for_ssh -> "true" succeeds + ] + inst = _make_vm_instance() + inst.wait_for_cloud_init() + + assert mock_ssh_run.call_count == 2 + assert mock_ssh_run.call_args_list[0] == call( + domain="testuser@192.168.122.10", + command=["sudo cloud-init status --wait || true"], + ssh_key="/tmp/fake_key", + ) + assert mock_ssh_run.call_args_list[1] == call( + domain="testuser@192.168.122.10", + command=["true"], + ssh_key="/tmp/fake_key", + ) + + +@patch("kt.ktlib.vm.SshCommand.run") +def test_wait_for_cloud_init_other_error_raises(mock_ssh_run): + """Non-reboot SSH errors propagate.""" + mock_ssh_run.side_effect = RuntimeError("Permission denied") + inst = _make_vm_instance() + + with pytest.raises(RuntimeError, match="Permission denied"): + inst.wait_for_cloud_init()