From baf81b1028bc3e07d85bf4b61a9ca6107c64e894 Mon Sep 17 00:00:00 2001 From: Jonathan Maple Date: Fri, 7 Aug 2026 18:06:40 -0400 Subject: [PATCH 1/4] [KT] Fix VM SSH auth to use config credentials VmInstance was using os.environ["USER"] instead of config.user and not passing the configured SSH key. Fix SshCommand to properly split -o arguments, add BatchMode=yes to prevent interactive prompts, and pass -i with the private key derived from config.ssh_key. Thread ssh_key through all SshCommand call sites in vm.py, content_release, and vm commands. --- kt/commands/content_release/impl.py | 23 +++++++++++++++++------ kt/commands/vm/impl.py | 6 +++++- kt/ktlib/ssh.py | 12 +++++++----- kt/ktlib/vm.py | 25 ++++++++++++++----------- 4 files changed, 43 insertions(+), 23 deletions(-) diff --git a/kt/commands/content_release/impl.py b/kt/commands/content_release/impl.py index 2e2cddf..e102a72 100644 --- a/kt/commands/content_release/impl.py +++ b/kt/commands/content_release/impl.py @@ -307,7 +307,11 @@ def test(cls, kernel_workspace: str): # Wait for dependencies to be installed if VM was just created logging.info("Waiting for VM dependencies to be installed...") - SshCommand.run(domain=vm_instance.domain, command=["sudo cloud-init status --wait || true"]) + SshCommand.run( + domain=vm_instance.domain, + command=["sudo cloud-init status --wait || true"], + ssh_key=vm_instance.ssh_key, + ) # Install the built RPMs build_files_dir = kernel_workspace_obj.folder / "build_files" @@ -331,17 +335,22 @@ def test(cls, kernel_workspace: str): command=[ 'sudo dnf install -y "https://depot.ciq.com/public/files/depot-client/depot/depot.x86_64.rpm"' ], + ssh_key=vm_instance.ssh_key, ) logging.info("Depot client installed") # Register depot with credentials SshCommand.run( - domain=vm_instance.domain, command=[f"sudo depot register -u {depot_user} -t {depot_token}"] + domain=vm_instance.domain, + command=[f"sudo depot register -u {depot_user} -t {depot_token}"], + ssh_key=vm_instance.ssh_key, ) logging.info("Depot registered") # Enable fips-legacy-8 - SshCommand.run(domain=vm_instance.domain, command=["sudo depot enable fips-legacy-8"]) + SshCommand.run( + domain=vm_instance.domain, command=["sudo depot enable fips-legacy-8"], ssh_key=vm_instance.ssh_key + ) logging.info("fips-legacy-8 enabled via depot") except RuntimeError as e: logging.error(f"Failed to enable depot for fipslegacy-8.6: {e}") @@ -372,7 +381,9 @@ def test(cls, kernel_workspace: str): logging.info(f"RPM install output will be written to {install_log}") try: - SshCommand.run_with_output(output_file=install_log, domain=vm_instance.domain, command=[install_cmd]) + SshCommand.run_with_output( + output_file=install_log, domain=vm_instance.domain, command=[install_cmd], ssh_key=vm_instance.ssh_key + ) logging.info("RPMs installed successfully") except RuntimeError as e: logging.error(f"RPM installation failed: {e}") @@ -407,7 +418,7 @@ def test(cls, kernel_workspace: str): "fi'" ) try: - SshCommand.run(domain=vm_instance.domain, command=[grubenv_fix_cmd]) + SshCommand.run(domain=vm_instance.domain, command=[grubenv_fix_cmd], ssh_key=vm_instance.ssh_key) logging.info("grubenv symlink fixed if needed") except RuntimeError as e: logging.error(f"Failed to fix grubenv symlink: {e}") @@ -419,7 +430,7 @@ def test(cls, kernel_workspace: str): kernel_path = f"/boot/vmlinuz-{expected_version}" set_default_cmd = f"sudo grubby --set-default={kernel_path}" try: - SshCommand.run(domain=vm_instance.domain, command=[set_default_cmd]) + SshCommand.run(domain=vm_instance.domain, command=[set_default_cmd], ssh_key=vm_instance.ssh_key) logging.info("Default boot kernel set successfully") except RuntimeError as e: logging.error(f"Failed to set default boot kernel: {e}") diff --git a/kt/commands/vm/impl.py b/kt/commands/vm/impl.py index cd92475..c398c30 100644 --- a/kt/commands/vm/impl.py +++ b/kt/commands/vm/impl.py @@ -43,7 +43,11 @@ def main( if test: logging.info("Waiting for cloud-init to finish...") - SshCommand.run(domain=vm_instance.domain, command=["sudo cloud-init status --wait || true"]) + SshCommand.run( + domain=vm_instance.domain, + command=["sudo cloud-init status --wait || true"], + ssh_key=vm_instance.ssh_key, + ) vm_instance.test(config=config) if console: diff --git a/kt/ktlib/ssh.py b/kt/ktlib/ssh.py index 300020c..04da7b5 100644 --- a/kt/ktlib/ssh.py +++ b/kt/ktlib/ssh.py @@ -3,12 +3,14 @@ class SshCommand(CommandRunner): COMMAND = "ssh" - EXTRA = "-o StrictHostKeyChecking=no" @classmethod - def _build_command(cls, domain: str, command: list[str]) -> list[str]: - return [cls.COMMAND, cls.EXTRA, domain] + command + def _build_command(cls, domain: str, command: list[str], ssh_key: str | None = None) -> list[str]: + cmd = [cls.COMMAND, "-o", "StrictHostKeyChecking=no", "-o", "BatchMode=yes"] + if ssh_key: + cmd += ["-i", ssh_key] + return cmd + [domain] + command @classmethod - def running_kernel_version(cls, domain): - return cls.run(domain=domain, command=["uname", "-r"]) + def running_kernel_version(cls, domain, ssh_key: str | None = None): + return cls.run(domain=domain, command=["uname", "-r"], ssh_key=ssh_key) diff --git a/kt/ktlib/vm.py b/kt/ktlib/vm.py index ad73023..3493785 100644 --- a/kt/ktlib/vm.py +++ b/kt/ktlib/vm.py @@ -351,19 +351,19 @@ def spin_up(self, config: Config, vcpus: int = 12, memory: int = 32768, no_depot self._create_image(config=config, vcpus=vcpus, memory=memory, no_depot=no_depot) self._wait_for_running() - return VmInstance(name=self.name, kernel_workspace=self.kernel_workspace) + return VmInstance(name=self.name, kernel_workspace=self.kernel_workspace, config=config) logging.info(f"Vm {self.name} already exists") if VirtHelper.is_running(vm_name=self.name): logging.info(f"Vm {self.name} is running, nothing to do") - return VmInstance(name=self.name, kernel_workspace=self.kernel_workspace) + return VmInstance(name=self.name, kernel_workspace=self.kernel_workspace, config=config) logging.info(f"Vm {self.name} is not running, starting it") VmCommand.start(vm_name=self.name) self._wait_for_running() - return VmInstance(name=self.name, kernel_workspace=self.kernel_workspace) + return VmInstance(name=self.name, kernel_workspace=self.kernel_workspace, config=config) def destroy(self): if VirtHelper.is_running(vm_name=self.name): @@ -381,19 +381,20 @@ class VmInstance: ssh_domain: str kernel_workspace: KernelWorkspace - def __init__(self, name: str, kernel_workspace: KernelWorkspace): + def __init__(self, name: str, kernel_workspace: KernelWorkspace, config: Config): self.name = name ip_addr = VirtHelper.ip_addr(vm_name=self.name) - username = os.environ["USER"] - self.domain = f"{username}@{ip_addr}" + self.domain = f"{config.user}@{ip_addr}" self.kernel_workspace = kernel_workspace + ssh_pub = str(config.ssh_key) + self.ssh_key = ssh_pub.removesuffix(".pub") if ssh_pub.endswith(".pub") else ssh_pub def reboot(self): logging.debug("Rebooting vm") command = ["sudo", "reboot"] try: - SshCommand.run(domain=self.domain, command=command) + SshCommand.run(domain=self.domain, command=command, ssh_key=self.ssh_key) except RuntimeError as e: if "closed by remote host" in str(e): pass @@ -416,7 +417,7 @@ def kselftests(self, config): output_file = self.kernel_workspace.folder.absolute() / Path(f"kselftest-{self.current_head_sha_short()}.log") ssh_cmd = f"cd {self.kernel_workspace.src_worktree.folder.absolute()} && sudo {script}" - SshCommand.run_with_output(output_file=output_file, domain=self.domain, command=[ssh_cmd]) + SshCommand.run_with_output(output_file=output_file, domain=self.domain, command=[ssh_cmd], ssh_key=self.ssh_key) def kselftests_internal(self, output_file: Path): """ @@ -434,7 +435,9 @@ def kselftests_internal(self, output_file: Path): kselftest_cmd = "sudo /usr/libexec/kselftests/run_kselftest.sh" try: - SshCommand.run_with_output(output_file=output_file, domain=self.domain, command=[kselftest_cmd]) + SshCommand.run_with_output( + output_file=output_file, domain=self.domain, command=[kselftest_cmd], ssh_key=self.ssh_key + ) logging.info("Kselftests completed successfully") except RuntimeError as e: logging.error(f"Kselftests failed: {e}") @@ -468,7 +471,7 @@ def running_kernel_version(self): Returns: str: The kernel version string (e.g., "5.14.0-284.30.1+23.1.el9_2_ciq.x86_64") """ - return SshCommand.running_kernel_version(domain=self.domain) + return SshCommand.running_kernel_version(domain=self.domain, ssh_key=self.ssh_key) def expected_kernel_version(self): """ @@ -499,7 +502,7 @@ def build_kernel(self, config): ) ssh_cmd = f"cd {self.kernel_workspace.src_worktree.folder.absolute()} && {build_script} -n" - SshCommand.run_with_output(output_file=output_file, domain=self.domain, command=[ssh_cmd]) + SshCommand.run_with_output(output_file=output_file, domain=self.domain, command=[ssh_cmd], ssh_key=self.ssh_key) def test(self, config): if self.expected_kernel_version(): From 543a1691b91a9ef77d4764c06ba61d3ea09b6e1a Mon Sep 17 00:00:00 2001 From: Jonathan Maple Date: Fri, 7 Aug 2026 18:08:07 -0400 Subject: [PATCH 2/4] [KT] Improve VM startup reliability with IP and SSH polling VirtHelper.ip_addr() now polls virsh domifaddr with regex-based IP validation instead of blindly parsing output that may contain separator lines before DHCP assigns an address. VmInstance constructor waits for SSH readiness after resolving the IP, preventing "No route to host" errors when the VM network stack isn't fully up yet. --- kt/ktlib/virt.py | 21 ++++++++++++++++++--- kt/ktlib/vm.py | 17 +++++++++++++++++ 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/kt/ktlib/virt.py b/kt/ktlib/virt.py index 7b83f6f..cbb8f16 100644 --- a/kt/ktlib/virt.py +++ b/kt/ktlib/virt.py @@ -1,3 +1,6 @@ +import logging +import re +import time from enum import Enum from pathlib3x import Path @@ -107,11 +110,23 @@ def domifaddr(cls, vm_name: str) -> list[str]: class VirtHelper: + IP_PATTERN = re.compile(r"\d+\.\d+\.\d+\.\d+") + IP_POLL_INTERVAL = 5 + IP_POLL_MAX_ATTEMPTS = 24 + @classmethod def ip_addr(cls, vm_name: str) -> str: - rc = VmCommand.domifaddr(vm_name=vm_name) - - return rc[-1].split("/")[0] + for attempt in range(cls.IP_POLL_MAX_ATTEMPTS): + output = VmCommand.domifaddr(vm_name=vm_name) + for token in output: + match = cls.IP_PATTERN.search(token) + if match: + return match.group() + logging.info(f"Waiting for IP address for {vm_name} (attempt {attempt + 1}/{cls.IP_POLL_MAX_ATTEMPTS})...") + time.sleep(cls.IP_POLL_INTERVAL) + raise RuntimeError( + f"VM {vm_name} did not get an IP address after {cls.IP_POLL_MAX_ATTEMPTS * cls.IP_POLL_INTERVAL}s" + ) @classmethod def is_running(cls, vm_name: str) -> bool: diff --git a/kt/ktlib/vm.py b/kt/ktlib/vm.py index 3493785..96512ce 100644 --- a/kt/ktlib/vm.py +++ b/kt/ktlib/vm.py @@ -388,6 +388,23 @@ def __init__(self, name: str, kernel_workspace: KernelWorkspace, config: Config) self.kernel_workspace = kernel_workspace ssh_pub = str(config.ssh_key) self.ssh_key = ssh_pub.removesuffix(".pub") if ssh_pub.endswith(".pub") else ssh_pub + self._wait_for_ssh() + + def _wait_for_ssh(self): + for attempt in range(Constants.VM_POLL_MAX_ATTEMPTS): + try: + SshCommand.run(domain=self.domain, command=["true"], ssh_key=self.ssh_key) + logging.info(f"SSH connection to {self.domain} established") + return + except RuntimeError: + logging.info( + f"Waiting for SSH on {self.domain} (attempt {attempt + 1}/{Constants.VM_POLL_MAX_ATTEMPTS})..." + ) + time.sleep(Constants.VM_POLL_INTERVAL_SECONDS) + raise RuntimeError( + f"SSH to {self.domain} not available after " + f"{Constants.VM_POLL_MAX_ATTEMPTS * Constants.VM_POLL_INTERVAL_SECONDS}s" + ) def reboot(self): logging.debug("Rebooting vm") From b82ecb5c864d17fef0087d41fddb522b877ef0af Mon Sep 17 00:00:00 2001 From: Jonathan Maple Date: Fri, 7 Aug 2026 18:11:56 -0400 Subject: [PATCH 3/4] [KT] CentOS 7 content-release support Add bootcmd to cloud_init_centos7.yaml so the NFS mount point exists before cloud-init's mounts module runs. Detect CentOS 7 via os_variant and switch to yum for RPM installation. Handle cloud-init triggered reboots (power_state: reboot) by catching the SSH disconnect and reconnecting. Skip kselftests on CentOS 7 where they are not available. --- kt/commands/content_release/impl.py | 52 +++++++++++++++++++---------- kt/commands/vm/impl.py | 17 +++++++--- kt/data/cloud_init_centos7.yaml | 4 +++ kt/ktlib/vm.py | 1 + 4 files changed, 52 insertions(+), 22 deletions(-) diff --git a/kt/commands/content_release/impl.py b/kt/commands/content_release/impl.py index e102a72..f506d77 100644 --- a/kt/commands/content_release/impl.py +++ b/kt/commands/content_release/impl.py @@ -6,6 +6,7 @@ from kt.ktlib.config import Config from kt.ktlib.kernel_workspace import KernelWorkspace +from kt.ktlib.kernels import KernelsInfo from kt.ktlib.local import LocalCommand from kt.ktlib.mock import Mock from kt.ktlib.ssh import SshCommand @@ -302,16 +303,30 @@ def test(cls, kernel_workspace: str): # Load kernel workspace kernel_workspace_obj = KernelWorkspace.load_from_name(kernel_workspace) + # Determine package manager based on OS variant + config = Config.load() + kernels_info = KernelsInfo.from_yaml(config) + kernel_name = Vm._extract_kernel_name(kernel_workspace) + kernel_info = kernels_info.kernels.get(kernel_name) + pkg_mgr = "yum" if (kernel_info and (kernel_info.os_variant or "").startswith("centos7")) else "dnf" + # Setup and spin up the VM (reuses common code from vm command) vm_instance = Vm.setup_and_spinup(kernel_workspace_name=kernel_workspace) - # Wait for dependencies to be installed if VM was just created + # Wait for cloud-init to finish (may reboot the VM, e.g. CentOS 7) logging.info("Waiting for VM dependencies to be installed...") - SshCommand.run( - domain=vm_instance.domain, - command=["sudo cloud-init status --wait || true"], - ssh_key=vm_instance.ssh_key, - ) + try: + SshCommand.run( + domain=vm_instance.domain, + command=["sudo cloud-init status --wait || true"], + ssh_key=vm_instance.ssh_key, + ) + except RuntimeError as e: + if "closed by remote host" in str(e): + logging.info("VM rebooted during cloud-init, waiting for it to come back...") + vm_instance._wait_for_ssh() + else: + raise # Install the built RPMs build_files_dir = kernel_workspace_obj.folder / "build_files" @@ -370,11 +385,13 @@ def test(cls, kernel_workspace: str): raise RuntimeError(f"No installable RPMs found in {build_files_dir}") rpm_paths = " ".join(str(rpm.absolute()) for rpm in install_rpms) - # Remove libtraceevent first to avoid file conflicts with perf package - install_cmd = ( - f"sudo dnf remove -y libtraceevent || true && sudo dnf install --skip-broken --allowerasing {rpm_paths} -y" - ) - # install_cmd = f"sudo dnf clean all && sudo dnf install --skip-broken --allowerasing {rpm_paths} -y" + if pkg_mgr == "yum": + install_cmd = f"sudo yum install -y {rpm_paths}" + else: + install_cmd = ( + f"sudo dnf remove -y libtraceevent || true && " + f"sudo dnf install --skip-broken --allowerasing {rpm_paths} -y" + ) install_log = kernel_workspace_obj.folder.absolute() / "install.log" logging.info(f"Installing {len(install_rpms)} RPM(s)") @@ -449,11 +466,12 @@ def test(cls, kernel_workspace: str): raise RuntimeError(f"Kernel version mismatch! Expected: {expected_version}, Running: {kernel_version}") logging.info("Verified VM is running the newly installed kernel") - # Run kselftests using the installed kselftests - kselftest_log = kernel_workspace_obj.folder.absolute() / f"selftest-{kernel_version}.log" - vm_instance.kselftests_internal(kselftest_log) - - # Count passed tests - vm_instance.count_kselftest_passed(kselftest_log) + # Run kselftests using the installed kselftests (not available on CentOS 7) + if pkg_mgr == "yum": + logging.info("Skipping kselftests (not available on CentOS 7)") + else: + kselftest_log = kernel_workspace_obj.folder.absolute() / f"selftest-{kernel_version}.log" + vm_instance.kselftests_internal(kselftest_log) + vm_instance.count_kselftest_passed(kselftest_log) logging.info("Test step completed successfully") diff --git a/kt/commands/vm/impl.py b/kt/commands/vm/impl.py index c398c30..644e54e 100644 --- a/kt/commands/vm/impl.py +++ b/kt/commands/vm/impl.py @@ -43,11 +43,18 @@ def main( if test: logging.info("Waiting for cloud-init to finish...") - SshCommand.run( - domain=vm_instance.domain, - command=["sudo cloud-init status --wait || true"], - ssh_key=vm_instance.ssh_key, - ) + try: + SshCommand.run( + domain=vm_instance.domain, + command=["sudo cloud-init status --wait || true"], + ssh_key=vm_instance.ssh_key, + ) + except RuntimeError as e: + if "closed by remote host" in str(e): + logging.info("VM rebooted during cloud-init, waiting for it to come back...") + vm_instance._wait_for_ssh() + else: + raise vm_instance.test(config=config) if console: diff --git a/kt/data/cloud_init_centos7.yaml b/kt/data/cloud_init_centos7.yaml index 1fbb12e..5a00a5d 100644 --- a/kt/data/cloud_init_centos7.yaml +++ b/kt/data/cloud_init_centos7.yaml @@ -15,6 +15,10 @@ ssh_pwauth: true # Ensure the system does not update on boot. package_upgrade: False +# Create mount point before mounts module runs +bootcmd: + - mkdir -p SHARED_DIR_PLACEHOLDER + mounts: - - NFS_SOURCE_PLACEHOLDER - SHARED_DIR_PLACEHOLDER diff --git a/kt/ktlib/vm.py b/kt/ktlib/vm.py index 96512ce..eaa56f2 100644 --- a/kt/ktlib/vm.py +++ b/kt/ktlib/vm.py @@ -232,6 +232,7 @@ def _setup_cloud_init(self, config: Config, no_depot: bool = False): base_path_str = str(config.base_path.absolute()) if self._is_centos7(): nfs_source = f"{Constants.LIBVIRT_HOST_IP}:{base_path_str}" + data["bootcmd"][0] = f"mkdir -p {base_path_str}" data["mounts"][0][0] = nfs_source data["mounts"][0][1] = base_path_str data["mounts"][1][0] = base_path_str From df9940a0dfeeeab7623978693344c6b56783df20 Mon Sep 17 00:00:00 2001 From: Jonathan Maple Date: Fri, 7 Aug 2026 18:15:36 -0400 Subject: [PATCH 4/4] [KT] Extract wait_for_cloud_init() helper on VmInstance The cloud-init wait + reboot recovery pattern was duplicated in both content_release and vm command implementations. Extract it into VmInstance.wait_for_cloud_init() and add tests covering the success, reboot-recovery, and error-propagation paths. --- kt/commands/content_release/impl.py | 13 +----- kt/commands/vm/impl.py | 14 +------ kt/ktlib/vm.py | 15 +++++++ tests/kt/ktlib/test_vm.py | 63 ++++++++++++++++++++++++++++- 4 files changed, 78 insertions(+), 27 deletions(-) diff --git a/kt/commands/content_release/impl.py b/kt/commands/content_release/impl.py index f506d77..4ec8e02 100644 --- a/kt/commands/content_release/impl.py +++ b/kt/commands/content_release/impl.py @@ -315,18 +315,7 @@ def test(cls, kernel_workspace: str): # Wait for cloud-init to finish (may reboot the VM, e.g. CentOS 7) logging.info("Waiting for VM dependencies to be installed...") - try: - SshCommand.run( - domain=vm_instance.domain, - command=["sudo cloud-init status --wait || true"], - ssh_key=vm_instance.ssh_key, - ) - except RuntimeError as e: - if "closed by remote host" in str(e): - logging.info("VM rebooted during cloud-init, waiting for it to come back...") - vm_instance._wait_for_ssh() - else: - raise + vm_instance.wait_for_cloud_init() # Install the built RPMs build_files_dir = kernel_workspace_obj.folder / "build_files" diff --git a/kt/commands/vm/impl.py b/kt/commands/vm/impl.py index 644e54e..edd3e05 100644 --- a/kt/commands/vm/impl.py +++ b/kt/commands/vm/impl.py @@ -1,7 +1,6 @@ import logging from kt.ktlib.config import Config -from kt.ktlib.ssh import SshCommand from kt.ktlib.virt import VmCommand from kt.ktlib.vm import Vm @@ -43,18 +42,7 @@ def main( if test: logging.info("Waiting for cloud-init to finish...") - try: - SshCommand.run( - domain=vm_instance.domain, - command=["sudo cloud-init status --wait || true"], - ssh_key=vm_instance.ssh_key, - ) - except RuntimeError as e: - if "closed by remote host" in str(e): - logging.info("VM rebooted during cloud-init, waiting for it to come back...") - vm_instance._wait_for_ssh() - else: - raise + vm_instance.wait_for_cloud_init() vm_instance.test(config=config) if console: diff --git a/kt/ktlib/vm.py b/kt/ktlib/vm.py index eaa56f2..ceedc87 100644 --- a/kt/ktlib/vm.py +++ b/kt/ktlib/vm.py @@ -407,6 +407,21 @@ def _wait_for_ssh(self): f"{Constants.VM_POLL_MAX_ATTEMPTS * Constants.VM_POLL_INTERVAL_SECONDS}s" ) + def wait_for_cloud_init(self): + """ "Wait for cloud-init to finish on the VM. This method will block until cloud-init has completed its tasks.""" + try: + SshCommand.run( + domain=self.domain, + command=["sudo cloud-init status --wait || true"], + ssh_key=self.ssh_key, + ) + except RuntimeError as e: + if "closed by remote host" in str(e): + logging.info("VM rebooted during cloud-init, waiting for it to come back...") + self._wait_for_ssh() + else: + raise + def reboot(self): logging.debug("Rebooting vm") diff --git a/tests/kt/ktlib/test_vm.py b/tests/kt/ktlib/test_vm.py index 990bb0e..c23bc0f 100644 --- a/tests/kt/ktlib/test_vm.py +++ b/tests/kt/ktlib/test_vm.py @@ -1,12 +1,13 @@ import os from io import StringIO -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock, call, patch +import pytest from ruamel.yaml import YAML from pathlib3x import Path from kt.ktlib.util import Constants -from kt.ktlib.vm import Vm +from kt.ktlib.vm import Vm, VmInstance _real_open = open @@ -225,3 +226,61 @@ def test_cloud_init_no_depot_skips_depot_install(): runcmd_strs = [str(c) for c in runcmd] assert not any("depot.x86_64.rpm" in s for s in runcmd_strs) assert any("kernel_install_dep.sh" in s for s in runcmd_strs) + + +def _make_vm_instance(**overrides): + inst = VmInstance.__new__(VmInstance) + inst.name = "lts-9.2" + inst.domain = "testuser@192.168.122.10" + inst.kernel_workspace = MagicMock() + inst.ssh_key = "/tmp/fake_key" + for k, v in overrides.items(): + setattr(inst, k, v) + return inst + + +@patch("kt.ktlib.vm.SshCommand.run") +def test_wait_for_cloud_init_success(mock_ssh_run): + """cloud-init finishes normally, no reboot.""" + inst = _make_vm_instance() + inst.wait_for_cloud_init() + + mock_ssh_run.assert_called_once_with( + domain="testuser@192.168.122.10", + command=["sudo cloud-init status --wait || true"], + ssh_key="/tmp/fake_key", + ) + + +@patch("kt.ktlib.vm.time.sleep") +@patch("kt.ktlib.vm.SshCommand.run") +def test_wait_for_cloud_init_reboot_recovery(mock_ssh_run, mock_sleep): + """cloud-init reboots the VM, then SSH comes back.""" + mock_ssh_run.side_effect = [ + RuntimeError("Connection to 192.168.122.10 closed by remote host."), + None, # _wait_for_ssh -> "true" succeeds + ] + inst = _make_vm_instance() + inst.wait_for_cloud_init() + + assert mock_ssh_run.call_count == 2 + assert mock_ssh_run.call_args_list[0] == call( + domain="testuser@192.168.122.10", + command=["sudo cloud-init status --wait || true"], + ssh_key="/tmp/fake_key", + ) + assert mock_ssh_run.call_args_list[1] == call( + domain="testuser@192.168.122.10", + command=["true"], + ssh_key="/tmp/fake_key", + ) + + +@patch("kt.ktlib.vm.SshCommand.run") +def test_wait_for_cloud_init_other_error_raises(mock_ssh_run): + """Non-reboot SSH errors propagate.""" + mock_ssh_run.side_effect = RuntimeError("Permission denied") + inst = _make_vm_instance() + + with pytest.raises(RuntimeError, match="Permission denied"): + inst.wait_for_cloud_init()