From d0ef48c1f39cc16252737b37b03c7e71fb6fadc4 Mon Sep 17 00:00:00 2001 From: JosepSampe Date: Fri, 19 Jun 2026 14:54:11 +0200 Subject: [PATCH 1/4] Fix project creation --- .../backends/code_engine/code_engine.py | 99 ++++++++++++++++--- .../serverless/backends/code_engine/config.py | 2 + 2 files changed, 88 insertions(+), 13 deletions(-) diff --git a/lithops/serverless/backends/code_engine/code_engine.py b/lithops/serverless/backends/code_engine/code_engine.py index cf99c4a2..2fbae6a4 100644 --- a/lithops/serverless/backends/code_engine/code_engine.py +++ b/lithops/serverless/backends/code_engine/code_engine.py @@ -114,7 +114,48 @@ def _create_code_engine_client(self): self.ce_client = CodeEngineV2(authenticator=authenticator) self.ce_client.set_service_url(config.BASE_URL_V2.format(self.region)) - def _get_or_create_namespace(self, create=True): + def _wait_for_project_active(self, project_id=None): + """ + Waits until a Code Engine project becomes active + """ + project_id = project_id or self.project_id + if not project_id: + return + + self._create_code_engine_client() + logger.debug(f"Waiting for Code Engine project {self.project_name} to become active") + deadline = time.time() + config.PROJECT_READY_TIMEOUT + while time.time() < deadline: + try: + project = self.ce_client.get_project(id=project_id).get_result() + except ApiException as e: + not_ready = ( + e.status_code in (403, 404, 503) + or 'not yet active' in (e.message or '').lower() + ) + if not_ready: + logger.debug( + f"Code Engine project {self.project_name} status: creating" + ) + time.sleep(config.PROJECT_POLL_INTERVAL) + continue + raise e + + status = project.get('status') + if status == 'active': + logger.debug(f"Code Engine project {self.project_name} is active") + return + if status == 'creation_failed': + raise Exception(f"Code Engine project {self.project_name} creation failed") + + logger.debug(f"Code Engine project {self.project_name} status: {status}") + time.sleep(config.PROJECT_POLL_INTERVAL) + + raise Exception( + f"Timed out waiting for Code Engine project {self.project_name} to become active" + ) + + def _get_or_create_namespace(self, create=True, wait_for_active=True): """ Gets or creates the Code Engine project. Namespace is kept for runtime key compatibility. @@ -126,8 +167,26 @@ def _get_or_create_namespace(self, create=True): self.namespace = ce_data.get('namespace') if self.project_id: - self._sync_project_config() - return self.namespace + self._create_code_engine_client() + try: + project = self.ce_client.get_project(id=self.project_id).get_result() + status = project.get('status') + if status == 'active': + self._sync_project_config() + return self.namespace + if status in ('creating', 'preparing') and wait_for_active: + self._wait_for_project_active() + self._sync_project_config() + return self.namespace + if status in ('creating', 'preparing'): + self._sync_project_config() + return self.namespace + except ApiException as e: + if e.status_code not in (403, 404): + raise e + logger.debug(f"Cached project {self.project_id} is unavailable, refreshing") + self.project_id = None + self.namespace = None self._create_code_engine_client() @@ -136,15 +195,28 @@ def _get_or_create_namespace(self, create=True): if project['name'] == self.project_name: logger.debug(f"Found Code Engine project: {self.project_name}") self.project_id = project['id'] + if project.get('status') != 'active' and wait_for_active: + self._wait_for_project_active() break if not self.project_id and create: logger.debug(f"Creating new Code Engine project: {self.project_name}") - response = self.ce_client.create_project( - name=self.project_name, - resource_group_id=self.config['resource_group_id'] - ).get_result() + try: + response = self.ce_client.create_project( + name=self.project_name, + resource_group_id=self.config['resource_group_id'] + ).get_result() + except ApiException as e: + if e.status_code == 409 and 'soft-deleted' in (e.message or '').lower(): + raise Exception( + f"The Code Engine project '{self.project_name}' is soft-deleted " + f"and kept for up to 7 days before it is permanently removed. " + f"To delete it completely and reuse the name now, run:\n" + f"ibmcloud ce project delete --name {self.project_name} --hard -f" + ) from e + raise e self.project_id = response['id'] + self._wait_for_project_active() if not self.project_id: return None @@ -667,7 +739,7 @@ def delete_runtime(self, runtime_name, memory, version=__version__, jobdef_name= Deletes a runtime. We need to delete the job definition. """ - if not self._get_or_create_namespace(create=False): + if not self._get_or_create_namespace(create=False, wait_for_active=False): logger.info(f"Project {self.project_name} does not exist") return @@ -682,7 +754,7 @@ def clean(self, all=False): Deletes all runtimes from all packages """ logger.info(f'Cleaning project {self.project_name}') - if not self._get_or_create_namespace(create=False): + if not self._get_or_create_namespace(create=False, wait_for_active=False): logger.info(f"Project {self.project_name} does not exist") if os.path.exists(self.cache_file): os.remove(self.cache_file) @@ -694,17 +766,18 @@ def clean(self, all=False): self._delete_lithops_config_maps() - if all and os.path.exists(self.cache_file): + if all and self.project_id: logger.info(f"Deleting Code Engine project: {self.project_name}") self.ce_client.delete_project(id=self.project_id) - os.remove(self.cache_file) + if os.path.exists(self.cache_file): + os.remove(self.cache_file) def list_runtimes(self, docker_image_name='all'): """ List all the runtimes return: list of tuples (docker_image_name, memory, version, jobdef_name) """ - if not self._get_or_create_namespace(create=False): + if not self._get_or_create_namespace(create=False, wait_for_active=False): logger.info(f"Project {self.project_name} does not exist") return [] @@ -714,7 +787,7 @@ def clear(self, job_keys=None): """ Clean all completed jobruns in the current executor """ - if not self._get_or_create_namespace(create=False): + if not self._get_or_create_namespace(create=False, wait_for_active=False): logger.info(f"Project {self.project_name} does not exist") return diff --git a/lithops/serverless/backends/code_engine/config.py b/lithops/serverless/backends/code_engine/config.py index 42b5b9b6..b4ebce72 100644 --- a/lithops/serverless/backends/code_engine/config.py +++ b/lithops/serverless/backends/code_engine/config.py @@ -34,6 +34,8 @@ PYTHON_BIN = '/usr/local/bin/python' METADATA_JOBRUN_NAME = 'lithops-runtime-metadata' JOB_RUN_POLL_INTERVAL = 2 +PROJECT_POLL_INTERVAL = 10 +PROJECT_READY_TIMEOUT = 600 # https://cloud.ibm.com/docs/codeengine?topic=codeengine-mem-cpu-combo VALID_CPU_VALUES = [0.125, 0.25, 0.5, 1, 2, 4, 6, 8] From f8736bada2189f41189d736e25d8d138655c3193 Mon Sep 17 00:00:00 2001 From: JosepSampe Date: Fri, 19 Jun 2026 15:32:37 +0200 Subject: [PATCH 2/4] Update VPC default image to Ubuntu 24 --- lithops/standalone/backends/ibm_vpc/config.py | 11 +- .../standalone/backends/ibm_vpc/ibm_vpc.py | 99 ++++++++++--- lithops/standalone/utils.py | 15 +- runtime/ibm_vpc/README.md | 140 +++++++++++------- runtime/ibm_vpc/build_lithops_vm_image.sh | 72 ++++----- 5 files changed, 220 insertions(+), 117 deletions(-) diff --git a/lithops/standalone/backends/ibm_vpc/config.py b/lithops/standalone/backends/ibm_vpc/config.py index e8333e35..29c6881b 100644 --- a/lithops/standalone/backends/ibm_vpc/config.py +++ b/lithops/standalone/backends/ibm_vpc/config.py @@ -31,7 +31,7 @@ 'master_profile_name': 'cx2-2x4', 'worker_profile_name': 'cx2-2x4', 'boot_volume_profile': 'general-purpose', - 'ssh_username': 'root', + 'ssh_username': 'ubuntu', 'ssh_password': str(uuid.uuid4()), 'ssh_key_filename': '~/.ssh/id_rsa', 'delete_on_dismantle': True, @@ -45,6 +45,15 @@ REGIONS = ["jp-tok", "jp-osa", "au-syd", "eu-gb", "eu-de", "eu-es", "us-south", "us-east", "br-sao", "ca-tor"] +INSTANCE_START_TIMEOUT = 180 +VPC_API_VERSION = '2021-09-21' + +DEFAULT_LITHOPS_IMAGE_NAME = 'lithops-ubuntu-24-04-4-minimal-amd64-1' +DEFAULT_UBUNTU_LTS_MAJOR = 24 +DEFAULT_STOCK_UBUNTU_IMAGE_PREFIX = f'ibm-ubuntu-{DEFAULT_UBUNTU_LTS_MAJOR}' +# Ubuntu LTS major versions shown by `lithops image list` (inclusive range). +LIST_UBUNTU_LTS_RANGE = (22, DEFAULT_UBUNTU_LTS_MAJOR) + def load_config(config_data): diff --git a/lithops/standalone/backends/ibm_vpc/ibm_vpc.py b/lithops/standalone/backends/ibm_vpc/ibm_vpc.py index d137edfc..cf6252a2 100644 --- a/lithops/standalone/backends/ibm_vpc/ibm_vpc.py +++ b/lithops/standalone/backends/ibm_vpc/ibm_vpc.py @@ -41,14 +41,17 @@ prepare_standalone_clean, standalone_clean_stop_early, ) +from .config import ( + DEFAULT_LITHOPS_IMAGE_NAME, + DEFAULT_STOCK_UBUNTU_IMAGE_PREFIX, + DEFAULT_UBUNTU_LTS_MAJOR, + INSTANCE_START_TIMEOUT, + LIST_UBUNTU_LTS_RANGE, + VPC_API_VERSION, +) logger = logging.getLogger(__name__) -INSTANCE_START_TIMEOUT = 180 -VPC_API_VERSION = '2021-09-21' - -DEFAULT_LITHOPS_IMAGE_NAME = 'lithops-ubuntu-22-04-3-minimal-amd64-1' - class IBMVPCBackend: @@ -504,8 +507,8 @@ def _request_image_id(self): if 'image_id' in self.vpc_data: for image in images_def: - if image['id'] == self.vpc_data['image_id'] and \ - not image['name'].startswith('ibm-ubuntu-22'): + if image['id'] == self.vpc_data['image_id'] \ + and self._is_deploy_ubuntu_image(image): self.config['image_id'] = self.vpc_data['image_id'] break @@ -517,11 +520,15 @@ def _request_image_id(self): break if 'image_id' not in self.config: - for image in images_def: - if image['name'].startswith('ibm-ubuntu-22') \ - and "amd64" in image['name']: - self.config['image_id'] = image['id'] - break + stock_images = [ + image for image in images_def + if image['name'].startswith(DEFAULT_STOCK_UBUNTU_IMAGE_PREFIX) + and 'amd64' in image['name'] + ] + if stock_images: + stock_images.sort(key=lambda image: image['name'], reverse=True) + self.config['image_id'] = stock_images[0]['id'] + logger.debug(f"Using stock VM image: {stock_images[0]['name']}") def _create_master_instance(self): """ @@ -728,9 +735,56 @@ def list_images(): time.sleep(2) logger.debug(f"VM Image '{image_name}' successfully deleted") + @staticmethod + def _ubuntu_lts_major_version(image_name, display_name=None): + for text in (display_name or '', image_name): + match = re.search(r'ubuntu[- ](\d{2})', text.lower()) + if match: + return int(match.group(1)) + match = re.search(r'(\d{2})\.\d{2}', text) + if match: + return int(match.group(1)) + return None + + @classmethod + def _is_ubuntu_amd64_image(cls, image): + return ( + image['operating_system']['family'] == 'Ubuntu Linux' + and 'amd64' in image['name'] + ) + + @classmethod + def _is_supported_ubuntu_image(cls, image, lts_range=None): + if not cls._is_ubuntu_amd64_image(image): + return False + if 'lithops' in image['name'].lower(): + return True + major = cls._ubuntu_lts_major_version( + image['name'], + image['operating_system'].get('display_name'), + ) + if major is None: + return False + if lts_range is None: + return major == DEFAULT_UBUNTU_LTS_MAJOR + min_major, max_major = lts_range + return min_major <= major <= max_major + + @classmethod + def _is_deploy_ubuntu_image(cls, image): + """ + Images valid for provisioning master/worker VSIs (Ubuntu 24 stock or Lithops custom). + """ + if not cls._is_ubuntu_amd64_image(image): + return False + if 'lithops' in image['name'].lower(): + return cls._is_supported_ubuntu_image(image) + return image['name'].startswith(DEFAULT_STOCK_UBUNTU_IMAGE_PREFIX) + def list_images(self): """ - List VM Images + List Ubuntu LTS images (22.04–24.04) and Lithops custom images. + Returns tuples of (name, image_id, creation_date). """ images_def = self.vpc_cli.list_images().result['images'] images_user = self.vpc_cli.list_images(resource_group_id=self.config['resource_group_id']).result['images'] @@ -739,14 +793,14 @@ def list_images(self): result = set() for img in images_def: - if img['operating_system']['family'] == 'Ubuntu Linux': - opsys = img['operating_system']['display_name'] - image_name = img['name'] - image_id = img['id'] - created_at = datetime.strptime(img['created_at'], "%Y-%m-%dT%H:%M:%SZ") - created_at = created_at.strftime("%Y-%m-%d %H:%M:%S") - if '22' in opsys: - result.add((image_name, image_id, created_at)) + if not self._is_supported_ubuntu_image(img, lts_range=LIST_UBUNTU_LTS_RANGE): + continue + + image_name = img['name'] + image_id = img['id'] + created_at = datetime.strptime(img['created_at'], "%Y-%m-%dT%H:%M:%SZ") + created_at = created_at.strftime("%Y-%m-%d %H:%M:%S") + result.add((image_name, image_id, created_at)) return sorted(result, key=lambda x: x[2], reverse=True) @@ -1011,7 +1065,8 @@ def __init__(self, name, ibm_vpc_config, ibm_vpc_client=None, public=False): self.instance_data = None self.private_ip = None self.public_ip = None - self.home_dir = '/root' + ssh_user = self.config['ssh_username'] + self.home_dir = '/root' if ssh_user == 'root' else f'/home/{ssh_user}' self.ssh_credentials = { 'username': self.config['ssh_username'], diff --git a/lithops/standalone/utils.py b/lithops/standalone/utils.py index d324d18c..aa481fc5 100644 --- a/lithops/standalone/utils.py +++ b/lithops/standalone/utils.py @@ -299,6 +299,10 @@ def get_host_setup_script( # --ignore-installed: do not uninstall Debian python packages (avoids RECORD errors) pip3 install --ignore-installed -U pip pip3 install --ignore-installed flask gevent {lithops_pip_spec} + if echo "{lithops_pip_spec}" | grep -q ibm; then + echo "--> Upgrading pyOpenSSL/cryptography (required for ibm_cos on Ubuntu 24.04)" + pip3 install --ignore-installed --upgrade 'pyopenssl>=24.0.0' 'cryptography>=42.0.0' + fi; fi; EXTRA_PY="{extra_python_packages}" @@ -350,11 +354,18 @@ def get_master_setup_script(config, vm_data): generate_ssh_key(){{ echo ' StrictHostKeyChecking no UserKnownHostsFile=/dev/null' >> /etc/ssh/ssh_config; + mkdir -p $USER_HOME/.ssh; + chmod 700 $USER_HOME/.ssh; + chown ${{SUDO_USER}}:${{SUDO_USER}} $USER_HOME/.ssh; ssh-keygen -f $USER_HOME/.ssh/lithops_id_rsa -t rsa -N ''; - chown ${{SUDO_USER}}:${{SUDO_USER}} $USER_HOME/.ssh/lithops_id_rsa*; cp $USER_HOME/.ssh/lithops_id_rsa $USER_HOME/.ssh/id_rsa cp $USER_HOME/.ssh/lithops_id_rsa.pub $USER_HOME/.ssh/id_rsa.pub - cp $USER_HOME/.ssh/* /root/.ssh; + chown ${{SUDO_USER}}:${{SUDO_USER}} $USER_HOME/.ssh/lithops_id_rsa* $USER_HOME/.ssh/id_rsa $USER_HOME/.ssh/id_rsa.pub + chmod 600 $USER_HOME/.ssh/lithops_id_rsa $USER_HOME/.ssh/id_rsa + chmod 644 $USER_HOME/.ssh/lithops_id_rsa.pub $USER_HOME/.ssh/id_rsa.pub + cp $USER_HOME/.ssh/lithops_id_rsa /root/.ssh/lithops_id_rsa + cp $USER_HOME/.ssh/lithops_id_rsa.pub /root/.ssh/lithops_id_rsa.pub + chmod 600 /root/.ssh/lithops_id_rsa echo '127.0.0.1 lithops-master' >> /etc/hosts; cat $USER_HOME/.ssh/id_rsa.pub >> $USER_HOME/.ssh/authorized_keys; }} diff --git a/runtime/ibm_vpc/README.md b/runtime/ibm_vpc/README.md index 673f81c5..1c57516e 100644 --- a/runtime/ibm_vpc/README.md +++ b/runtime/ibm_vpc/README.md @@ -1,113 +1,141 @@ # Lithops runtime for IBM VPC -In IBM VPC, you can execute functions using a Virtual Machine (VM). These functions operate through parallel processes within the VM. When utilizing Lithops for the first time, there's no need to manually install anything on the remote VMs, as Lithops handles this process automatically. However, employing a custom VM is recommended, as utilizing a pre-built custom image significantly enhances overall execution time. To implement this approach effectively, follow these steps: +In IBM VPC, Lithops runs functions as parallel processes inside Virtual Server Instances (VSIs). On first use, Lithops can install all dependencies on each VM automatically, but that adds several minutes to every cold start. A pre-built custom image avoids that cost. -## Option 1: +The Lithops backend key is `ibm_vpc`. -For building the default VM image that contains all dependencies required by Lithops, execute: +By default Lithops provisions VSIs from the IBM stock image **`ibm-ubuntu-24-*-minimal-amd64-*`** (Ubuntu 24.04 LTS). Ubuntu 26 and other versions are not selected automatically. -``` +## Option 1: Build the default Lithops image + +Build the default VM image with all Lithops dependencies: + +```bash lithops image build -b ibm_vpc ``` -This command will create an image called "lithops-ubuntu-22-04-3-minimal-amd64-1" in the target region. -If the image already exists, and you want to update it, use the `--overwrite` or `-o` parameter: +This creates an image named **`lithops-ubuntu-24-04-4-minimal-amd64-1`** in your IBM VPC region. -``` +To rebuild when the image already exists: + +```bash lithops image build -b ibm_vpc --overwrite ``` -Note that if you want to use this default image, there is no need to provide the image ID in the configuration, since Lithops will automatically look for it. +If you use this default image name, you do not need to set `image_id` in the config; Lithops discovers it automatically. -For creating a custom VM image, you can provide an `.sh` script with all the desired commands as an input of the previous command, and you can also provide a custom name: +List available Ubuntu and Lithops images: +```bash +lithops image list -b ibm_vpc ``` + +Use the **Image ID** column as `image_id` when you use a custom image name. + +### Custom image name and extra setup + +Provide an install script and optional image name: + +```bash lithops image build -b ibm_vpc -f myscript.sh custom-lithops-runtime ``` -If you want to upload local files to the custom VM Image, you can include them using the `--include` or `-i` parameter (src:dst), for example: +Upload local files into the image with `--include` / `-i` (`src:dst`): -``` -lithops image build -b ibm_vpc -f myscript.sh -i /home/user/test.bin:/home/ubuntu/test.bin custom-lithops-runtime +```bash +lithops image build -b ibm_vpc -f myscript.sh \ + -i /home/user/test.bin:/home/ubuntu/test.bin custom-lithops-runtime ``` -In the case of using a custom name, you must provide the Image ID, printed at the end of the build command, in your lithops config, for example: +When using a custom name, set `image_id` in your Lithops config to the value printed at the end of the build: ```yaml ibm_vpc: - ... - image_id: - ... + image_id: ``` -## Option 2: +Delete a custom image: + +```bash +lithops image delete -b ibm_vpc +``` -You can create a VM image manually. For example, you can create a VM in your IBM Cloud region, access the VM, install all the dependencies in the VM itself (apt-get, pip3 install, ...), stop the VM, create a VM Image, and then put the image_id in your lithops config, for example: +## Option 2: Manual image + +Create a VSI from an IBM Ubuntu **24.04** image (`ibm-ubuntu-24-*-minimal-amd64-*`), install dependencies (apt, pip, Lithops, Redis, Docker as needed), stop the VSI, then create a custom image from the [IBM VPC images console](https://cloud.ibm.com/vpc-ext/compute/images). Set `image_id` in your config: ```yaml ibm_vpc: - ... - image_id: - ... + image_id: ``` -## Option 3 (Discontinued): +If you name the image `lithops-ubuntu-24-04-4-minimal-amd64-1`, Lithops picks it up without an explicit `image_id` entry. + +## SSH access -For building the VM image that contains all dependencies required by Lithops, execute the [build script](build_lithops_runtime.sh) located in this folder. The best is to use vanilla Ubuntu machine to run this script and this script will use a base image based on **ubuntu-20.04-server-cloudimg-amd64**. There is need to have sudo privileges to run this script. -Once you accessed the machine, download the script +IBM Ubuntu images use the **`ubuntu`** SSH user (not `root`). Lithops sets this by default in `ibm_vpc.ssh_username`. - wget https://raw.githubusercontent.com/lithops-cloud/lithops/master/runtime/ibm_vpc/build_lithops_vm_image.sh +## Option 3 (legacy): Manual qcow2 build -and make it executable with +The [build_lithops_vm_image.sh](build_lithops_vm_image.sh) script builds a qcow2 from the Ubuntu **24.04** cloud image for manual upload to IBM COS and registration as a custom VPC image. This path is deprecated in favour of `lithops image build -b ibm_vpc`, but remains available for advanced use. - chmod +x build_lithops_vm_image.sh +Run the script on a vanilla Ubuntu machine with sudo privileges. Requirements: `libguestfs-tools`, `qemu-img`, and `expect`. -### Build the Image with Docker runtime +Download the script if needed: -If you plan to run your function within a **docker runtime** in the VM, it is preferable to include the docker image into the VM image. In this way, you will avoid the initial `docker pull ` command, thus reducing the overall execution time. To do so, add the `-d` flag followed by the docker image name you plan to use, for example: +```bash +wget https://raw.githubusercontent.com/lithops-cloud/lithops/master/runtime/ibm_vpc/build_lithops_vm_image.sh +chmod +x build_lithops_vm_image.sh +``` - ``` - $ ./build_lithops_vm_image.sh -d lithopscloud/ibmcf-python-v312 lithops-ubuntu-20.04.qcow2 - ``` -**Important** +### Build the image with a Docker runtime -Lithops will include all the local Docker images together with the Lithops runtime. To avoid this and include only Lithops runtime, it's advised to delete all local Docker images or run the script in a vanilla Ubuntu 20.04 VM. To delete all local images and include only Lithops runtime you need to execute +If you plan to run functions within a **docker runtime** in the VM, bake the Docker image into the VM image to avoid `docker pull` on every cold start. Add the `-d` flag followed by the Docker image name: +```bash +./build_lithops_vm_image.sh -d lithopscloud/ibmcf-python-v312 lithops-ubuntu-24.04.qcow2 ``` - $ ./build_lithops_vm_image.sh -p prune -d lithopscloud/ibmcf-python-v312 lithops-ubuntu-20.04.qcow2 + +**Important:** Lithops will include all local Docker images together with the Lithops runtime. To include only the Lithops runtime, delete all local Docker images first or run the script on a clean Ubuntu 24.04 VM. To prune local images before baking: + +```bash +./build_lithops_vm_image.sh -p prune -d lithopscloud/ibmcf-python-v312 lithops-ubuntu-24.04.qcow2 ``` -In this example the script generates a VM image named `lithops-ubuntu-20.04.qcow2` that contains all dependencies required by Lithops. +### Build the image without a Docker runtime -### Build the Image without a Docker runtime -Alternative is to build a VM image without a Docker runtime. This approach is mainly focused to run Lithops functions within the VM in the python3 interpreter, without using a docker runtime. If you plan to use a docker runtime to run the functions within the VM, consider to follow the previous approach. The default `build_lithops_vm_image.sh` file contains contains all required dependencies for Lithops. If you need extra linux packages and python libraries, you must edit the `build_lithops_vm_image.sh` file and include all them. +To run Lithops functions with the VM `python3` interpreter (no Docker runtime), build without `-d`: - ``` - $ ./build_lithops_vm_image.sh lithops-ubuntu-20.04.qcow2 - ``` -In this example the script generates a VM image named `lithops-ubuntu-20.04.qcow2` that contains all dependencies required by Lithops. +```bash +./build_lithops_vm_image.sh lithops-ubuntu-24.04.qcow2 +``` +The default `build_lithops_vm_image.sh` installs Lithops dependencies. To add extra Linux or Python packages, edit the script before running it. ### Deploy the image -Once local image is ready you need to upload it to COS. The best would be to use the `lithops storage` CLI: +Once the local qcow2 image is ready, upload it to IBM COS and register it as a custom VPC image: -1. Upload the `lithops-ubuntu-20.04.qcow2` image to your IBM COS instance, and place it under the root of a bucket +1. Upload `lithops-ubuntu-24.04.qcow2` to your IBM COS bucket: - ``` - lithops storage put lithops-ubuntu-20.04.qcow2 your-bucket-name - ``` + ```bash + lithops storage put lithops-ubuntu-24.04.qcow2 your-bucket-name + ``` -2. Grant permissions to the IBM VPC service to allow access to your IBM Cloud Object Storage instance +2. Grant IBM VPC permission to read your COS instance: - * Get the GUID of your cloud object storage account by running the next command: - ``` - $ ibmcloud resource service-instance "cloud-object-storage-instance-name" - ``` - * Create the authorization policy + * Get the GUID of your Cloud Object Storage instance: + + ```bash + ibmcloud resource service-instance "cloud-object-storage-instance-name" ``` - $ ibmcloud iam authorization-policy-create is cloud-object-storage Reader --source-resource-type image \ - --target-service-instance-id "cos-guid" + + * Create the authorization policy: + + ```bash + ibmcloud iam authorization-policy-create is cloud-object-storage Reader \ + --source-resource-type image \ + --target-service-instance-id "cos-guid" ``` -3. [Navigate to IBM VPC dashboard, custom images](https://cloud.ibm.com/vpc-ext/compute/images) and follow instructions to create new custom image based on the `lithops-ubuntu-20.04.qcow2` +3. [Navigate to IBM VPC custom images](https://cloud.ibm.com/vpc-ext/compute/images) and create a new custom image from `lithops-ubuntu-24.04.qcow2`. diff --git a/runtime/ibm_vpc/build_lithops_vm_image.sh b/runtime/ibm_vpc/build_lithops_vm_image.sh index cffb2776..4cfe724e 100644 --- a/runtime/ibm_vpc/build_lithops_vm_image.sh +++ b/runtime/ibm_vpc/build_lithops_vm_image.sh @@ -6,24 +6,27 @@ sudo apt-get update sudo apt-get install libguestfs-tools expect -y printf "\n\n" +BASE_IMAGE="ubuntu-24.04-server-cloudimg-amd64.img" +BASE_URL="https://cloud-images.ubuntu.com/releases/noble/release/${BASE_IMAGE}" + # Download base image and show image information and partitions echo "----------------------------------------------------------" -echo "--> Downloading ubuntu-20.04-server-cloudimg-amd64.img <--" +echo "--> Downloading ${BASE_IMAGE} <--" echo "----------------------------------------------------------" -curl -O https://cloud-images.ubuntu.com/releases/focal/release/ubuntu-20.04-server-cloudimg-amd64.img -qemu-img info ubuntu-20.04-server-cloudimg-amd64.img -virt-df -h -a ubuntu-20.04-server-cloudimg-amd64.img +curl -L -O "${BASE_URL}" +qemu-img info "${BASE_IMAGE}" +virt-df -h -a "${BASE_IMAGE}" printf "\n\n" -# Resize /dev/sda1 Partition +# Resize root partition echo "-------------------------------------------------------" -echo "--> Resizing ubuntu-20.04-server-cloudimg-amd64.img <--" +echo "--> Resizing ${BASE_IMAGE} <--" echo "-------------------------------------------------------" -cp ubuntu-20.04-server-cloudimg-amd64.img ubuntu-20.04-server-cloudimg-amd64-orig.img -qemu-img resize ubuntu-20.04-server-cloudimg-amd64.img +7.5G -virt-resize --expand /dev/sda1 ubuntu-20.04-server-cloudimg-amd64-orig.img ubuntu-20.04-server-cloudimg-amd64.img -rm ubuntu-20.04-server-cloudimg-amd64-orig.img -virt-filesystems --long -h --all -a ubuntu-20.04-server-cloudimg-amd64.img +cp "${BASE_IMAGE}" "${BASE_IMAGE%.img}-orig.img" +qemu-img resize "${BASE_IMAGE}" +7.5G +virt-resize --expand /dev/sda1 "${BASE_IMAGE%.img}-orig.img" "${BASE_IMAGE}" +rm "${BASE_IMAGE%.img}-orig.img" +virt-filesystems --long -h --all -a "${BASE_IMAGE}" printf "\n\n" # Fix partitions @@ -33,11 +36,11 @@ echo "---------------------------------------" /usr/bin/expect <*" send -- "mkdir /mnt\r" expect "**" - send -- "mount /dev/sda3 /mnt\r" + send -- "mount /dev/sda1 /mnt\r" expect "**" send -- "mount --bind /dev /mnt/dev\r" expect "**" @@ -62,18 +65,18 @@ echo "--------------------------------------------" echo "--> Installing lithops dependencies <--" echo "--------------------------------------------" sleep 5 -virt-customize -a ubuntu-20.04-server-cloudimg-amd64.img --run-command 'rm /var/lib/apt/lists/* -vfR ' -virt-customize -a ubuntu-20.04-server-cloudimg-amd64.img --run-command 'apt-get clean' -virt-customize -a ubuntu-20.04-server-cloudimg-amd64.img --run-command 'apt-get update' -virt-customize -a ubuntu-20.04-server-cloudimg-amd64.img --run-command 'apt-get install apt-transport-https ca-certificates curl software-properties-common gnupg-agent -y' -virt-customize -a ubuntu-20.04-server-cloudimg-amd64.img --run-command 'curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -' -virt-customize -a ubuntu-20.04-server-cloudimg-amd64.img --run-command 'add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"' -virt-customize -a ubuntu-20.04-server-cloudimg-amd64.img --run-command 'apt-get update' -virt-customize -a ubuntu-20.04-server-cloudimg-amd64.img --run-command 'apt-get install unzip redis-server python3-pip docker-ce docker-ce-cli containerd.io -y' -virt-customize -a ubuntu-20.04-server-cloudimg-amd64.img --run-command 'pip3 install -U flask gevent lithops' -virt-customize -a ubuntu-20.04-server-cloudimg-amd64.img --run-command 'rm -rf /var/lib/apt/lists/*' -virt-customize -a ubuntu-20.04-server-cloudimg-amd64.img --run-command 'rm -rf /var/cache/apt/archives/*' -virt-customize -a ubuntu-20.04-server-cloudimg-amd64.img --run-command 'apt-cache search linux-headers-generic' +virt-customize -a "${BASE_IMAGE}" --run-command 'rm /var/lib/apt/lists/* -vfR ' +virt-customize -a "${BASE_IMAGE}" --run-command 'apt-get clean' +virt-customize -a "${BASE_IMAGE}" --run-command 'apt-get update' +virt-customize -a "${BASE_IMAGE}" --run-command 'apt-get install apt-transport-https ca-certificates curl software-properties-common gnupg-agent -y' +virt-customize -a "${BASE_IMAGE}" --run-command 'curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg' +virt-customize -a "${BASE_IMAGE}" --run-command 'echo "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" > /etc/apt/sources.list.d/docker.list' +virt-customize -a "${BASE_IMAGE}" --run-command 'apt-get update' +virt-customize -a "${BASE_IMAGE}" --run-command 'apt-get install unzip redis-server python3-pip docker-ce docker-ce-cli containerd.io -y' +virt-customize -a "${BASE_IMAGE}" --run-command 'pip3 install -U flask gevent lithops' +virt-customize -a "${BASE_IMAGE}" --run-command 'rm -rf /var/lib/apt/lists/*' +virt-customize -a "${BASE_IMAGE}" --run-command 'rm -rf /var/cache/apt/archives/*' +virt-customize -a "${BASE_IMAGE}" --run-command 'apt-cache search linux-headers-generic' printf "\n\n" @@ -92,12 +95,12 @@ include_docker(){ fi docker pull $DOCKER_IMAGE - + sudo tar -cvf docker.tar /var/lib/docker > /dev/null 2>&1 - virt-customize -a ubuntu-20.04-server-cloudimg-amd64.img --run-command 'mkdir -p /tmp' - virt-copy-in -a ubuntu-20.04-server-cloudimg-amd64.img docker.tar /tmp - virt-customize -a ubuntu-20.04-server-cloudimg-amd64.img --run-command 'tar -xvf /tmp/docker.tar -C /' - virt-customize -a ubuntu-20.04-server-cloudimg-amd64.img --run-command 'rm -R /tmp' + virt-customize -a "${BASE_IMAGE}" --run-command 'mkdir -p /tmp' + virt-copy-in -a "${BASE_IMAGE}" docker.tar /tmp + virt-customize -a "${BASE_IMAGE}" --run-command 'tar -xvf /tmp/docker.tar -C /' + virt-customize -a "${BASE_IMAGE}" --run-command 'rm -R /tmp' sudo rm docker.tar printf "\n\n" } @@ -125,15 +128,12 @@ fi # Finished echo "-------------------------------------------------------------------" -echo "--> Compressing image ubuntu-20.04-server-cloudimg-amd64.img <--" +echo "--> Compressing image ${BASE_IMAGE} <--" echo "-------------------------------------------------------------------" echo "Final VM image: $FINAL_IMAGE" echo "" -virt-sparsify ubuntu-20.04-server-cloudimg-amd64.img --compress $FINAL_IMAGE - -#rm ubuntu-20.04-server-cloudimg-amd64.img -#kvm -net nic -net user -hda ubuntu-20.04-server-cloudimg-amd64.img -m 512 +virt-sparsify "${BASE_IMAGE}" --compress "$FINAL_IMAGE" echo "----------------------------------------------" echo "--> Congratulations! VM Image Created <--" -echo "----------------------------------------------" \ No newline at end of file +echo "----------------------------------------------" From 900d57c40946b0f0396b4949eee92e6904b874b9 Mon Sep 17 00:00:00 2001 From: JosepSampe Date: Mon, 22 Jun 2026 10:27:39 +0200 Subject: [PATCH 3/4] Update changelog --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2b9251eb..f2564f0e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ - [Azure Funcions] Updated default funcions plan to Flex Consumption - [AWS EC2] Updated default Ubuntu Image to Ubuntu 24 - [Azure VMS] Updated default Ubuntu Image to Ubuntu 24 +- [IBM VPC] Updated default Ubuntu image to 24.04 (`ibm-ubuntu-24-*` stock and `lithops-ubuntu-24-04-4-minimal-amd64-1` custom), default SSH user to `ubuntu`, and `lithops image list` to show Ubuntu 22/24 images - [Aliyun FC] Updated backend to Function Compute 3.0 (FC3 API) and added custom-container deploy mode support - [Code Engine] Rewrote backend to use the IBM Code Engine SDK v2 (`CodeEngineV2`) instead of the Kubernetes API @@ -30,6 +31,8 @@ - [Azure] Fixed Azure Functions deployment on Flex Consumption and consolidated container registry login across Azure backends - [Oracle Object Storage] Fixed authentication with `~` in `key_file`, resource principal fallback, and bucket name generation - [Oracle Functions] Fixed Python 3.12 runtime build, added OCIR registry login with auto-derived `docker_user`, and default `docker_server` from region +- [Standalone] Fixed master SSH key ownership for the `ubuntu` user and upgraded pyOpenSSL/cryptography on Ubuntu 24.04 VMs when using IBM backends +- [IBM VPC] Fixed `home_dir` for non-root SSH users and image selection to prefer Lithops custom images over stock Ubuntu 26 ## [v3.6.4] From a2b70e997577bf763b3e59794b9811c3427bbdd6 Mon Sep 17 00:00:00 2001 From: JosepSampe Date: Thu, 25 Jun 2026 00:52:38 +0200 Subject: [PATCH 4/4] Update --- CHANGELOG.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f2564f0e..b42b28a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,7 +18,9 @@ - [Azure Funcions] Updated default funcions plan to Flex Consumption - [AWS EC2] Updated default Ubuntu Image to Ubuntu 24 - [Azure VMS] Updated default Ubuntu Image to Ubuntu 24 -- [IBM VPC] Updated default Ubuntu image to 24.04 (`ibm-ubuntu-24-*` stock and `lithops-ubuntu-24-04-4-minimal-amd64-1` custom), default SSH user to `ubuntu`, and `lithops image list` to show Ubuntu 22/24 images +- [IBM VPC] Updated default Ubuntu image to 24.04 +- [IBM VPC] Changed default SSH user to `ubuntu` +- [IBM VPC] Updated `lithops image list` to show Ubuntu 22 and 24 images - [Aliyun FC] Updated backend to Function Compute 3.0 (FC3 API) and added custom-container deploy mode support - [Code Engine] Rewrote backend to use the IBM Code Engine SDK v2 (`CodeEngineV2`) instead of the Kubernetes API @@ -31,8 +33,10 @@ - [Azure] Fixed Azure Functions deployment on Flex Consumption and consolidated container registry login across Azure backends - [Oracle Object Storage] Fixed authentication with `~` in `key_file`, resource principal fallback, and bucket name generation - [Oracle Functions] Fixed Python 3.12 runtime build, added OCIR registry login with auto-derived `docker_user`, and default `docker_server` from region -- [Standalone] Fixed master SSH key ownership for the `ubuntu` user and upgraded pyOpenSSL/cryptography on Ubuntu 24.04 VMs when using IBM backends -- [IBM VPC] Fixed `home_dir` for non-root SSH users and image selection to prefer Lithops custom images over stock Ubuntu 26 +- [Standalone] Fixed SSH key permissions for the `ubuntu` user on master VMs +- [Standalone] Fixed pyOpenSSL/cryptography conflict on Ubuntu 24.04 for `ibm_cos` +- [IBM VPC] Fixed `home_dir` when using non-root SSH users +- [IBM VPC] Fixed default image selection to use Ubuntu 24 stock images only ## [v3.6.4]