diff --git a/.github/workflows/regression-whisk.yml b/.github/workflows/regression-whisk.yml index c8250027..7d6ed992 100644 --- a/.github/workflows/regression-whisk.yml +++ b/.github/workflows/regression-whisk.yml @@ -132,9 +132,7 @@ jobs: - name: Configure OpenWhisk run: | WORKER_IP=$(kubectl get node kind-worker -o jsonpath='{.status.addresses[0].address}') - HOST_IP=$(hostname -I | awk '{print $1}') echo "WORKER_IP=${WORKER_IP}" >> $GITHUB_ENV - echo "HOST_IP=${HOST_IP}" >> $GITHUB_ENV git clone --depth 1 https://github.com/apache/openwhisk-deploy-kube.git /tmp/ow @@ -159,25 +157,16 @@ jobs: - name: Create OpenWhisk regression config run: | - jq \ - --arg host_ip "${HOST_IP}" \ - ' - .object.minio.address = ($host_ip + ":" + (.object.minio.mapped_port | tostring)) - | .nosql.scylladb.address = ($host_ip + ":" + (.nosql.scylladb.mapped_port | tostring)) - ' storage.json > storage-openwhisk.json - jq \ --arg language "${LANGUAGE}" \ --arg version "${LANGUAGE_VERSION}" \ --arg architecture "${ARCHITECTURE}" \ --arg registry "localhost:${REGISTRY_PORT}" \ - --slurpfile storage storage-openwhisk.json \ ' .experiments.architecture = $architecture | .experiments.runtime.language = $language | .experiments.runtime.version = $version | .deployment.openwhisk.docker_registry.registry = $registry - | .deployment.openwhisk.storage = $storage[0] ' configs/openwhisk.json > openwhisk-regression.json - name: Run OpenWhisk regression @@ -188,6 +177,7 @@ jobs: set -o pipefail uv run sebs benchmark regression test \ --config openwhisk-regression.json \ + --storage-configuration storage.json \ --deployment openwhisk \ --language ${LANGUAGE} \ --language-version ${LANGUAGE_VERSION} \ @@ -267,7 +257,6 @@ jobs: diagnostics/ regression-cache/ storage.json - storage-openwhisk.json openwhisk-regression.json regression_*.json if-no-files-found: ignore diff --git a/CHANGELOG.md b/CHANGELOG.md index 5881fa87..feca358a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,12 +6,18 @@ * Change to input of 120.uploader benchmark to conform with new Wikipedia policies (#308) +* Pull Minio images from quay.io, as the `minio/minio` repository was removed from Docker Hub. + +* Restore cached ScyllaDB configuration independently of Minio; make the OpenWhisk `shutdownStorage` option work again. + ### Improvements * Support for multiple variants of the same programming language (#286). * Support for new versions of Python and Java on AWS (#311). +* Self-hosted storage (Minio, ScyllaDB) advertises an externally reachable address to functions, detected automatically or set with `--external-address` when starting the storage, and overridable with `--storage-address` when running benchmarks. This removes the manual editing of storage addresses for OpenWhisk (#229). + ### Deprecations ### Contributors diff --git a/configs/example.json b/configs/example.json index a3f3f9b2..799de59f 100644 --- a/configs/example.json +++ b/configs/example.json @@ -89,14 +89,19 @@ }, "local": { "storage": { - "address": "", - "mapped_port": -1, - "access_key": "", - "secret_key": "", - "instance_id": "", - "input_buckets": [], - "output_buckets": [], - "type": "minio" + "object": { + "type": "minio", + "minio": { + "address": "", + "external_address": "", + "mapped_port": -1, + "access_key": "", + "secret_key": "", + "instance_id": "", + "input_buckets": [], + "output_buckets": [] + } + } } }, "openwhisk": { @@ -112,14 +117,19 @@ "password": "" }, "storage": { - "address": "", - "mapped_port": -1, - "access_key": "", - "secret_key": "", - "instance_id": "", - "input_buckets": [], - "output_buckets": [], - "type": "minio" + "object": { + "type": "minio", + "minio": { + "address": "", + "external_address": "", + "mapped_port": -1, + "access_key": "", + "secret_key": "", + "instance_id": "", + "input_buckets": [], + "output_buckets": [] + } + } } } } diff --git a/configs/openwhisk.json b/configs/openwhisk.json index e02f3553..c774a7a2 100644 --- a/configs/openwhisk.json +++ b/configs/openwhisk.json @@ -35,7 +35,8 @@ "secret_key": "", "instance_id": "", "input_buckets": [], - "output_buckets": [] + "output_buckets": [], + "external_address": "" } } } diff --git a/docs/platforms.md b/docs/platforms.md index b01937c4..efd0698a 100644 --- a/docs/platforms.md +++ b/docs/platforms.md @@ -461,5 +461,7 @@ To use that feature in SeBS, set the `experimentalManifest` flag to true. ### Storage +Start the storage with `sebs storage start` and pass the generated configuration to benchmark commands with `--storage-configuration`; see the [storage documentation](storage.md) for details, including how to override the address advertised to functions with `--storage-address`. + OpenWhisk has a `shutdownStorage` switch that controls the behavior of SeBS. -When set to true, SeBS will remove the Minio instance after finishing all work. +When set to true, SeBS will stop the Minio and ScyllaDB instances after finishing all work. diff --git a/docs/storage.md b/docs/storage.md index 73645a57..2dbcef3e 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -37,6 +37,7 @@ This file contains all the necessary information to connect to the storage servi "type": "minio", "minio": { "address": "172.17.0.2:9000", + "external_address": "10.10.1.15:9011", "mapped_port": 9011, "access_key": "XXX", "secret_key": "XXX", @@ -52,6 +53,7 @@ This file contains all the necessary information to connect to the storage servi "type": "scylladb", "scylladb": { "address": "172.17.0.3:8000", + "external_address": "10.10.1.15:9012", "mapped_port": 9012, "alternator_port": 8000, "access_key": "None", @@ -67,65 +69,51 @@ This file contains all the necessary information to connect to the storage servi } ``` -As we can see, the Minio container is running on the default Docker bridge network with address `172.17.0.2` and uses port `9000`. -The default configuration maps the container's port to the host, making the storage instance available directly without referring to the container's IP address. Minio is mapped to port 9011, and ScyllaDB is mapped to port 9012. +Each storage instance has two addresses: -## Network Configuration - -The storage instance must be accessible from the host network, and in some cases, from external networks. -For example, the storage can be deployed on a separate virtual machine or container. -Furthermore, even on a local machine, it's necessary to configure the network address, as OpenWhisk functions -are running isolated from the host network and won't be able to reach other containers running on the Docker bridge. +* `address` is used by SeBS itself, e.g., to upload benchmark inputs. On Linux, this is the container's address on the default Docker bridge network (`172.17.0.2`) and the container's port (`9000`). Functions of the local deployment run on the same bridge network and use this address as well. +* `external_address` is advertised to benchmark functions that run outside of the Docker bridge network, e.g., in a Kubernetes cluster hosting OpenWhisk. It combines the IP address of the machine with the port mapped on the host: Minio is mapped to port 9011, and ScyllaDB to port 9012. -When using Minio with cloud-hosted FaaS platforms like OpenWhisk or for local deployment, you need to ensure that the functions can reach the storage instance. -By default, the container runs on the Docker bridge network with an address (e.g., `172.17.0.2`) that is not accessible from outside the host. -Even when deploying both OpenWhisk and storage on the same system, the local bridge network is not accessible from the Kubernetes cluster. -To make it accessible, functions need to use the public IP address of the machine hosting the container instance and the mapped port. -You can typically find an externally accessible address via `ip addr`, and then replace the storage's address with the external address of the machine and the mapped port. - -For example, for an external address `10.10.1.15` (a LAN-local address on CloudLab) and mapped port `9011`, set the SeBS configuration as follows: +The external address is detected automatically as the IP address of the host's default network interface, and SeBS verifies that the storage answers on it. To use a different interface or a hostname, pass the `--external-address` flag when starting the storage: ```bash -# For a LAN-local address (e.g., on CloudLab) -jq --slurpfile file1 storage.json '.deployment.openwhisk.storage = $file1[0] | .deployment.openwhisk.storage.object.minio.address = "10.10.1.15:9011"' configs/example.json > configs/openwhisk.json +sebs storage start all configs/storage.json --output-json storage.json --external-address 10.10.1.15 ``` -You can validate the configuration of Minio with an HTTP request by using `curl`: +> [!WARNING] +> The mapped ports are bound on all interfaces of the host. On a machine with a public IP address, restrict access to these ports with a firewall or use a private address. + +## Network Configuration + +To use the deployed storage with a benchmark, pass the generated configuration file with the `--storage-configuration` flag. +The storage configuration is merged into the deployment section of the SeBS configuration, so no manual editing of JSON files is needed: ```bash -$ curl -i 10.10.1.15:9011/minio/health/live -HTTP/1.1 200 OK -Accept-Ranges: bytes -Content-Length: 0 -Content-Security-Policy: block-all-mixed-content -Server: MinIO -Strict-Transport-Security: max-age=31536000; includeSubDomains -Vary: Origin -X-Amz-Request-Id: 16F3D9B9FDFFA340 -X-Content-Type-Options: nosniff -X-Xss-Protection: 1; mode=block -Date: Mon, 30 May 2022 10:01:21 GMT +sebs benchmark invoke 210.thumbnailer test --config configs/openwhisk.json --storage-configuration storage.json ``` -If you use benchmarks relying on NoSQL storage (ScyllaDB), then you need to apply the same change to reflect the different address as well. -Here, we again assume the external IP address of the system is `10.10.1.15`, and the mapped port changes to `9012`. +Functions running in OpenWhisk or another Kubernetes-based platform cannot reach the Docker bridge network of the host, even when the cluster runs on the same machine. +They connect to the storage through the external address, which is detected when starting the storage. +If the detected address is not reachable from the functions, e.g., because the machine has multiple network interfaces or the storage runs on a different host, override it without changing any files: ```bash -# For a LAN-local address (e.g., on CloudLab) -jq '.deployment.openwhisk.storage.nosql.scylladb.address = "10.10.1.15:9012"' configs/openwhisk.json | sponge configs/openwhisk.json +sebs benchmark invoke 210.thumbnailer test --config configs/openwhisk.json --storage-configuration storage.json --storage-address 10.10.1.15 ``` -You can validate the configuration of ScyllaDB with an HTTP request by using `curl`: +The override applies to all storage instances, each with its own mapped port. Alternatively, provide the address once when starting the storage with `--external-address`. + +You can validate that the storage is reachable with an HTTP request to Minio's health endpoint and ScyllaDB's root endpoint: ```bash -curl -i 10.10.1.15:9012 +$ curl -i 10.10.1.15:9011/minio/health/live HTTP/1.1 200 OK -Content-Length: 26 -Content-Type: text/plain -Date: Sun, 07 Dec 2025 14:07:29 GMT -Server: Seastar httpd +... +Server: MinIO -healthy: 192.168.0.20:9012 +$ curl -i 10.10.1.15:9012 +HTTP/1.1 200 OK +... +healthy: 10.10.1.15:9012 ``` ## Lifecycle Management diff --git a/install.py b/install.py index 879ac16e..53a04c38 100755 --- a/install.py +++ b/install.py @@ -73,7 +73,7 @@ def execute(cmd, cwd=None): print("Install Python dependencies for local") execute(". {}/bin/activate && pip3 install -r requirements.local.txt".format(env_dir)) print("Initialize Docker image for local storage.") - execute("docker pull minio/minio:latest") + execute("docker pull quay.io/minio/minio:latest") # One of the installed dependencies causes a downgrade, which in turns breaks static typing. print("Update typing-extensions (resolving bug with mypy)") diff --git a/sebs/cli.py b/sebs/cli.py index d99daf94..1f051046 100755 --- a/sebs/cli.py +++ b/sebs/cli.py @@ -13,7 +13,7 @@ import os import sys import traceback -from typing import cast, List, Optional +from typing import cast, Dict, List, Optional import click import docker @@ -144,6 +144,62 @@ def wrapper(*args, **kwargs): return wrapper +def storage_params(func): + """Decorator that adds CLI parameters for user-deployed storage.""" + + @click.option( + "--storage-configuration", + type=str, + multiple=True, + help="JSON configuration of deployed storage, as written by 'sebs storage start'.", + ) + @click.option( + "--storage-address", + default=None, + type=str, + help="Override the address (IP or hostname, optional port) that benchmark functions " + "use to reach self-hosted storage. Applied to all storage types.", + ) + @functools.wraps(func) + def wrapper(*args, **kwargs): + """Internal Click wrapper.""" + return func(*args, **kwargs) + + return wrapper + + +def override_storage_address( + config_obj: dict, deployment: str, storage_address: str +) -> Dict[str, str]: + """Override the externally reachable address of all self-hosted storage instances. + + Each storage instance receives the given host combined with its own mapped + port, unless the user already provided a port. + + Args: + config_obj: Full SeBS configuration + deployment: Name of the selected deployment + storage_address: IP address or hostname, optionally with a port + + Returns: + Dict[str, str]: Applied address per storage type + """ + from sebs.utils import resolve_external_address + + applied: Dict[str, str] = {} + storage_cfg = config_obj.get("deployment", {}).get(deployment, {}).get("storage", {}) + for storage_type, type_cfg in storage_cfg.items(): + impl = type_cfg.get("type") + if impl is None or impl not in type_cfg: + continue + impl_cfg = type_cfg[impl] + impl_cfg["external_address"] = resolve_external_address( + storage_address, impl_cfg.get("mapped_port", -1) + ) + applied[storage_type] = impl_cfg["external_address"] + return applied + + def parse_common_params( config, output_dir, @@ -163,6 +219,7 @@ def parse_common_params( initialize_deployment: bool = True, ignore_cache: bool = False, storage_configuration: Optional[List[str]] = None, + storage_address: Optional[str] = None, ): """Parse and process common CLI parameters, initialize SeBS and deployment clients. @@ -207,7 +264,12 @@ def parse_common_params( sebs_client.logging.info(f"Loading storage configuration from {cfg_f}") cfg = json.load(open(cfg_f, "r")) - append_nested_dict(config_obj, ["deployment", deployment, "storage"], cfg) + append_nested_dict(config_obj, ["deployment", selected_deployment, "storage"], cfg) + + if storage_address is not None: + overrides = override_storage_address(config_obj, selected_deployment, storage_address) + for storage_type, address in overrides.items(): + sebs_client.logging.info(f"Using storage address {address} for {storage_type} storage") if initialize_deployment: deployment_client = sebs_client.get_deployment( @@ -273,12 +335,7 @@ def benchmark(): type=str, help="Attach prefix to generated Docker image tag.", ) -@click.option( - "--storage-configuration", - type=str, - multiple=True, - help="JSON configuration of deployed storage.", -) +@storage_params @click.option( "--validate/--no-validate", default=False, @@ -483,12 +540,7 @@ def package( type=str, help="Run only the selected benchmark.", ) -@click.option( - "--storage-configuration", - type=str, - multiple=True, - help="JSON configuration of deployed storage.", -) +@storage_params @click.option( "--selected-architecture/--all-architectures", type=bool, @@ -506,6 +558,7 @@ def regression( benchmark_input_size, benchmark_name, storage_configuration, + storage_address, selected_architecture, filter_output, **kwargs, @@ -523,6 +576,7 @@ def regression( (config, output_dir, logging_filename, sebs_client, _) = parse_common_params( initialize_deployment=False, storage_configuration=storage_configuration, + storage_address=storage_address, **kwargs, ) architecture = config["experiments"]["architecture"] if selected_architecture else None @@ -563,8 +617,21 @@ def storage(): default=True, help="Remove containers after stopping.", ) -def storage_start(storage, config, output_json, remove_containers): - """Start local storage instances (object storage, NoSQL, or both).""" +@click.option( + "--external-address", + default=None, + type=str, + help="Address (IP or hostname) advertised to benchmark functions. Each storage instance " + "appends its mapped port. Defaults to the IP of the host's default network interface.", +) +def storage_start(storage, config, output_json, remove_containers, external_address): + """Start local storage instances (object storage, NoSQL, or both). + + The written configuration contains two addresses per storage instance: the + address used by SeBS on this host, and the external address advertised to + benchmark functions, which is required for functions running outside of the + Docker bridge network, e.g., in a Kubernetes cluster. + """ import docker sebs.utils.global_logging() @@ -578,11 +645,14 @@ def storage_start(storage, config, output_json, remove_containers): storage_config = sebs.SeBS.get_storage_config_implementation(storage_type_enum) config = storage_config.deserialize(user_storage_config["object"][storage_type_name]) config.remove_containers = remove_containers + if external_address is not None: + config.external_address = external_address storage_instance = storage_type(docker.from_env(), None, None, True) storage_instance.config = config storage_instance.start() + storage_instance.check_external_address() user_storage_config["object"][storage_type_name] = storage_instance.serialize() else: @@ -596,10 +666,13 @@ def storage_start(storage, config, output_json, remove_containers): storage_config = sebs.SeBS.get_nosql_config_implementation(storage_type_enum) config = storage_config.deserialize(user_storage_config["nosql"][storage_type_name]) config.remove_containers = remove_containers + if external_address is not None: + config.external_address = external_address storage_instance = storage_type(docker.from_env(), None, config) storage_instance.start() + storage_instance.check_external_address() key, value = storage_instance.serialize() user_storage_config["nosql"][key] = value @@ -662,12 +735,7 @@ def local(): @click.argument("benchmark-input-size", type=click.Choice(["test", "small", "large"])) @click.argument("output", type=str) @click.option("--deployments", default=1, type=int, help="Number of deployed containers.") -@click.option( - "--storage-configuration", - type=str, - multiple=True, - help="JSON configuration of deployed storage.", -) +@storage_params @click.option( "--measure-interval", type=int, @@ -692,6 +760,7 @@ def start( output, deployments, storage_configuration, + storage_address, measure_interval, remove_containers, architecture, @@ -706,6 +775,7 @@ def start( update_storage=False, deployment="local", storage_configuration=storage_configuration, + storage_address=storage_address, system_variant="package", architecture=architecture, **kwargs, diff --git a/sebs/faas/nosql.py b/sebs/faas/nosql.py index 502b904d..81f3163b 100644 --- a/sebs/faas/nosql.py +++ b/sebs/faas/nosql.py @@ -128,13 +128,18 @@ def update_cache(self, benchmark: str): """ pass - def envs(self) -> dict: + def envs(self, external: bool = True) -> dict: """ Return a dictionary of environment variables that are required by functions to access this NoSQL storage (e.g., connection strings, table names). Default implementation returns an empty dictionary. Subclasses should override if they need to expose environment variables. + Args: + external: For self-hosted storage, advertise the externally reachable + address. Functions running on the same Docker bridge as the storage + (local deployment) must use the internal address instead. + Returns: dict: Dictionary of environment variables """ diff --git a/sebs/local/local.py b/sebs/local/local.py index 99c03c7a..19ef80d3 100644 --- a/sebs/local/local.py +++ b/sebs/local/local.py @@ -270,12 +270,15 @@ def _start_container( "CONTAINER_GID": str(os.getgid()), "CONTAINER_USER": self._system_config.username(self.name(), code_package.language_name), } + # Function containers share the Docker bridge with the storage containers, + # so they use the internal addresses. if self.config.resources.storage_config: - environment = {**self.config.resources.storage_config.envs(), **environment} + storage_envs = self.config.resources.storage_config.envs(external=False) + environment = {**storage_envs, **environment} if code_package.uses_nosql: nosql_storage = self.system_resources.get_nosql_storage() - environment = {**environment, **nosql_storage.envs()} + environment = {**environment, **nosql_storage.envs(external=False)} for original_name, actual_name in nosql_storage.get_tables( code_package.benchmark diff --git a/sebs/openwhisk/openwhisk.py b/sebs/openwhisk/openwhisk.py index 12a5abd2..9250b5d4 100644 --- a/sebs/openwhisk/openwhisk.py +++ b/sebs/openwhisk/openwhisk.py @@ -147,8 +147,11 @@ def shutdown(self) -> None: This method stops storage services if configured and optionally removes the OpenWhisk cluster based on configuration settings. """ - if hasattr(self, "storage") and self.config.shutdownStorage: - self.storage.stop() + if self.config.shutdownStorage: + if self.config.resources.storage_config: + cast(Minio, self.system_resources.get_storage()).stop() + if self.config.resources.nosql_storage_config: + cast(ScyllaDB, self.system_resources.get_nosql_storage()).stop() if self.config.removeCluster: from tools.openwhisk_preparation import delete_cluster # type: ignore diff --git a/sebs/storage/config.py b/sebs/storage/config.py index 883cea42..577e8483 100644 --- a/sebs/storage/config.py +++ b/sebs/storage/config.py @@ -37,9 +37,15 @@ def serialize(self) -> Dict[str, Any]: pass @abstractmethod - def envs(self) -> Dict[str, str]: + def envs(self, external: bool = True) -> Dict[str, str]: """Generate environment variables for the storage configuration. + Args: + external: Advertise the externally reachable address. Functions running + on the same Docker bridge as the storage (local deployment) must use + the internal address instead, as Docker does not route traffic from + the bridge to ports published on the host. + Returns: Dict[str, str]: Environment variables to be set in benchmark runtime """ @@ -55,7 +61,12 @@ class MinioConfig(PersistentStorageConfig): parameters for deploying and connecting to a MinIO instance. Attributes: - address: Network address where MinIO is accessible (auto-detected) + address: Network address used by SeBS itself to reach MinIO (auto-detected). + On Linux this is the container's bridge IP and internal port. + external_address: Network address advertised to benchmark functions, + e.g., the host's IP and the mapped port. Functions running outside + the Docker bridge network (OpenWhisk, Kubernetes) need this address. + Falls back to `address` when empty. mapped_port: Host port mapped to MinIO's internal port 9000 access_key: Access key for MinIO authentication (auto-generated) secret_key: Secret key for MinIO authentication (auto-generated) @@ -68,6 +79,7 @@ class MinioConfig(PersistentStorageConfig): """ address: str = "" + external_address: str = "" mapped_port: int = -1 access_key: str = "" secret_key: str = "" @@ -124,17 +136,21 @@ def serialize(self) -> Dict[str, Any]: """ return self.__dict__ - def envs(self) -> Dict[str, str]: + def envs(self, external: bool = True) -> Dict[str, str]: """Generate environment variables for MinIO configuration. Creates environment variables that can be used by benchmark functions to connect to the MinIO storage instance. + Args: + external: Advertise the externally reachable address instead of the + internal one; see PersistentStorageConfig.envs. + Returns: Dict[str, str]: Environment variables for MinIO connection """ return { - "MINIO_ADDRESS": self.address, + "MINIO_ADDRESS": (self.external_address or self.address) if external else self.address, "MINIO_ACCESS_KEY": self.access_key, "MINIO_SECRET_KEY": self.secret_key, } @@ -174,7 +190,10 @@ class ScyllaDBConfig(NoSQLStorageConfig): the necessary parameters for deploying and connecting to a ScyllaDB instance. Attributes: - address: Network address where ScyllaDB is accessible (auto-detected) + address: Network address used by SeBS itself to reach ScyllaDB (auto-detected). + On Linux this is the container's bridge IP and the Alternator port. + external_address: Network address advertised to benchmark functions, + e.g., the host's IP and the mapped port. Falls back to `address` when empty. mapped_port: Host port mapped to ScyllaDB's Alternator port alternator_port: Internal port for DynamoDB-compatible API (default: 8000) access_key: Access key for DynamoDB API (placeholder value) @@ -188,6 +207,7 @@ class ScyllaDBConfig(NoSQLStorageConfig): """ address: str = "" + external_address: str = "" mapped_port: int = -1 alternator_port: int = 8000 access_key: str = "None" diff --git a/sebs/storage/minio.py b/sebs/storage/minio.py index 37d4bb14..12790927 100644 --- a/sebs/storage/minio.py +++ b/sebs/storage/minio.py @@ -21,7 +21,7 @@ from sebs.faas.config import Resources from sebs.faas.storage import PersistentStorage from sebs.storage.config import MinioConfig -from sebs.utils import is_linux +from sebs.utils import is_linux, probe_http, resolve_external_address class Minio(PersistentStorage): @@ -58,6 +58,9 @@ def deployment_name() -> str: # The region setting is required by S3 API but not used for local MinIO MINIO_REGION = "us-east-1" + # Docker Hub no longer serves minio/minio; the same images are published on quay.io + MINIO_IMAGE = "quay.io/minio/minio" + def __init__( self, docker_client: docker.DockerClient, @@ -164,7 +167,7 @@ def start(self) -> None: self.logging.info(f"Starting storage Minio on port {self._cfg.mapped_port}") # Run the MinIO container self._storage_container = self._docker_client.containers.run( - f"minio/minio:{self._cfg.version}", + f"{self.MINIO_IMAGE}:{self._cfg.version}", command="server /data", network_mode="bridge", user=os.getuid(), @@ -195,9 +198,15 @@ def configure_connection(self) -> None: Determines the appropriate address to connect to the MinIO container based on the host platform. For Linux, it uses the container's - bridge IP address, hile for Windows, macOS, or WSL it uses + bridge IP address, while for Windows, macOS, or WSL it uses localhost with the mapped port. + Additionally, it determines the address advertised to benchmark + functions: the user-provided external address, or the host's + default-route IP combined with the mapped port. This address is + reachable from outside the Docker bridge network, e.g., from + Kubernetes pods. + Raises: RuntimeError: If the MinIO container is not available or if the IP address cannot be detected @@ -236,10 +245,50 @@ def configure_connection(self) -> None: f"{self._cfg.instance_id}" ) self.logging.info("Starting minio instance at {}".format(self._cfg.address)) + self.configure_external_address() # Create the connection using the configured address self.connection = self.get_connection() + def configure_external_address(self) -> None: + """Determine the address advertised to benchmark functions. + + Uses the user-provided external address, or the host's default-route IP, + combined with the mapped port. + """ + self._cfg.external_address = resolve_external_address( + self._cfg.external_address, self._cfg.mapped_port + ) + if self._cfg.external_address: + self.logging.info(f"Minio advertised to functions at {self._cfg.external_address}") + else: + self.logging.warning( + "Could not detect the host's IP address. Functions running outside of the " + "Docker bridge network will not reach Minio; provide --external-address." + ) + + def check_external_address(self) -> bool: + """Verify that Minio is reachable through the address advertised to functions. + + Failures are reported as warnings, since the host running SeBS is not + always able to reach the same network as benchmark functions. + + Returns: + bool: True if the probe succeeded + """ + if not self._cfg.external_address: + return False + url = f"http://{self._cfg.external_address}/minio/health/live" + error = probe_http(url, timeout_seconds=15) + if error is None: + self.logging.info(f"Minio is reachable at {url}") + return True + self.logging.warning( + f"Minio is not reachable at {url}: {error}. Benchmark functions might not be " + f"able to reach the storage. Verify the address with: curl -i {url}" + ) + return False + def stop(self) -> None: """ Stop the MinIO container. diff --git a/sebs/storage/resources.py b/sebs/storage/resources.py index 0f190ac6..29547315 100644 --- a/sebs/storage/resources.py +++ b/sebs/storage/resources.py @@ -139,7 +139,7 @@ def _deserialize_storage( cached_config is not None and "resources" in cached_config and "storage" in cached_config["resources"] - and "object" in cached_config["resources"]["storage"] + and storage_type in cached_config["resources"]["storage"] ): storage_impl = cached_config["resources"]["storage"][storage_type]["type"] storage_config = cached_config["resources"]["storage"][storage_type][storage_impl] @@ -244,7 +244,9 @@ def get_storage(self, replace_existing: Optional[bool] = None) -> PersistentStor if storage_config is None: self.logging.error( f"The {self._name} deployment is missing the " - "configuration of pre-allocated storage!" + "configuration of pre-allocated storage! Start the storage with " + "'sebs storage start object configs/storage.json --output-json storage.json' " + "and pass the result with '--storage-configuration storage.json'." ) raise RuntimeError(f"Cannot run {self._name} deployment without any object storage") @@ -285,7 +287,9 @@ def get_nosql_storage(self) -> NoSQLStorage: if storage_config is None: self.logging.error( f"The {self._name} deployment is missing the configuration " - "of pre-allocated NoSQL storage!" + "of pre-allocated NoSQL storage! Start the storage with " + "'sebs storage start nosql configs/storage.json --output-json storage.json' " + "and pass the result with '--storage-configuration storage.json'." ) raise RuntimeError("Cannot allocate NoSQL storage!") diff --git a/sebs/storage/scylladb.py b/sebs/storage/scylladb.py index 6c532686..f2a0cbe2 100644 --- a/sebs/storage/scylladb.py +++ b/sebs/storage/scylladb.py @@ -23,6 +23,7 @@ from sebs.faas.nosql import NoSQLStorage from sebs.sebs_types import NoSQLStorage as StorageType from sebs.storage.config import ScyllaDBConfig +from sebs.utils import probe_http, resolve_external_address class ScyllaDB(NoSQLStorage): @@ -208,6 +209,10 @@ def configure_connection(self) -> None: based on the host platform. For Linux, it uses the container's IP address, while for Windows, macOS, or WSL it uses localhost with the mapped port. + Additionally, it determines the address advertised to benchmark + functions: the user-provided external address, or the host's + default-route IP combined with the mapped port. + Creates a boto3 DynamoDB client configured to connect to ScyllaDB's Alternator interface. @@ -244,6 +249,7 @@ def configure_connection(self) -> None: f"{self._cfg.instance_id}" ) self.logging.info("Starting ScyllaDB instance at {}".format(self._cfg.address)) + self.configure_external_address() # Create the DynamoDB client for ScyllaDB's Alternator interface self.client = boto3.client( @@ -254,6 +260,45 @@ def configure_connection(self) -> None: endpoint_url=f"http://{self._cfg.address}", ) + def configure_external_address(self) -> None: + """Determine the address advertised to benchmark functions. + + Uses the user-provided external address, or the host's default-route IP, + combined with the mapped port. + """ + self._cfg.external_address = resolve_external_address( + self._cfg.external_address, self._cfg.mapped_port + ) + if self._cfg.external_address: + self.logging.info(f"ScyllaDB advertised to functions at {self._cfg.external_address}") + else: + self.logging.warning( + "Could not detect the host's IP address. Functions running outside of the " + "Docker bridge network will not reach ScyllaDB; provide --external-address." + ) + + def check_external_address(self) -> bool: + """Verify that ScyllaDB is reachable through the address advertised to functions. + + Failures are reported as warnings, since the host running SeBS is not + always able to reach the same network as benchmark functions. + + Returns: + bool: True if the probe succeeded + """ + if not self._cfg.external_address: + return False + url = f"http://{self._cfg.external_address}/" + error = probe_http(url, timeout_seconds=15) + if error is None: + self.logging.info(f"ScyllaDB is reachable at {url}") + return True + self.logging.warning( + f"ScyllaDB is not reachable at {url}: {error}. Benchmark functions might not be " + f"able to reach the storage. Verify the address with: curl -i {url}" + ) + return False + def stop(self) -> None: """Stop the ScyllaDB container. @@ -266,16 +311,25 @@ def stop(self) -> None: else: self.logging.error("Stopping ScyllaDB was not successful, storage container not known!") - def envs(self) -> Dict[str, str]: + def envs(self, external: bool = True) -> Dict[str, str]: """Generate environment variables for ScyllaDB configuration. Creates environment variables that can be used by benchmark functions to connect to the ScyllaDB storage instance. + Args: + external: Advertise the externally reachable address instead of the + internal one; see NoSQLStorage.envs. + Returns: Dict[str, str]: Environment variables for ScyllaDB connection """ - return {"NOSQL_STORAGE_TYPE": "scylladb", "NOSQL_STORAGE_ENDPOINT": self._cfg.address} + return { + "NOSQL_STORAGE_TYPE": "scylladb", + "NOSQL_STORAGE_ENDPOINT": ( + (self._cfg.external_address or self._cfg.address) if external else self._cfg.address + ), + } def serialize(self) -> Tuple[StorageType, Dict[str, Any]]: """Serialize ScyllaDB configuration to a tuple. diff --git a/sebs/utils.py b/sebs/utils.py index 4ebdcb82..1567b033 100644 --- a/sebs/utils.py +++ b/sebs/utils.py @@ -19,7 +19,9 @@ import click import datetime import platform +import socket import threading +import time import re from pathlib import Path @@ -191,7 +193,7 @@ def append_nested_dict(cfg: dict, keys: List[str], value: Optional[dict]) -> Non # make sure parent keys exist for key in keys[:-1]: cfg = cfg.setdefault(key, {}) - cfg[keys[-1]] = {**cfg[keys[-1]], **value} + cfg[keys[-1]] = {**cfg.get(keys[-1], {}), **value} def find(name: str, path: str) -> Optional[str]: @@ -708,6 +710,75 @@ def is_linux() -> bool: return platform.system() == "Linux" and "microsoft" not in platform.release().lower() +def detect_external_address() -> str: + """ + Detect the IP address of the host on its default-route network interface. + + No packet is sent: connecting a UDP socket only selects the outgoing interface. + + Returns: + str: IPv4 address of the default-route interface, or an empty string + if the detection fails, e.g., on a host without a default route. + """ + try: + with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as sock: + sock.connect(("8.8.8.8", 80)) + return sock.getsockname()[0] + except OSError: + return "" + + +def resolve_external_address(address: str, port: int) -> str: + """ + Determine the address advertised to benchmark functions for self-hosted storage. + + Functions running outside the Docker bridge network, e.g., in a Kubernetes + cluster, reach the storage through the host's IP and the port mapped on the host. + + Args: + address: User-provided IP or hostname, optionally with a port. When empty, + the IP of the host's default-route interface is used. + port: Port mapped on the host, appended when the address has no port. + + Returns: + str: Address in the form "host:port", or an empty string if no address was + given and the detection failed. + """ + host = address if address else detect_external_address() + if not host: + return "" + has_port = "]:" in host if host.startswith("[") else ":" in host + return host if has_port else f"{host}:{port}" + + +def probe_http(url: str, timeout_seconds: int) -> Optional[str]: + """ + Repeatedly query an HTTP endpoint until it answers with status 200. + + Args: + url: Endpoint to query + timeout_seconds: How long to keep retrying, e.g., while a server starts up + + Returns: + Optional[str]: None on success, otherwise a description of the last failure + """ + import urllib3 + + http = urllib3.PoolManager(timeout=urllib3.util.Timeout(connect=2, read=2)) + last_error = "timeout" + deadline = time.monotonic() + timeout_seconds + while time.monotonic() < deadline: + try: + resp = http.request("GET", url, retries=False) + if resp.status == 200: + return None + last_error = f"status {resp.status}" + except Exception as e: + last_error = str(e) + time.sleep(0.5) + return last_error + + def catch_interrupt() -> None: """ Set up a signal handler to catch interrupt signals (Ctrl+C).